From 7b5f25ecd05b8e37e29ce3c66b1c381c30b62b92 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 12:40:11 +0100 Subject: [PATCH 01/67] resolve go.mod conflicts --- core/services/job/models.go | 1 + .../conversions/conversions.go | 4 + core/services/workflows/artifacts/v2/orm.go | 9 +- core/services/workflows/syncer/fetcher.go | 18 +- core/services/workflows/syncer/v2/fetcher.go | 18 +- core/services/workflows/syncer/v2/handler.go | 119 +++++- .../workflows/syncer/v2/handler_test.go | 258 +++++++++++++ .../workflows/v2/confidential_module.go | 188 ++++++++++ .../workflows/v2/confidential_module_test.go | 343 ++++++++++++++++++ .../0295_add_workflow_attributes_column.sql | 5 + 10 files changed, 938 insertions(+), 25 deletions(-) create mode 100644 core/services/workflows/v2/confidential_module.go create mode 100644 core/services/workflows/v2/confidential_module_test.go create mode 100644 core/store/migrate/migrations/0295_add_workflow_attributes_column.sql diff --git a/core/services/job/models.go b/core/services/job/models.go index 20e7da2c643..2d257a33ca5 100644 --- a/core/services/job/models.go +++ b/core/services/job/models.go @@ -930,6 +930,7 @@ type WorkflowSpec struct { CreatedAt time.Time `toml:"-"` UpdatedAt time.Time `toml:"-"` SpecType WorkflowSpecType `toml:"spec_type" db:"spec_type"` + Attributes []byte `db:"attributes"` sdkWorkflow *sdk.WorkflowSpec rawSpec []byte config []byte diff --git a/core/services/standardcapabilities/conversions/conversions.go b/core/services/standardcapabilities/conversions/conversions.go index b9a233243af..d623fa9a409 100644 --- a/core/services/standardcapabilities/conversions/conversions.go +++ b/core/services/standardcapabilities/conversions/conversions.go @@ -50,6 +50,8 @@ func GetCapabilityIDFromCommand(command string, config string) string { return "http-trigger@1.0.0-alpha" case "http_action": return "http-actions@1.0.0-alpha" // plural "actions" + case "mock": + return "mock@1.0.0" default: return "" } @@ -71,6 +73,8 @@ func GetCommandFromCapabilityID(capabilityID string) string { return "http_trigger" case strings.HasPrefix(capabilityID, "http-actions"): return "http_action" + case strings.HasPrefix(capabilityID, "mock"): + return "mock" default: return "" } diff --git a/core/services/workflows/artifacts/v2/orm.go b/core/services/workflows/artifacts/v2/orm.go index 7b37afda16b..6d8887dbe06 100644 --- a/core/services/workflows/artifacts/v2/orm.go +++ b/core/services/workflows/artifacts/v2/orm.go @@ -63,7 +63,8 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) config_url, created_at, updated_at, - spec_type + spec_type, + attributes ) VALUES ( :workflow, :config, @@ -76,7 +77,8 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) :config_url, :created_at, :updated_at, - :spec_type + :spec_type, + :attributes ) ON CONFLICT (workflow_id) DO UPDATE SET workflow = EXCLUDED.workflow, @@ -89,7 +91,8 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) config_url = EXCLUDED.config_url, created_at = EXCLUDED.created_at, updated_at = EXCLUDED.updated_at, - spec_type = EXCLUDED.spec_type + spec_type = EXCLUDED.spec_type, + attributes = EXCLUDED.attributes RETURNING id ` diff --git a/core/services/workflows/syncer/fetcher.go b/core/services/workflows/syncer/fetcher.go index c959b6b4da7..33ba4b10a2e 100644 --- a/core/services/workflows/syncer/fetcher.go +++ b/core/services/workflows/syncer/fetcher.go @@ -177,12 +177,20 @@ func newFileFetcher(basePath string, lggr logger.Logger) types.FetcherFunc { if err != nil { return nil, fmt.Errorf("invalid URL: %w", err) } - fullPath := filepath.Clean(u.Path) - // ensure that the incoming request URL is either relative or absolute but within the basePath - if !filepath.IsAbs(fullPath) { - // If it's not absolute, we assume it's relative to the basePath - fullPath = filepath.Join(basePath, fullPath) + var fullPath string + if u.Scheme == "http" || u.Scheme == "https" { + // For HTTP(S) URLs, extract just the filename and resolve against basePath. + // This supports confidential workflows where the on-chain URL must be HTTP + // (so the enclave can fetch the binary), but the syncer reads from the local filesystem. + fullPath = filepath.Join(basePath, filepath.Base(u.Path)) + } else { + fullPath = filepath.Clean(u.Path) + // ensure that the incoming request URL is either relative or absolute but within the basePath + if !filepath.IsAbs(fullPath) { + // If it's not absolute, we assume it's relative to the basePath + fullPath = filepath.Join(basePath, fullPath) + } } if !strings.HasPrefix(fullPath, basePath) { return nil, fmt.Errorf("request URL %s is not within the basePath %s", fullPath, basePath) diff --git a/core/services/workflows/syncer/v2/fetcher.go b/core/services/workflows/syncer/v2/fetcher.go index e86f998f6bb..4c79fffbe24 100644 --- a/core/services/workflows/syncer/v2/fetcher.go +++ b/core/services/workflows/syncer/v2/fetcher.go @@ -211,12 +211,20 @@ func newFileFetcher(basePath string, lggr logger.Logger) types.FetcherFunc { if err != nil { return nil, fmt.Errorf("invalid URL: %w", err) } - fullPath := filepath.Clean(u.Path) - // ensure that the incoming request URL is either relative or absolute but within the basePath - if !filepath.IsAbs(fullPath) { - // If it's not absolute, we assume it's relative to the basePath - fullPath = filepath.Join(basePath, fullPath) + var fullPath string + if u.Scheme == "http" || u.Scheme == "https" { + // For HTTP(S) URLs, extract just the filename and resolve against basePath. + // This supports confidential workflows where the on-chain URL must be HTTP + // (so the enclave can fetch the binary), but the syncer reads from the local filesystem. + fullPath = filepath.Join(basePath, filepath.Base(u.Path)) + } else { + fullPath = filepath.Clean(u.Path) + // ensure that the incoming request URL is either relative or absolute but within the basePath + if !filepath.IsAbs(fullPath) { + // If it's not absolute, we assume it's relative to the basePath + fullPath = filepath.Join(basePath, fullPath) + } } if !strings.HasPrefix(fullPath, basePath) { return nil, fmt.Errorf("request URL %s is not within the basePath %s", fullPath, basePath) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 6dd3dbf9f06..b298db91722 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -558,6 +558,7 @@ func (h *eventHandler) createWorkflowSpec(ctx context.Context, payload WorkflowR SpecType: job.WASMFile, BinaryURL: payload.BinaryURL, ConfigURL: payload.ConfigURL, + Attributes: payload.Attributes, } if _, err = h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, entry); err != nil { @@ -819,6 +820,15 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp return fmt.Errorf("invalid workflow name: %w", err) } + confidential, err := v2.IsConfidential(spec.Attributes) + if err != nil { + return fmt.Errorf("failed to parse workflow attributes: %w", err) + } + if confidential { + h.lggr.Infow("routing workflow to confidential execution", "workflowID", spec.WorkflowID) + return h.tryConfidentialEngineCreate(ctx, spec, wid, workflowName, decodedBinary, source) + } + // Create a channel to receive the initialization result. // This allows us to wait for the engine to complete initialization (including trigger subscriptions) // before emitting the workflowActivated event, ensuring the event accurately reflects deployment status. @@ -841,7 +851,21 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp return fmt.Errorf("failed to create workflow engine: %w", err) } - if err = engine.Start(ctx); err != nil { + return h.startAndRegisterEngine(ctx, engine, initDone, wid, spec.WorkflowID, source) +} + +// startAndRegisterEngine starts a workflow engine, waits for initialization to +// complete, and registers it in the engine registry. Used by both the normal +// and confidential engine creation paths. +func (h *eventHandler) startAndRegisterEngine( + ctx context.Context, + engine services.Service, + initDone <-chan error, + wid types.WorkflowID, + workflowID string, + source string, +) error { + if err := engine.Start(ctx); err != nil { return fmt.Errorf("failed to start workflow engine: %w", err) } @@ -849,32 +873,25 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp // This ensures we don't emit workflowActivated events before the engine initializes successfully. select { case <-ctx.Done(): - // Context cancelled while waiting for initialization if closeErr := engine.Close(); closeErr != nil { - h.lggr.Errorw("failed to close engine after context cancellation", "error", closeErr, "workflowID", spec.WorkflowID) + h.lggr.Errorw("failed to close engine after context cancellation", "error", closeErr, "workflowID", workflowID) } return fmt.Errorf("context cancelled while waiting for engine initialization: %w", ctx.Err()) case initErr := <-initDone: if initErr != nil { - // Engine initialization failed (e.g., trigger subscription failed) // TODO (cre-1482) add logic to mark a deployment as failed to avoid churn. - // Currently, failed deployments will be retried on each poll cycle (with exponential backoff). - // If the failure is due to user error (e.g., invalid trigger config), this causes unnecessary retries. - // Consider marking the workflow spec as "failed" in the database and requiring workflow redeployment. if closeErr := engine.Close(); closeErr != nil { - h.lggr.Errorw("failed to close engine after initialization failure", "error", closeErr, "workflowID", spec.WorkflowID) + h.lggr.Errorw("failed to close engine after initialization failure", "error", closeErr, "workflowID", workflowID) } return fmt.Errorf("engine initialization failed: %w", initErr) } } - // Engine is fully initialized, add to registry with source tracking if err := h.engineRegistry.Add(wid, source, engine); err != nil { if closeErr := engine.Close(); closeErr != nil { return fmt.Errorf("failed to close workflow engine: %w during invariant violation: %w", closeErr, err) } - // Check for WorkflowID collision across sources if errors.Is(err, ErrAlreadyExists) { existingEntry, found := h.engineRegistry.Get(wid) if found { @@ -886,13 +903,91 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp } } - // This shouldn't happen because we call the handler serially and - // check for running engines above, see the call to engineRegistry.Contains. return fmt.Errorf("invariant violation: %w", err) } return nil } +// tryConfidentialEngineCreate creates a V2 engine backed by a ConfidentialModule +// instead of a local WASM module. The ConfidentialModule delegates execution to +// the confidential-workflows capability which runs the WASM inside a TEE. +func (h *eventHandler) tryConfidentialEngineCreate( + ctx context.Context, + spec *job.WorkflowSpec, + wid types.WorkflowID, + workflowName types.WorkflowName, + decodedBinary []byte, + source string, +) error { + attrs, err := v2.ParseWorkflowAttributes(spec.Attributes) + if err != nil { + return fmt.Errorf("failed to parse workflow attributes: %w", err) + } + + binaryHash := v2.ComputeBinaryHash(decodedBinary) + + lggr := logger.Named(h.lggr, "WorkflowEngine.ConfidentialModule") + lggr = logger.With(lggr, "workflowID", spec.WorkflowID, "workflowName", spec.WorkflowName, "workflowOwner", spec.WorkflowOwner) + + module := v2.NewConfidentialModule( + h.capRegistry, + spec.BinaryURL, + binaryHash, + spec.WorkflowID, + spec.WorkflowOwner, + workflowName.String(), + spec.WorkflowTag, + attrs.VaultDonSecrets, + lggr, + ) + + initDone := make(chan error, 1) + + cfg := &v2.EngineConfig{ + Lggr: h.lggr, + Module: module, + WorkflowConfig: []byte(spec.Config), + CapRegistry: h.capRegistry, + DonSubscriber: h.workflowDonSubscriber, + UseLocalTimeProvider: h.useLocalTimeProvider, + DonTimeStore: h.donTimeStore, + ExecutionsStore: h.workflowStore, + WorkflowID: spec.WorkflowID, + WorkflowOwner: spec.WorkflowOwner, + WorkflowName: workflowName, + WorkflowTag: spec.WorkflowTag, + WorkflowEncryptionKey: h.workflowEncryptionKey, + + LocalLimits: v2.EngineLimits{}, + LocalLimiters: h.engineLimiters, + GlobalExecutionConcurrencyLimiter: h.workflowLimits, + + BeholderEmitter: h.emitter, + BillingClient: h.billingClient, + + WorkflowRegistryAddress: h.workflowRegistryAddress, + WorkflowRegistryChainSelector: h.workflowRegistryChainSelector, + OrgResolver: h.orgResolver, + SecretsFetcher: h.secretsFetcher, + FeatureFlags: h.featureFlags, + } + + existingHook := cfg.Hooks.OnInitialized + cfg.Hooks.OnInitialized = func(err error) { + initDone <- err + if existingHook != nil { + existingHook(err) + } + } + + engine, err := v2.NewEngine(cfg) + if err != nil { + return fmt.Errorf("failed to create confidential workflow engine: %w", err) + } + + return h.startAndRegisterEngine(ctx, engine, initDone, wid, spec.WorkflowID, source) +} + // logCustMsg emits a custom message to the external sink and logs an error if that fails. func logCustMsg(ctx context.Context, cma custmsg.MessageEmitter, msg string, log logger.Logger) { err := cma.Emit(ctx, msg) diff --git a/core/services/workflows/syncer/v2/handler_test.go b/core/services/workflows/syncer/v2/handler_test.go index ad7b80d06b9..775f059613e 100644 --- a/core/services/workflows/syncer/v2/handler_test.go +++ b/core/services/workflows/syncer/v2/handler_test.go @@ -618,6 +618,264 @@ func Test_workflowRegisteredHandler(t *testing.T) { } } +func Test_workflowRegisteredHandler_confidentialRouting(t *testing.T) { + t.Run("confidential workflow bypasses engine factory and routes to confidential path", func(t *testing.T) { + var ( + ctx = testutils.Context(t) + lggr = logger.TestLogger(t) + lf = limits.Factory{Logger: lggr} + db = pgtest.NewSqlxDB(t) + orm = artifacts.NewWorkflowRegistryDS(db, lggr) + emitter = custmsg.NewLabeler() + + binary = wasmtest.CreateTestBinary(binaryCmd, true, t) + encodedBinary = []byte(base64.StdEncoding.EncodeToString(binary)) + config = []byte("") + wfOwner = []byte("0xOwner") + workflowEncryptionKey = workflowkey.MustNewXXXTestingOnly(big.NewInt(1)) + ) + + giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, "workflow-name", binary, config, "") + require.NoError(t, err) + wfIDString := hex.EncodeToString(giveWFID[:]) + + binaryURL := "http://example.com/" + wfIDString + "/binary" + configURL := "http://example.com/" + wfIDString + "/config" + signedURLParameter := "?auth=abc123" + signedBinaryURL := binaryURL + signedURLParameter + signedConfigURL := configURL + signedURLParameter + + fetcher := newMockFetcher(map[string]mockFetchResp{ + wfIDString + "-ARTIFACT_TYPE_BINARY": {Body: []byte(signedBinaryURL), Err: nil}, + wfIDString + "-ARTIFACT_TYPE_CONFIG": {Body: []byte(signedConfigURL), Err: nil}, + signedBinaryURL: {Body: encodedBinary, Err: nil}, + signedConfigURL: {Body: config, Err: nil}, + }) + artifactStore, err := artifacts.NewStore(lggr, orm, fetcher.FetcherFunc(), fetcher.RetrieverFunc(), clockwork.NewFakeClock(), workflowkey.Key{}, custmsg.NewLabeler(), lf, artifacts.WithConfig(artifacts.StoreConfig{ + ArtifactStorageHost: "example.com", + })) + require.NoError(t, err) + + er := NewEngineRegistry() + + // Track whether the engine factory is called. The confidential path + // should bypass it entirely. + factoryCalled := false + trackingFactory := func(ctx context.Context, wfid string, owner string, name types.WorkflowName, tag string, config []byte, binary []byte, initDone chan<- error) (services.Service, error) { + factoryCalled = true + if initDone != nil { + initDone <- nil + } + return &mockEngine{}, nil + } + + wfStore := store.NewInMemoryStore(lggr, clockwork.NewFakeClock()) + registry := capabilities.NewRegistry(lggr) + registry.SetLocalRegistry(&capabilities.TestMetadataRegistry{}) + limiters, err := v2.NewLimiters(lf, nil) + require.NoError(t, err) + rl, err := ratelimiter.NewRateLimiter(rlConfig) + require.NoError(t, err) + workflowLimits, err := syncerlimiter.NewWorkflowLimits(lggr, syncerlimiter.Config{Global: 200, PerOwner: 200}, lf) + require.NoError(t, err) + + h, err := NewEventHandler(lggr, wfStore, nil, true, registry, er, emitter, limiters, nil, rl, workflowLimits, artifactStore, workflowEncryptionKey, &testDonNotifier{}, + WithEngineRegistry(er), + WithEngineFactoryFn(trackingFactory), + ) + require.NoError(t, err) + servicetest.Run(t, h) + + event := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: giveWFID, + WorkflowOwner: wfOwner, + WorkflowName: "workflow-name", + WorkflowTag: "workflow-tag", + BinaryURL: binaryURL, + ConfigURL: configURL, + Attributes: []byte(`{"confidential":true,"vault_don_secrets":[{"key":"API_KEY"}]}`), + } + + ctx = contexts.WithCRE(ctx, contexts.CRE{Owner: hex.EncodeToString(wfOwner), Workflow: wfIDString}) + err = h.workflowRegisteredEvent(ctx, event) + + // The confidential path creates a real v2.Engine. With test data + // (non-hex owner), engine creation fails. The error comes from the + // confidential path, proving routing worked correctly. + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to create confidential workflow engine") + + // The engine factory must NOT have been called; the confidential path + // bypasses it. + assert.False(t, factoryCalled, "engine factory should not be called for confidential workflows") + + // The engine should NOT be in the registry since init failed. + _, ok := er.Get(giveWFID) + assert.False(t, ok, "engine should not be registered after failed init") + }) + + t.Run("non-confidential workflow uses engine factory", func(t *testing.T) { + var ( + ctx = testutils.Context(t) + lggr = logger.TestLogger(t) + lf = limits.Factory{Logger: lggr} + db = pgtest.NewSqlxDB(t) + orm = artifacts.NewWorkflowRegistryDS(db, lggr) + emitter = custmsg.NewLabeler() + + binary = wasmtest.CreateTestBinary(binaryCmd, true, t) + encodedBinary = []byte(base64.StdEncoding.EncodeToString(binary)) + config = []byte("") + wfOwner = []byte("0xOwner") + workflowEncryptionKey = workflowkey.MustNewXXXTestingOnly(big.NewInt(1)) + ) + + giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, "workflow-name", binary, config, "") + require.NoError(t, err) + wfIDString := hex.EncodeToString(giveWFID[:]) + + binaryURL := "http://example.com/" + wfIDString + "/binary" + configURL := "http://example.com/" + wfIDString + "/config" + signedURLParameter := "?auth=abc123" + signedBinaryURL := binaryURL + signedURLParameter + signedConfigURL := configURL + signedURLParameter + + fetcher := newMockFetcher(map[string]mockFetchResp{ + wfIDString + "-ARTIFACT_TYPE_BINARY": {Body: []byte(signedBinaryURL), Err: nil}, + wfIDString + "-ARTIFACT_TYPE_CONFIG": {Body: []byte(signedConfigURL), Err: nil}, + signedBinaryURL: {Body: encodedBinary, Err: nil}, + signedConfigURL: {Body: config, Err: nil}, + }) + artifactStore, err := artifacts.NewStore(lggr, orm, fetcher.FetcherFunc(), fetcher.RetrieverFunc(), clockwork.NewFakeClock(), workflowkey.Key{}, custmsg.NewLabeler(), lf, artifacts.WithConfig(artifacts.StoreConfig{ + ArtifactStorageHost: "example.com", + })) + require.NoError(t, err) + + er := NewEngineRegistry() + + factoryCalled := false + trackingFactory := func(ctx context.Context, wfid string, owner string, name types.WorkflowName, tag string, config []byte, binary []byte, initDone chan<- error) (services.Service, error) { + factoryCalled = true + if initDone != nil { + initDone <- nil + } + return &mockEngine{}, nil + } + + wfStore := store.NewInMemoryStore(lggr, clockwork.NewFakeClock()) + registry := capabilities.NewRegistry(lggr) + registry.SetLocalRegistry(&capabilities.TestMetadataRegistry{}) + limiters, err := v2.NewLimiters(lf, nil) + require.NoError(t, err) + rl, err := ratelimiter.NewRateLimiter(rlConfig) + require.NoError(t, err) + workflowLimits, err := syncerlimiter.NewWorkflowLimits(lggr, syncerlimiter.Config{Global: 200, PerOwner: 200}, lf) + require.NoError(t, err) + + h, err := NewEventHandler(lggr, wfStore, nil, true, registry, er, emitter, limiters, nil, rl, workflowLimits, artifactStore, workflowEncryptionKey, &testDonNotifier{}, + WithEngineRegistry(er), + WithEngineFactoryFn(trackingFactory), + ) + require.NoError(t, err) + servicetest.Run(t, h) + + event := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: giveWFID, + WorkflowOwner: wfOwner, + WorkflowName: "workflow-name", + WorkflowTag: "workflow-tag", + BinaryURL: binaryURL, + ConfigURL: configURL, + // No Attributes, or non-confidential attributes. + } + + ctx = contexts.WithCRE(ctx, contexts.CRE{Owner: hex.EncodeToString(wfOwner), Workflow: wfIDString}) + err = h.workflowRegisteredEvent(ctx, event) + require.NoError(t, err) + + assert.True(t, factoryCalled, "engine factory should be called for non-confidential workflows") + + engine, ok := er.Get(giveWFID) + require.True(t, ok, "engine should be registered") + require.NoError(t, engine.Ready()) + }) + + t.Run("malformed attributes returns error", func(t *testing.T) { + var ( + ctx = testutils.Context(t) + lggr = logger.TestLogger(t) + lf = limits.Factory{Logger: lggr} + db = pgtest.NewSqlxDB(t) + orm = artifacts.NewWorkflowRegistryDS(db, lggr) + emitter = custmsg.NewLabeler() + + binary = wasmtest.CreateTestBinary(binaryCmd, true, t) + encodedBinary = []byte(base64.StdEncoding.EncodeToString(binary)) + config = []byte("") + wfOwner = []byte("0xOwner") + workflowEncryptionKey = workflowkey.MustNewXXXTestingOnly(big.NewInt(1)) + ) + + giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, "workflow-name", binary, config, "") + require.NoError(t, err) + wfIDString := hex.EncodeToString(giveWFID[:]) + + binaryURL := "http://example.com/" + wfIDString + "/binary" + configURL := "http://example.com/" + wfIDString + "/config" + signedURLParameter := "?auth=abc123" + signedBinaryURL := binaryURL + signedURLParameter + signedConfigURL := configURL + signedURLParameter + + fetcher := newMockFetcher(map[string]mockFetchResp{ + wfIDString + "-ARTIFACT_TYPE_BINARY": {Body: []byte(signedBinaryURL), Err: nil}, + wfIDString + "-ARTIFACT_TYPE_CONFIG": {Body: []byte(signedConfigURL), Err: nil}, + signedBinaryURL: {Body: encodedBinary, Err: nil}, + signedConfigURL: {Body: config, Err: nil}, + }) + artifactStore, err := artifacts.NewStore(lggr, orm, fetcher.FetcherFunc(), fetcher.RetrieverFunc(), clockwork.NewFakeClock(), workflowkey.Key{}, custmsg.NewLabeler(), lf, artifacts.WithConfig(artifacts.StoreConfig{ + ArtifactStorageHost: "example.com", + })) + require.NoError(t, err) + + er := NewEngineRegistry() + wfStore := store.NewInMemoryStore(lggr, clockwork.NewFakeClock()) + registry := capabilities.NewRegistry(lggr) + registry.SetLocalRegistry(&capabilities.TestMetadataRegistry{}) + limiters, err := v2.NewLimiters(lf, nil) + require.NoError(t, err) + rl, err := ratelimiter.NewRateLimiter(rlConfig) + require.NoError(t, err) + workflowLimits, err := syncerlimiter.NewWorkflowLimits(lggr, syncerlimiter.Config{Global: 200, PerOwner: 200}, lf) + require.NoError(t, err) + + h, err := NewEventHandler(lggr, wfStore, nil, true, registry, er, emitter, limiters, nil, rl, workflowLimits, artifactStore, workflowEncryptionKey, &testDonNotifier{}, + WithEngineRegistry(er), + WithEngineFactoryFn(mockEngineFactory), + ) + require.NoError(t, err) + servicetest.Run(t, h) + + event := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: giveWFID, + WorkflowOwner: wfOwner, + WorkflowName: "workflow-name", + WorkflowTag: "workflow-tag", + BinaryURL: binaryURL, + ConfigURL: configURL, + Attributes: []byte(`{not valid json`), + } + + ctx = contexts.WithCRE(ctx, contexts.CRE{Owner: hex.EncodeToString(wfOwner), Workflow: wfIDString}) + err = h.workflowRegisteredEvent(ctx, event) + + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse workflow attributes") + }) +} + type testCase struct { Name string BinaryURLFactory func(string) string diff --git a/core/services/workflows/v2/confidential_module.go b/core/services/workflows/v2/confidential_module.go new file mode 100644 index 00000000000..4cc3761e18a --- /dev/null +++ b/core/services/workflows/v2/confidential_module.go @@ -0,0 +1,188 @@ +package v2 + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/types/core" + "github.com/smartcontractkit/chainlink-common/pkg/workflows/wasm/host" + + confworkflowtypes "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/actions/confidentialworkflow" + sdkpb "github.com/smartcontractkit/chainlink-protos/cre/go/sdk" +) + +const confidentialWorkflowsCapabilityID = "confidential-workflows@1.0.0-alpha" + +// WorkflowAttributes is the JSON structure stored in WorkflowSpec.Attributes. +type WorkflowAttributes struct { + Confidential bool `json:"confidential"` + VaultDonSecrets []SecretIdentifier `json:"vault_don_secrets"` +} + +// SecretIdentifier identifies a secret in VaultDON. +type SecretIdentifier struct { + Key string `json:"key"` + Namespace string `json:"namespace,omitempty"` +} + +// ParseWorkflowAttributes parses the Attributes JSON from a WorkflowSpec. +// Returns a zero-value struct if data is nil or empty. +func ParseWorkflowAttributes(data []byte) (WorkflowAttributes, error) { + var attrs WorkflowAttributes + if len(data) == 0 { + return attrs, nil + } + if err := json.Unmarshal(data, &attrs); err != nil { + return attrs, fmt.Errorf("failed to parse workflow attributes: %w", err) + } + return attrs, nil +} + +// IsConfidential returns true if the Attributes JSON has "confidential": true. +// Returns an error if the attributes contain malformed JSON, so callers can +// fail loudly rather than silently falling through to non-confidential execution. +func IsConfidential(data []byte) (bool, error) { + attrs, err := ParseWorkflowAttributes(data) + if err != nil { + return false, err + } + return attrs.Confidential, nil +} + +// ConfidentialModule implements host.ModuleV2 for confidential workflows. +// Instead of running WASM locally, it delegates execution to the +// confidential-workflows capability via the CapabilitiesRegistry. +type ConfidentialModule struct { + capRegistry core.CapabilitiesRegistry + binaryURL string + binaryHash []byte + workflowID string + workflowOwner string + workflowName string + workflowTag string + vaultDonSecrets []SecretIdentifier + lggr logger.Logger +} + +var _ host.ModuleV2 = (*ConfidentialModule)(nil) + +func NewConfidentialModule( + capRegistry core.CapabilitiesRegistry, + binaryURL string, + binaryHash []byte, + workflowID string, + workflowOwner string, + workflowName string, + workflowTag string, + vaultDonSecrets []SecretIdentifier, + lggr logger.Logger, +) *ConfidentialModule { + return &ConfidentialModule{ + capRegistry: capRegistry, + binaryURL: binaryURL, + binaryHash: binaryHash, + workflowID: workflowID, + workflowOwner: workflowOwner, + workflowName: workflowName, + workflowTag: workflowTag, + vaultDonSecrets: vaultDonSecrets, + lggr: lggr, + } +} + +func (m *ConfidentialModule) Start() {} +func (m *ConfidentialModule) Close() {} +func (m *ConfidentialModule) IsLegacyDAG() bool { return false } + +func (m *ConfidentialModule) Execute( + ctx context.Context, + request *sdkpb.ExecuteRequest, + helper host.ExecutionHelper, +) (*sdkpb.ExecutionResult, error) { + execReqBytes, err := proto.Marshal(request) + if err != nil { + return nil, fmt.Errorf("failed to marshal ExecuteRequest: %w", err) + } + + protoSecrets := make([]*confworkflowtypes.SecretIdentifier, len(m.vaultDonSecrets)) + for i, s := range m.vaultDonSecrets { + // VaultDON treats "main" as the default namespace for secrets. + ns := s.Namespace + if ns == "" { + ns = "main" + } + protoSecrets[i] = &confworkflowtypes.SecretIdentifier{ + Key: s.Key, + Namespace: &ns, + } + } + + capInput := &confworkflowtypes.ConfidentialWorkflowRequest{ + VaultDonSecrets: protoSecrets, + Execution: &confworkflowtypes.WorkflowExecution{ + WorkflowId: m.workflowID, + BinaryUrl: m.binaryURL, + BinaryHash: m.binaryHash, + ExecuteRequest: execReqBytes, + Owner: m.workflowOwner, + ExecutionId: helper.GetWorkflowExecutionID(), + }, + } + + payload, err := anypb.New(capInput) + if err != nil { + return nil, fmt.Errorf("failed to marshal capability payload: %w", err) + } + + executable, err := m.capRegistry.GetExecutable(ctx, confidentialWorkflowsCapabilityID) + if err != nil { + return nil, fmt.Errorf("failed to get confidential-workflows capability: %w", err) + } + + capReq := capabilities.CapabilityRequest{ + Payload: payload, + Method: "Execute", + CapabilityId: confidentialWorkflowsCapabilityID, + Metadata: capabilities.RequestMetadata{ + WorkflowID: m.workflowID, + WorkflowOwner: m.workflowOwner, + WorkflowName: m.workflowName, + WorkflowTag: m.workflowTag, + WorkflowExecutionID: helper.GetWorkflowExecutionID(), + }, + } + + capResp, err := executable.Execute(ctx, capReq) + if err != nil { + return nil, fmt.Errorf("confidential-workflows capability execution failed: %w", err) + } + + if capResp.Payload == nil { + return nil, fmt.Errorf("confidential-workflows capability returned nil payload") + } + + var confResp confworkflowtypes.ConfidentialWorkflowResponse + if err := capResp.Payload.UnmarshalTo(&confResp); err != nil { + return nil, fmt.Errorf("failed to unmarshal capability response: %w", err) + } + + var result sdkpb.ExecutionResult + if err := proto.Unmarshal(confResp.ExecutionResult, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal ExecutionResult: %w", err) + } + + return &result, nil +} + +// ComputeBinaryHash returns the SHA-256 hash of the given binary. +func ComputeBinaryHash(binary []byte) []byte { + h := sha256.Sum256(binary) + return h[:] +} diff --git a/core/services/workflows/v2/confidential_module_test.go b/core/services/workflows/v2/confidential_module_test.go new file mode 100644 index 00000000000..87dab426022 --- /dev/null +++ b/core/services/workflows/v2/confidential_module_test.go @@ -0,0 +1,343 @@ +package v2 + +import ( + "context" + "crypto/sha256" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + regmocks "github.com/smartcontractkit/chainlink-common/pkg/types/core/mocks" + "github.com/smartcontractkit/chainlink-common/pkg/workflows/wasm/host" + + confworkflowtypes "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/actions/confidentialworkflow" + capmocks "github.com/smartcontractkit/chainlink/v2/core/capabilities/mocks" + "github.com/smartcontractkit/chainlink/v2/core/utils/matches" + + sdkpb "github.com/smartcontractkit/chainlink-protos/cre/go/sdk" + valuespb "github.com/smartcontractkit/chainlink-protos/cre/go/values/pb" +) + +// stubExecutionHelper implements host.ExecutionHelper for testing. +type stubExecutionHelper struct { + executionID string +} + +var _ host.ExecutionHelper = (*stubExecutionHelper)(nil) + +func (s *stubExecutionHelper) CallCapability(context.Context, *sdkpb.CapabilityRequest) (*sdkpb.CapabilityResponse, error) { + return nil, nil +} +func (s *stubExecutionHelper) GetSecrets(context.Context, *sdkpb.GetSecretsRequest) ([]*sdkpb.SecretResponse, error) { + return nil, nil +} +func (s *stubExecutionHelper) GetWorkflowExecutionID() string { return s.executionID } +func (s *stubExecutionHelper) GetNodeTime() time.Time { return time.Time{} } +func (s *stubExecutionHelper) GetDONTime() (time.Time, error) { return time.Time{}, nil } +func (s *stubExecutionHelper) EmitUserLog(string) error { return nil } + +func TestParseWorkflowAttributes(t *testing.T) { + t.Run("valid JSON with all fields", func(t *testing.T) { + data := []byte(`{"confidential":true,"vault_don_secrets":[{"key":"API_KEY"},{"key":"SIGNING_KEY","namespace":"custom-ns"}]}`) + attrs, err := ParseWorkflowAttributes(data) + require.NoError(t, err) + assert.True(t, attrs.Confidential) + require.Len(t, attrs.VaultDonSecrets, 2) + assert.Equal(t, "API_KEY", attrs.VaultDonSecrets[0].Key) + assert.Equal(t, "", attrs.VaultDonSecrets[0].Namespace) + assert.Equal(t, "SIGNING_KEY", attrs.VaultDonSecrets[1].Key) + assert.Equal(t, "custom-ns", attrs.VaultDonSecrets[1].Namespace) + }) + + t.Run("empty data returns zero value", func(t *testing.T) { + attrs, err := ParseWorkflowAttributes(nil) + require.NoError(t, err) + assert.False(t, attrs.Confidential) + assert.Nil(t, attrs.VaultDonSecrets) + + attrs, err = ParseWorkflowAttributes([]byte{}) + require.NoError(t, err) + assert.False(t, attrs.Confidential) + }) + + t.Run("non-confidential workflow", func(t *testing.T) { + data := []byte(`{"confidential":false}`) + attrs, err := ParseWorkflowAttributes(data) + require.NoError(t, err) + assert.False(t, attrs.Confidential) + }) + + t.Run("malformed JSON returns error", func(t *testing.T) { + _, err := ParseWorkflowAttributes([]byte(`{not json}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse workflow attributes") + }) +} + +func TestIsConfidential(t *testing.T) { + t.Run("returns true for confidential", func(t *testing.T) { + ok, err := IsConfidential([]byte(`{"confidential":true}`)) + require.NoError(t, err) + assert.True(t, ok) + }) + + t.Run("returns false for non-confidential", func(t *testing.T) { + ok, err := IsConfidential([]byte(`{"confidential":false}`)) + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("returns false for empty data", func(t *testing.T) { + ok, err := IsConfidential(nil) + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("returns error for malformed JSON", func(t *testing.T) { + _, err := IsConfidential([]byte(`broken`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse workflow attributes") + }) +} + +func TestComputeBinaryHash(t *testing.T) { + binary := []byte("hello world") + hash := ComputeBinaryHash(binary) + expected := sha256.Sum256(binary) + assert.Equal(t, expected[:], hash) + + // Deterministic: same input produces same hash. + assert.Equal(t, hash, ComputeBinaryHash(binary)) +} + +func TestConfidentialModule_Execute(t *testing.T) { + ctx := context.Background() + lggr := logger.Nop() + + // Build an ExecuteRequest to send through the module. + execReq := &sdkpb.ExecuteRequest{ + Config: []byte("test-config"), + } + + // Build the expected ExecutionResult that the enclave returns. + expectedResult := &sdkpb.ExecutionResult{ + Result: &sdkpb.ExecutionResult_Value{ + Value: valuespb.NewStringValue("enclave-output"), + }, + } + + // Serialize the result into a ConfidentialWorkflowResponse, as the capability would. + resultBytes, err := proto.Marshal(expectedResult) + require.NoError(t, err) + + confResp := &confworkflowtypes.ConfidentialWorkflowResponse{ + ExecutionResult: resultBytes, + } + respPayload, err := anypb.New(confResp) + require.NoError(t, err) + + t.Run("success", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + execCap := capmocks.NewExecutableCapability(t) + + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(execCap, nil).Once() + + execCap.EXPECT().Execute(matches.AnyContext, mock.MatchedBy(func(req capabilities.CapabilityRequest) bool { + return req.Method == "Execute" && + req.CapabilityId == confidentialWorkflowsCapabilityID && + req.Metadata.WorkflowID == "wf-123" && + req.Metadata.WorkflowOwner == "owner-abc" && + req.Metadata.WorkflowExecutionID == "exec-456" && + req.Payload != nil + })).Return(capabilities.CapabilityResponse{Payload: respPayload}, nil).Once() + + mod := NewConfidentialModule( + capReg, + "https://example.com/binary.wasm", + []byte("fakehash"), + "wf-123", + "owner-abc", + "my-workflow", + "v1", + []SecretIdentifier{ + {Key: "API_KEY"}, + {Key: "SIGNING_KEY", Namespace: "custom-ns"}, + }, + lggr, + ) + + result, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-456"}) + require.NoError(t, err) + require.NotNil(t, result) + + val := result.GetValue() + require.NotNil(t, val) + assert.Equal(t, "enclave-output", val.GetStringValue()) + }) + + t.Run("default namespace is main", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + execCap := capmocks.NewExecutableCapability(t) + + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(execCap, nil).Once() + + // Capture the request to inspect proto secrets. + var capturedReq capabilities.CapabilityRequest + execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). + Run(func(_ context.Context, req capabilities.CapabilityRequest) { + capturedReq = req + }). + Return(capabilities.CapabilityResponse{Payload: respPayload}, nil).Once() + + mod := NewConfidentialModule( + capReg, + "https://example.com/binary.wasm", + []byte("hash"), + "wf-1", "owner", "name", "tag", + []SecretIdentifier{{Key: "SECRET_A"}}, // no namespace + lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-1"}) + require.NoError(t, err) + + // Unmarshal the captured request payload and verify namespace defaulted to "main". + var confReq confworkflowtypes.ConfidentialWorkflowRequest + require.NoError(t, capturedReq.Payload.UnmarshalTo(&confReq)) + require.Len(t, confReq.VaultDonSecrets, 1) + assert.Equal(t, "SECRET_A", confReq.VaultDonSecrets[0].Key) + assert.Equal(t, "main", confReq.VaultDonSecrets[0].GetNamespace()) + }) + + t.Run("GetExecutable error", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(nil, fmt.Errorf("capability not found")).Once() + + mod := NewConfidentialModule( + capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to get confidential-workflows capability") + }) + + t.Run("capability Execute error", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + execCap := capmocks.NewExecutableCapability(t) + + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(execCap, nil).Once() + execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). + Return(capabilities.CapabilityResponse{}, fmt.Errorf("enclave unavailable")).Once() + + mod := NewConfidentialModule( + capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "confidential-workflows capability execution failed") + }) + + t.Run("nil payload in response", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + execCap := capmocks.NewExecutableCapability(t) + + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(execCap, nil).Once() + execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). + Return(capabilities.CapabilityResponse{Payload: nil}, nil).Once() + + mod := NewConfidentialModule( + capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "returned nil payload") + }) + + t.Run("request fields are forwarded correctly", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + execCap := capmocks.NewExecutableCapability(t) + + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(execCap, nil).Once() + + var capturedReq capabilities.CapabilityRequest + execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). + Run(func(_ context.Context, req capabilities.CapabilityRequest) { + capturedReq = req + }). + Return(capabilities.CapabilityResponse{Payload: respPayload}, nil).Once() + + binaryHash := ComputeBinaryHash([]byte("some-binary")) + mod := NewConfidentialModule( + capReg, + "https://example.com/wasm", + binaryHash, + "wf-abc", + "0xowner", + "my-workflow", + "v2", + []SecretIdentifier{ + {Key: "K1", Namespace: "ns1"}, + {Key: "K2"}, + }, + lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-xyz"}) + require.NoError(t, err) + + // Verify metadata. + assert.Equal(t, "Execute", capturedReq.Method) + assert.Equal(t, confidentialWorkflowsCapabilityID, capturedReq.CapabilityId) + assert.Equal(t, "wf-abc", capturedReq.Metadata.WorkflowID) + assert.Equal(t, "0xowner", capturedReq.Metadata.WorkflowOwner) + assert.Equal(t, "my-workflow", capturedReq.Metadata.WorkflowName) + assert.Equal(t, "v2", capturedReq.Metadata.WorkflowTag) + assert.Equal(t, "exec-xyz", capturedReq.Metadata.WorkflowExecutionID) + + // Verify payload contents. + var confReq confworkflowtypes.ConfidentialWorkflowRequest + require.NoError(t, capturedReq.Payload.UnmarshalTo(&confReq)) + + assert.Equal(t, "wf-abc", confReq.Execution.WorkflowId) + assert.Equal(t, "https://example.com/wasm", confReq.Execution.BinaryUrl) + assert.Equal(t, binaryHash, confReq.Execution.BinaryHash) + + // Verify the serialized ExecuteRequest round-trips. + var roundTripped sdkpb.ExecuteRequest + require.NoError(t, proto.Unmarshal(confReq.Execution.ExecuteRequest, &roundTripped)) + assert.Equal(t, execReq.GetConfig(), roundTripped.GetConfig()) + + // Verify secrets. + require.Len(t, confReq.VaultDonSecrets, 2) + assert.Equal(t, "K1", confReq.VaultDonSecrets[0].Key) + assert.Equal(t, "ns1", confReq.VaultDonSecrets[0].GetNamespace()) + assert.Equal(t, "K2", confReq.VaultDonSecrets[1].Key) + assert.Equal(t, "main", confReq.VaultDonSecrets[1].GetNamespace()) + }) +} + +func TestConfidentialModule_InterfaceMethods(t *testing.T) { + mod := &ConfidentialModule{} + + // These are no-ops but should not panic. + mod.Start() + mod.Close() + assert.False(t, mod.IsLegacyDAG()) +} diff --git a/core/store/migrate/migrations/0295_add_workflow_attributes_column.sql b/core/store/migrate/migrations/0295_add_workflow_attributes_column.sql new file mode 100644 index 00000000000..30c85db7289 --- /dev/null +++ b/core/store/migrate/migrations/0295_add_workflow_attributes_column.sql @@ -0,0 +1,5 @@ +-- +goose Up +ALTER TABLE workflow_specs_v2 ADD COLUMN attributes bytea DEFAULT ''; + +-- +goose Down +ALTER TABLE workflow_specs_v2 DROP COLUMN attributes; From 55994205f8e9856015050d6d2d6cd1a8ce937af1 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 13:22:16 +0100 Subject: [PATCH 02/67] Revert file fetcher HTTP URL handling The filepath.Base() code for HTTP URLs in newFileFetcher was dead code. No test or production path sends an HTTP URL to a file-based fetcher. The enclave fetches binaries via its own BinaryFetcher, independent of the node syncer's fetcher. --- core/services/workflows/syncer/fetcher.go | 18 +++++------------- core/services/workflows/syncer/v2/fetcher.go | 18 +++++------------- 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/core/services/workflows/syncer/fetcher.go b/core/services/workflows/syncer/fetcher.go index 33ba4b10a2e..c959b6b4da7 100644 --- a/core/services/workflows/syncer/fetcher.go +++ b/core/services/workflows/syncer/fetcher.go @@ -177,20 +177,12 @@ func newFileFetcher(basePath string, lggr logger.Logger) types.FetcherFunc { if err != nil { return nil, fmt.Errorf("invalid URL: %w", err) } + fullPath := filepath.Clean(u.Path) - var fullPath string - if u.Scheme == "http" || u.Scheme == "https" { - // For HTTP(S) URLs, extract just the filename and resolve against basePath. - // This supports confidential workflows where the on-chain URL must be HTTP - // (so the enclave can fetch the binary), but the syncer reads from the local filesystem. - fullPath = filepath.Join(basePath, filepath.Base(u.Path)) - } else { - fullPath = filepath.Clean(u.Path) - // ensure that the incoming request URL is either relative or absolute but within the basePath - if !filepath.IsAbs(fullPath) { - // If it's not absolute, we assume it's relative to the basePath - fullPath = filepath.Join(basePath, fullPath) - } + // ensure that the incoming request URL is either relative or absolute but within the basePath + if !filepath.IsAbs(fullPath) { + // If it's not absolute, we assume it's relative to the basePath + fullPath = filepath.Join(basePath, fullPath) } if !strings.HasPrefix(fullPath, basePath) { return nil, fmt.Errorf("request URL %s is not within the basePath %s", fullPath, basePath) diff --git a/core/services/workflows/syncer/v2/fetcher.go b/core/services/workflows/syncer/v2/fetcher.go index 4c79fffbe24..e86f998f6bb 100644 --- a/core/services/workflows/syncer/v2/fetcher.go +++ b/core/services/workflows/syncer/v2/fetcher.go @@ -211,20 +211,12 @@ func newFileFetcher(basePath string, lggr logger.Logger) types.FetcherFunc { if err != nil { return nil, fmt.Errorf("invalid URL: %w", err) } + fullPath := filepath.Clean(u.Path) - var fullPath string - if u.Scheme == "http" || u.Scheme == "https" { - // For HTTP(S) URLs, extract just the filename and resolve against basePath. - // This supports confidential workflows where the on-chain URL must be HTTP - // (so the enclave can fetch the binary), but the syncer reads from the local filesystem. - fullPath = filepath.Join(basePath, filepath.Base(u.Path)) - } else { - fullPath = filepath.Clean(u.Path) - // ensure that the incoming request URL is either relative or absolute but within the basePath - if !filepath.IsAbs(fullPath) { - // If it's not absolute, we assume it's relative to the basePath - fullPath = filepath.Join(basePath, fullPath) - } + // ensure that the incoming request URL is either relative or absolute but within the basePath + if !filepath.IsAbs(fullPath) { + // If it's not absolute, we assume it's relative to the basePath + fullPath = filepath.Join(basePath, fullPath) } if !strings.HasPrefix(fullPath, basePath) { return nil, fmt.Errorf("request URL %s is not within the basePath %s", fullPath, basePath) From c1b8c7eac66faae82de14e56f391306fc2425fac Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 13:37:43 +0100 Subject: [PATCH 03/67] Restore comments in startAndRegisterEngine, extract newV2EngineConfig Restore comments that were dropped when extracting startAndRegisterEngine from tryEngineCreate. Extract common EngineConfig construction into newV2EngineConfig and initDone hook wiring into wireInitDoneHook, reducing duplication between the normal and confidential engine paths. --- core/services/workflows/syncer/v2/handler.go | 166 +++++++++---------- 1 file changed, 76 insertions(+), 90 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index b298db91722..5ad4343b784 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -654,60 +654,11 @@ func (h *eventHandler) engineFactoryFn(ctx context.Context, workflowID string, o } // V2 aka "NoDAG" - cfg := &v2.EngineConfig{ - Lggr: h.lggr, - Module: module, - WorkflowConfig: config, - CapRegistry: h.capRegistry, - DonSubscriber: h.workflowDonSubscriber, - UseLocalTimeProvider: h.useLocalTimeProvider, - DonTimeStore: h.donTimeStore, - ExecutionsStore: h.workflowStore, - WorkflowID: workflowID, - WorkflowOwner: owner, - WorkflowName: name, - WorkflowTag: tag, - WorkflowEncryptionKey: h.workflowEncryptionKey, - - LocalLimits: v2.EngineLimits{}, // all defaults - LocalLimiters: h.engineLimiters, - FeatureFlags: h.featureFlags, - GlobalExecutionConcurrencyLimiter: h.workflowLimits, - - BeholderEmitter: func() custmsg.MessageEmitter { - h.emitterMu.RLock() - defer h.emitterMu.RUnlock() - return h.emitter - }(), - BillingClient: h.billingClient, - - WorkflowRegistryAddress: h.workflowRegistryAddress, - WorkflowRegistryChainSelector: h.workflowRegistryChainSelector, - OrgResolver: h.orgResolver, - DebugMode: h.debugMode, - SecretsFetcher: h.secretsFetcher, - SdkName: sdkName, - - ShardOrchestratorClient: h.shardOrchestratorClient, - ShardingEnabled: h.shardingEnabled, - MyShardID: h.myShardID, - ShardRoutingSteady: h.shardRoutingSteady, - } + cfg := h.newV2EngineConfig(module, workflowID, owner, name, tag, config) + cfg.DebugMode = h.debugMode + cfg.SdkName = sdkName - // Wire the initDone channel to the OnInitialized lifecycle hook. - // This will be called when the engine completes initialization (including trigger subscriptions). - // We compose with any existing hook to avoid overwriting test hooks or other user-provided hooks. - if initDone != nil { - existingHook := cfg.Hooks.OnInitialized - cfg.Hooks.OnInitialized = func(err error) { - // Signal completion to the handler first - initDone <- err - // Then call any existing hook (e.g., from tests) - if existingHook != nil { - existingHook(err) - } - } - } + h.wireInitDoneHook(cfg, initDone) return v2.NewEngine(cfg) } @@ -873,13 +824,18 @@ func (h *eventHandler) startAndRegisterEngine( // This ensures we don't emit workflowActivated events before the engine initializes successfully. select { case <-ctx.Done(): + // Context cancelled while waiting for initialization if closeErr := engine.Close(); closeErr != nil { h.lggr.Errorw("failed to close engine after context cancellation", "error", closeErr, "workflowID", workflowID) } return fmt.Errorf("context cancelled while waiting for engine initialization: %w", ctx.Err()) case initErr := <-initDone: if initErr != nil { + // Engine initialization failed (e.g., trigger subscription failed) // TODO (cre-1482) add logic to mark a deployment as failed to avoid churn. + // Currently, failed deployments will be retried on each poll cycle (with exponential backoff). + // If the failure is due to user error (e.g., invalid trigger config), this causes unnecessary retries. + // Consider marking the workflow spec as "failed" in the database and requiring workflow redeployment. if closeErr := engine.Close(); closeErr != nil { h.lggr.Errorw("failed to close engine after initialization failure", "error", closeErr, "workflowID", workflowID) } @@ -887,11 +843,13 @@ func (h *eventHandler) startAndRegisterEngine( } } + // Engine is fully initialized, add to registry with source tracking if err := h.engineRegistry.Add(wid, source, engine); err != nil { if closeErr := engine.Close(); closeErr != nil { return fmt.Errorf("failed to close workflow engine: %w during invariant violation: %w", closeErr, err) } + // Check for WorkflowID collision across sources if errors.Is(err, ErrAlreadyExists) { existingEntry, found := h.engineRegistry.Get(wid) if found { @@ -903,11 +861,73 @@ func (h *eventHandler) startAndRegisterEngine( } } + // This shouldn't happen because we call the handler serially and + // check for running engines above, see the call to engineRegistry.Contains. return fmt.Errorf("invariant violation: %w", err) } return nil } +// newV2EngineConfig builds the common EngineConfig shared by both the normal +// WASM engine and the confidential engine paths. Caller supplies the module. +func (h *eventHandler) newV2EngineConfig( + module host.ModuleV2, + workflowID string, owner string, name types.WorkflowName, tag string, config []byte, +) *v2.EngineConfig { + h.emitterMu.RLock() + emitter := h.emitter + h.emitterMu.RUnlock() + + return &v2.EngineConfig{ + Lggr: h.lggr, + Module: module, + WorkflowConfig: config, + CapRegistry: h.capRegistry, + DonSubscriber: h.workflowDonSubscriber, + UseLocalTimeProvider: h.useLocalTimeProvider, + DonTimeStore: h.donTimeStore, + ExecutionsStore: h.workflowStore, + WorkflowID: workflowID, + WorkflowOwner: owner, + WorkflowName: name, + WorkflowTag: tag, + WorkflowEncryptionKey: h.workflowEncryptionKey, + + LocalLimits: v2.EngineLimits{}, + LocalLimiters: h.engineLimiters, + FeatureFlags: h.featureFlags, + GlobalExecutionConcurrencyLimiter: h.workflowLimits, + + BeholderEmitter: emitter, + BillingClient: h.billingClient, + + WorkflowRegistryAddress: h.workflowRegistryAddress, + WorkflowRegistryChainSelector: h.workflowRegistryChainSelector, + OrgResolver: h.orgResolver, + SecretsFetcher: h.secretsFetcher, + + ShardOrchestratorClient: h.shardOrchestratorClient, + ShardingEnabled: h.shardingEnabled, + MyShardID: h.myShardID, + ShardRoutingSteady: h.shardRoutingSteady, + } +} + +// wireInitDoneHook wires the initDone channel to the OnInitialized lifecycle hook. +// It composes with any existing hook to avoid overwriting test hooks. +func (h *eventHandler) wireInitDoneHook(cfg *v2.EngineConfig, initDone chan<- error) { + if initDone == nil { + return + } + existingHook := cfg.Hooks.OnInitialized + cfg.Hooks.OnInitialized = func(err error) { + initDone <- err + if existingHook != nil { + existingHook(err) + } + } +} + // tryConfidentialEngineCreate creates a V2 engine backed by a ConfidentialModule // instead of a local WASM module. The ConfidentialModule delegates execution to // the confidential-workflows capability which runs the WASM inside a TEE. @@ -941,44 +961,10 @@ func (h *eventHandler) tryConfidentialEngineCreate( lggr, ) - initDone := make(chan error, 1) - - cfg := &v2.EngineConfig{ - Lggr: h.lggr, - Module: module, - WorkflowConfig: []byte(spec.Config), - CapRegistry: h.capRegistry, - DonSubscriber: h.workflowDonSubscriber, - UseLocalTimeProvider: h.useLocalTimeProvider, - DonTimeStore: h.donTimeStore, - ExecutionsStore: h.workflowStore, - WorkflowID: spec.WorkflowID, - WorkflowOwner: spec.WorkflowOwner, - WorkflowName: workflowName, - WorkflowTag: spec.WorkflowTag, - WorkflowEncryptionKey: h.workflowEncryptionKey, - - LocalLimits: v2.EngineLimits{}, - LocalLimiters: h.engineLimiters, - GlobalExecutionConcurrencyLimiter: h.workflowLimits, + cfg := h.newV2EngineConfig(module, spec.WorkflowID, spec.WorkflowOwner, workflowName, spec.WorkflowTag, []byte(spec.Config)) - BeholderEmitter: h.emitter, - BillingClient: h.billingClient, - - WorkflowRegistryAddress: h.workflowRegistryAddress, - WorkflowRegistryChainSelector: h.workflowRegistryChainSelector, - OrgResolver: h.orgResolver, - SecretsFetcher: h.secretsFetcher, - FeatureFlags: h.featureFlags, - } - - existingHook := cfg.Hooks.OnInitialized - cfg.Hooks.OnInitialized = func(err error) { - initDone <- err - if existingHook != nil { - existingHook(err) - } - } + initDone := make(chan error, 1) + h.wireInitDoneHook(cfg, initDone) engine, err := v2.NewEngine(cfg) if err != nil { From 6667183757699dfa6813afc3cb351162adef7be0 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 1 Apr 2026 14:21:35 +0200 Subject: [PATCH 04/67] Unify engine creation flow for confidential and normal paths Replace the early-return pattern with a symmetric if/else that picks the factory, then converges on a single startAndRegisterEngine call. Rename tryConfidentialEngineCreate to confidentialEngineFactory and change its signature to return (services.Service, error). # Conflicts: # core/services/workflows/syncer/v2/handler.go --- core/services/workflows/syncer/v2/handler.go | 47 +++++-------------- .../workflows/syncer/v2/handler_test.go | 2 +- 2 files changed, 14 insertions(+), 35 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 5ad4343b784..62a1f5f8428 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -775,29 +775,16 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp if err != nil { return fmt.Errorf("failed to parse workflow attributes: %w", err) } - if confidential { - h.lggr.Infow("routing workflow to confidential execution", "workflowID", spec.WorkflowID) - return h.tryConfidentialEngineCreate(ctx, spec, wid, workflowName, decodedBinary, source) - } - // Create a channel to receive the initialization result. - // This allows us to wait for the engine to complete initialization (including trigger subscriptions) - // before emitting the workflowActivated event, ensuring the event accurately reflects deployment status. initDone := make(chan error, 1) + var engine services.Service - // Scope the engineFactory call so that decodedBinary goes out of scope immediately after the factory returns - engine, err := func() (services.Service, error) { - return h.engineFactory( - ctx, - spec.WorkflowID, - spec.WorkflowOwner, - workflowName, - spec.WorkflowTag, - configBytes, - decodedBinary, - initDone, - ) - }() + if confidential { + h.lggr.Infow("routing workflow to confidential execution", "workflowID", spec.WorkflowID) + engine, err = h.confidentialEngineFactory(spec, wid, workflowName, decodedBinary, initDone) + } else { + engine, err = h.engineFactory(ctx, spec.WorkflowID, spec.WorkflowOwner, workflowName, spec.WorkflowTag, configBytes, decodedBinary, initDone) + } if err != nil { return fmt.Errorf("failed to create workflow engine: %w", err) } @@ -928,20 +915,19 @@ func (h *eventHandler) wireInitDoneHook(cfg *v2.EngineConfig, initDone chan<- er } } -// tryConfidentialEngineCreate creates a V2 engine backed by a ConfidentialModule +// confidentialEngineFactory creates a V2 engine backed by a ConfidentialModule // instead of a local WASM module. The ConfidentialModule delegates execution to // the confidential-workflows capability which runs the WASM inside a TEE. -func (h *eventHandler) tryConfidentialEngineCreate( - ctx context.Context, +func (h *eventHandler) confidentialEngineFactory( spec *job.WorkflowSpec, wid types.WorkflowID, workflowName types.WorkflowName, decodedBinary []byte, - source string, -) error { + initDone chan<- error, +) (services.Service, error) { attrs, err := v2.ParseWorkflowAttributes(spec.Attributes) if err != nil { - return fmt.Errorf("failed to parse workflow attributes: %w", err) + return nil, fmt.Errorf("failed to parse workflow attributes: %w", err) } binaryHash := v2.ComputeBinaryHash(decodedBinary) @@ -962,16 +948,9 @@ func (h *eventHandler) tryConfidentialEngineCreate( ) cfg := h.newV2EngineConfig(module, spec.WorkflowID, spec.WorkflowOwner, workflowName, spec.WorkflowTag, []byte(spec.Config)) - - initDone := make(chan error, 1) h.wireInitDoneHook(cfg, initDone) - engine, err := v2.NewEngine(cfg) - if err != nil { - return fmt.Errorf("failed to create confidential workflow engine: %w", err) - } - - return h.startAndRegisterEngine(ctx, engine, initDone, wid, spec.WorkflowID, source) + return v2.NewEngine(cfg) } // logCustMsg emits a custom message to the external sink and logs an error if that fails. diff --git a/core/services/workflows/syncer/v2/handler_test.go b/core/services/workflows/syncer/v2/handler_test.go index 775f059613e..aa0a66c6362 100644 --- a/core/services/workflows/syncer/v2/handler_test.go +++ b/core/services/workflows/syncer/v2/handler_test.go @@ -704,7 +704,7 @@ func Test_workflowRegisteredHandler_confidentialRouting(t *testing.T) { // (non-hex owner), engine creation fails. The error comes from the // confidential path, proving routing worked correctly. require.Error(t, err) - assert.Contains(t, err.Error(), "failed to create confidential workflow engine") + assert.Contains(t, err.Error(), "failed to create workflow engine") // The engine factory must NOT have been called; the confidential path // bypasses it. From 5036ad64be68f11c03191ae108e82cb83db7c8c7 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 13:50:56 +0100 Subject: [PATCH 05/67] Restore initDone comment in tryEngineCreate --- core/services/workflows/syncer/v2/handler.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 62a1f5f8428..803dba075b3 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -776,6 +776,9 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp return fmt.Errorf("failed to parse workflow attributes: %w", err) } + // Create a channel to receive the initialization result. + // This allows us to wait for the engine to complete initialization (including trigger subscriptions) + // before emitting the workflowActivated event, ensuring the event accurately reflects deployment status. initDone := make(chan error, 1) var engine services.Service From 417f1d3e5f49f94b3fba6b252c36115b365858b5 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 13:53:28 +0100 Subject: [PATCH 06/67] Inline startAndRegisterEngine back into tryEngineCreate No longer needed as a separate method now that both engine paths converge in tryEngineCreate. --- core/services/workflows/syncer/v2/handler.go | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 803dba075b3..8afd0673786 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -792,21 +792,7 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp return fmt.Errorf("failed to create workflow engine: %w", err) } - return h.startAndRegisterEngine(ctx, engine, initDone, wid, spec.WorkflowID, source) -} - -// startAndRegisterEngine starts a workflow engine, waits for initialization to -// complete, and registers it in the engine registry. Used by both the normal -// and confidential engine creation paths. -func (h *eventHandler) startAndRegisterEngine( - ctx context.Context, - engine services.Service, - initDone <-chan error, - wid types.WorkflowID, - workflowID string, - source string, -) error { - if err := engine.Start(ctx); err != nil { + if err = engine.Start(ctx); err != nil { return fmt.Errorf("failed to start workflow engine: %w", err) } @@ -816,7 +802,7 @@ func (h *eventHandler) startAndRegisterEngine( case <-ctx.Done(): // Context cancelled while waiting for initialization if closeErr := engine.Close(); closeErr != nil { - h.lggr.Errorw("failed to close engine after context cancellation", "error", closeErr, "workflowID", workflowID) + h.lggr.Errorw("failed to close engine after context cancellation", "error", closeErr, "workflowID", spec.WorkflowID) } return fmt.Errorf("context cancelled while waiting for engine initialization: %w", ctx.Err()) case initErr := <-initDone: @@ -827,7 +813,7 @@ func (h *eventHandler) startAndRegisterEngine( // If the failure is due to user error (e.g., invalid trigger config), this causes unnecessary retries. // Consider marking the workflow spec as "failed" in the database and requiring workflow redeployment. if closeErr := engine.Close(); closeErr != nil { - h.lggr.Errorw("failed to close engine after initialization failure", "error", closeErr, "workflowID", workflowID) + h.lggr.Errorw("failed to close engine after initialization failure", "error", closeErr, "workflowID", spec.WorkflowID) } return fmt.Errorf("engine initialization failed: %w", initErr) } From f0d05e0c0ea162ba6d4601b16d2f6c2b8518b153 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 13:55:49 +0100 Subject: [PATCH 07/67] Restore inline comments in wireInitDoneHook --- core/services/workflows/syncer/v2/handler.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 8afd0673786..96fd2abe378 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -890,14 +890,17 @@ func (h *eventHandler) newV2EngineConfig( } // wireInitDoneHook wires the initDone channel to the OnInitialized lifecycle hook. -// It composes with any existing hook to avoid overwriting test hooks. +// This will be called when the engine completes initialization (including trigger subscriptions). +// We compose with any existing hook to avoid overwriting test hooks or other user-provided hooks. func (h *eventHandler) wireInitDoneHook(cfg *v2.EngineConfig, initDone chan<- error) { if initDone == nil { return } existingHook := cfg.Hooks.OnInitialized cfg.Hooks.OnInitialized = func(err error) { + // Signal completion to the handler first initDone <- err + // Then call any existing hook (e.g., from tests) if existingHook != nil { existingHook(err) } From b0fdf2d8587b00ad3ff7f483616eda8832dfd550 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 13:59:20 +0100 Subject: [PATCH 08/67] Restore original BeholderEmitter closure pattern in newV2EngineConfig --- core/services/workflows/syncer/v2/handler.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 96fd2abe378..86d6b57c636 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -850,10 +850,6 @@ func (h *eventHandler) newV2EngineConfig( module host.ModuleV2, workflowID string, owner string, name types.WorkflowName, tag string, config []byte, ) *v2.EngineConfig { - h.emitterMu.RLock() - emitter := h.emitter - h.emitterMu.RUnlock() - return &v2.EngineConfig{ Lggr: h.lggr, Module: module, @@ -869,13 +865,17 @@ func (h *eventHandler) newV2EngineConfig( WorkflowTag: tag, WorkflowEncryptionKey: h.workflowEncryptionKey, - LocalLimits: v2.EngineLimits{}, + LocalLimits: v2.EngineLimits{}, // all defaults LocalLimiters: h.engineLimiters, FeatureFlags: h.featureFlags, GlobalExecutionConcurrencyLimiter: h.workflowLimits, - BeholderEmitter: emitter, - BillingClient: h.billingClient, + BeholderEmitter: func() custmsg.MessageEmitter { + h.emitterMu.RLock() + defer h.emitterMu.RUnlock() + return h.emitter + }(), + BillingClient: h.billingClient, WorkflowRegistryAddress: h.workflowRegistryAddress, WorkflowRegistryChainSelector: h.workflowRegistryChainSelector, From b44ba459522f320fbca9d9b98677793a3d1ca86d Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 1 Apr 2026 14:24:49 +0200 Subject: [PATCH 09/67] Clean up factory signatures and newV2EngineConfig param ordering Group string params together in newV2EngineConfig, move SdkName and DebugMode into the constructor, drop unused wid param from confidentialEngineFactory. --- core/services/workflows/syncer/v2/handler.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 86d6b57c636..49706219ca5 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -654,9 +654,7 @@ func (h *eventHandler) engineFactoryFn(ctx context.Context, workflowID string, o } // V2 aka "NoDAG" - cfg := h.newV2EngineConfig(module, workflowID, owner, name, tag, config) - cfg.DebugMode = h.debugMode - cfg.SdkName = sdkName + cfg := h.newV2EngineConfig(module, workflowID, owner, tag, sdkName, name, config) h.wireInitDoneHook(cfg, initDone) @@ -784,7 +782,7 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp if confidential { h.lggr.Infow("routing workflow to confidential execution", "workflowID", spec.WorkflowID) - engine, err = h.confidentialEngineFactory(spec, wid, workflowName, decodedBinary, initDone) + engine, err = h.confidentialEngineFactory(spec, workflowName, decodedBinary, initDone) } else { engine, err = h.engineFactory(ctx, spec.WorkflowID, spec.WorkflowOwner, workflowName, spec.WorkflowTag, configBytes, decodedBinary, initDone) } @@ -848,7 +846,9 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp // WASM engine and the confidential engine paths. Caller supplies the module. func (h *eventHandler) newV2EngineConfig( module host.ModuleV2, - workflowID string, owner string, name types.WorkflowName, tag string, config []byte, + workflowID, owner, tag, sdkName string, + name types.WorkflowName, + config []byte, ) *v2.EngineConfig { return &v2.EngineConfig{ Lggr: h.lggr, @@ -881,6 +881,8 @@ func (h *eventHandler) newV2EngineConfig( WorkflowRegistryChainSelector: h.workflowRegistryChainSelector, OrgResolver: h.orgResolver, SecretsFetcher: h.secretsFetcher, + DebugMode: h.debugMode, + SdkName: sdkName, ShardOrchestratorClient: h.shardOrchestratorClient, ShardingEnabled: h.shardingEnabled, @@ -912,7 +914,6 @@ func (h *eventHandler) wireInitDoneHook(cfg *v2.EngineConfig, initDone chan<- er // the confidential-workflows capability which runs the WASM inside a TEE. func (h *eventHandler) confidentialEngineFactory( spec *job.WorkflowSpec, - wid types.WorkflowID, workflowName types.WorkflowName, decodedBinary []byte, initDone chan<- error, @@ -939,7 +940,7 @@ func (h *eventHandler) confidentialEngineFactory( lggr, ) - cfg := h.newV2EngineConfig(module, spec.WorkflowID, spec.WorkflowOwner, workflowName, spec.WorkflowTag, []byte(spec.Config)) + cfg := h.newV2EngineConfig(module, spec.WorkflowID, spec.WorkflowOwner, spec.WorkflowTag, "", workflowName, []byte(spec.Config)) h.wireInitDoneHook(cfg, initDone) return v2.NewEngine(cfg) From c438c0767121b5553a6d84d180cb6f1a393d44e6 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 25 Mar 2026 13:47:48 +0100 Subject: [PATCH 10/67] Fix lint: errors.New, assert.Empty --- core/services/workflows/v2/confidential_module.go | 3 ++- core/services/workflows/v2/confidential_module_test.go | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/core/services/workflows/v2/confidential_module.go b/core/services/workflows/v2/confidential_module.go index 4cc3761e18a..16bbb83052c 100644 --- a/core/services/workflows/v2/confidential_module.go +++ b/core/services/workflows/v2/confidential_module.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "google.golang.org/protobuf/proto" @@ -165,7 +166,7 @@ func (m *ConfidentialModule) Execute( } if capResp.Payload == nil { - return nil, fmt.Errorf("confidential-workflows capability returned nil payload") + return nil, errors.New("confidential-workflows capability returned nil payload") } var confResp confworkflowtypes.ConfidentialWorkflowResponse diff --git a/core/services/workflows/v2/confidential_module_test.go b/core/services/workflows/v2/confidential_module_test.go index 87dab426022..598f82c0119 100644 --- a/core/services/workflows/v2/confidential_module_test.go +++ b/core/services/workflows/v2/confidential_module_test.go @@ -3,7 +3,7 @@ package v2 import ( "context" "crypto/sha256" - "fmt" + "errors" "testing" "time" @@ -52,7 +52,7 @@ func TestParseWorkflowAttributes(t *testing.T) { assert.True(t, attrs.Confidential) require.Len(t, attrs.VaultDonSecrets, 2) assert.Equal(t, "API_KEY", attrs.VaultDonSecrets[0].Key) - assert.Equal(t, "", attrs.VaultDonSecrets[0].Namespace) + assert.Empty(t, attrs.VaultDonSecrets[0].Namespace) assert.Equal(t, "SIGNING_KEY", attrs.VaultDonSecrets[1].Key) assert.Equal(t, "custom-ns", attrs.VaultDonSecrets[1].Namespace) }) @@ -222,7 +222,7 @@ func TestConfidentialModule_Execute(t *testing.T) { t.Run("GetExecutable error", func(t *testing.T) { capReg := regmocks.NewCapabilitiesRegistry(t) capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). - Return(nil, fmt.Errorf("capability not found")).Once() + Return(nil, errors.New("capability not found")).Once() mod := NewConfidentialModule( capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, @@ -240,7 +240,7 @@ func TestConfidentialModule_Execute(t *testing.T) { capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). Return(execCap, nil).Once() execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). - Return(capabilities.CapabilityResponse{}, fmt.Errorf("enclave unavailable")).Once() + Return(capabilities.CapabilityResponse{}, errors.New("enclave unavailable")).Once() mod := NewConfidentialModule( capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, From d1126248744ab49dc139675e7bf5bc10b50273c6 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 26 Mar 2026 17:36:46 +0100 Subject: [PATCH 11/67] Add BinaryURLResolver to ConfidentialModule for presigned URL support The enclave needs an authenticated URL to download WASM binaries from the CRE storage service. BinaryURLResolver resolves the raw on-chain URL into a presigned/ephemeral URL per execution. Nil-safe: falls back to the raw URL when no resolver is set. PR 5/5 (#21642) wires this to the storage service retriever. --- core/services/workflows/syncer/v2/handler.go | 9 +- .../workflows/v2/confidential_module.go | 61 +++++++---- .../workflows/v2/confidential_module_test.go | 103 +++++++++++++++++- 3 files changed, 143 insertions(+), 30 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 49706219ca5..0f357ba996e 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -928,15 +928,16 @@ func (h *eventHandler) confidentialEngineFactory( lggr := logger.Named(h.lggr, "WorkflowEngine.ConfidentialModule") lggr = logger.With(lggr, "workflowID", spec.WorkflowID, "workflowName", spec.WorkflowName, "workflowOwner", spec.WorkflowOwner) + // nil resolver: raw binaryURL is passed to the enclave as-is. + // PR 5/5 (#21642) wires this to the storage service retriever + // so the enclave receives a presigned URL. module := v2.NewConfidentialModule( h.capRegistry, spec.BinaryURL, binaryHash, - spec.WorkflowID, - spec.WorkflowOwner, - workflowName.String(), - spec.WorkflowTag, + spec.WorkflowID, spec.WorkflowOwner, workflowName.String(), spec.WorkflowTag, attrs.VaultDonSecrets, + nil, lggr, ) diff --git a/core/services/workflows/v2/confidential_module.go b/core/services/workflows/v2/confidential_module.go index 16bbb83052c..7a1983d2a27 100644 --- a/core/services/workflows/v2/confidential_module.go +++ b/core/services/workflows/v2/confidential_module.go @@ -57,19 +57,26 @@ func IsConfidential(data []byte) (bool, error) { return attrs.Confidential, nil } +// BinaryURLResolver resolves a raw binary URL into an ephemeral/presigned +// URL that the enclave can fetch without authentication. In production this +// calls the CRE storage service; nil means the raw URL is used as-is. +// PR 5/5 (#21642) wires this to the storage service retriever. +type BinaryURLResolver func(ctx context.Context, workflowID string) (string, error) + // ConfidentialModule implements host.ModuleV2 for confidential workflows. // Instead of running WASM locally, it delegates execution to the // confidential-workflows capability via the CapabilitiesRegistry. type ConfidentialModule struct { - capRegistry core.CapabilitiesRegistry - binaryURL string - binaryHash []byte - workflowID string - workflowOwner string - workflowName string - workflowTag string - vaultDonSecrets []SecretIdentifier - lggr logger.Logger + capRegistry core.CapabilitiesRegistry + binaryURL string + binaryHash []byte + workflowID string + workflowOwner string + workflowName string + workflowTag string + vaultDonSecrets []SecretIdentifier + binaryURLResolver BinaryURLResolver + lggr logger.Logger } var _ host.ModuleV2 = (*ConfidentialModule)(nil) @@ -78,23 +85,22 @@ func NewConfidentialModule( capRegistry core.CapabilitiesRegistry, binaryURL string, binaryHash []byte, - workflowID string, - workflowOwner string, - workflowName string, - workflowTag string, + workflowID, workflowOwner, workflowName, workflowTag string, vaultDonSecrets []SecretIdentifier, + binaryURLResolver BinaryURLResolver, lggr logger.Logger, ) *ConfidentialModule { return &ConfidentialModule{ - capRegistry: capRegistry, - binaryURL: binaryURL, - binaryHash: binaryHash, - workflowID: workflowID, - workflowOwner: workflowOwner, - workflowName: workflowName, - workflowTag: workflowTag, - vaultDonSecrets: vaultDonSecrets, - lggr: lggr, + capRegistry: capRegistry, + binaryURL: binaryURL, + binaryHash: binaryHash, + workflowID: workflowID, + workflowOwner: workflowOwner, + workflowName: workflowName, + workflowTag: workflowTag, + vaultDonSecrets: vaultDonSecrets, + binaryURLResolver: binaryURLResolver, + lggr: lggr, } } @@ -125,11 +131,20 @@ func (m *ConfidentialModule) Execute( } } + binaryURL := m.binaryURL + if m.binaryURLResolver != nil { + resolved, resolveErr := m.binaryURLResolver(ctx, m.workflowID) + if resolveErr != nil { + return nil, fmt.Errorf("failed to resolve binary URL: %w", resolveErr) + } + binaryURL = resolved + } + capInput := &confworkflowtypes.ConfidentialWorkflowRequest{ VaultDonSecrets: protoSecrets, Execution: &confworkflowtypes.WorkflowExecution{ WorkflowId: m.workflowID, - BinaryUrl: m.binaryURL, + BinaryUrl: binaryURL, BinaryHash: m.binaryHash, ExecuteRequest: execReqBytes, Owner: m.workflowOwner, diff --git a/core/services/workflows/v2/confidential_module_test.go b/core/services/workflows/v2/confidential_module_test.go index 598f82c0119..e72d7164b29 100644 --- a/core/services/workflows/v2/confidential_module_test.go +++ b/core/services/workflows/v2/confidential_module_test.go @@ -172,6 +172,7 @@ func TestConfidentialModule_Execute(t *testing.T) { {Key: "API_KEY"}, {Key: "SIGNING_KEY", Namespace: "custom-ns"}, }, + nil, lggr, ) @@ -205,6 +206,7 @@ func TestConfidentialModule_Execute(t *testing.T) { []byte("hash"), "wf-1", "owner", "name", "tag", []SecretIdentifier{{Key: "SECRET_A"}}, // no namespace + nil, lggr, ) @@ -225,7 +227,7 @@ func TestConfidentialModule_Execute(t *testing.T) { Return(nil, errors.New("capability not found")).Once() mod := NewConfidentialModule( - capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, + capReg, "", nil, "wf", "owner", "name", "tag", nil, nil, lggr, ) _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) @@ -243,7 +245,7 @@ func TestConfidentialModule_Execute(t *testing.T) { Return(capabilities.CapabilityResponse{}, errors.New("enclave unavailable")).Once() mod := NewConfidentialModule( - capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, + capReg, "", nil, "wf", "owner", "name", "tag", nil, nil, lggr, ) _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) @@ -261,7 +263,7 @@ func TestConfidentialModule_Execute(t *testing.T) { Return(capabilities.CapabilityResponse{Payload: nil}, nil).Once() mod := NewConfidentialModule( - capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, + capReg, "", nil, "wf", "owner", "name", "tag", nil, nil, lggr, ) _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) @@ -296,6 +298,7 @@ func TestConfidentialModule_Execute(t *testing.T) { {Key: "K1", Namespace: "ns1"}, {Key: "K2"}, }, + nil, lggr, ) @@ -333,6 +336,100 @@ func TestConfidentialModule_Execute(t *testing.T) { }) } +func TestConfidentialModule_BinaryURLResolver(t *testing.T) { + ctx := context.Background() + lggr := logger.Nop() + + execReq := &sdkpb.ExecuteRequest{Config: []byte("cfg")} + + expectedResult := &sdkpb.ExecutionResult{ + Result: &sdkpb.ExecutionResult_Value{ + Value: valuespb.NewStringValue("ok"), + }, + } + resultBytes, err := proto.Marshal(expectedResult) + require.NoError(t, err) + confResp := &confworkflowtypes.ConfidentialWorkflowResponse{ExecutionResult: resultBytes} + respPayload, err := anypb.New(confResp) + require.NoError(t, err) + + t.Run("resolver replaces binary URL", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + execCap := capmocks.NewExecutableCapability(t) + + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(execCap, nil).Once() + + var capturedReq capabilities.CapabilityRequest + execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). + Run(func(_ context.Context, req capabilities.CapabilityRequest) { + capturedReq = req + }). + Return(capabilities.CapabilityResponse{Payload: respPayload}, nil).Once() + + resolver := func(_ context.Context, wfID string) (string, error) { + return "https://presigned.example.com/" + wfID + "?token=abc", nil + } + + mod := NewConfidentialModule( + capReg, "https://storage.example.com/raw", []byte("hash"), + "wf-1", "owner", "name", "tag", nil, resolver, lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-1"}) + require.NoError(t, err) + + var confReq confworkflowtypes.ConfidentialWorkflowRequest + require.NoError(t, capturedReq.Payload.UnmarshalTo(&confReq)) + assert.Equal(t, "https://presigned.example.com/wf-1?token=abc", confReq.Execution.BinaryUrl) + }) + + t.Run("nil resolver uses raw URL", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + execCap := capmocks.NewExecutableCapability(t) + + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(execCap, nil).Once() + + var capturedReq capabilities.CapabilityRequest + execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). + Run(func(_ context.Context, req capabilities.CapabilityRequest) { + capturedReq = req + }). + Return(capabilities.CapabilityResponse{Payload: respPayload}, nil).Once() + + mod := NewConfidentialModule( + capReg, "https://storage.example.com/raw", []byte("hash"), + "wf-1", "owner", "name", "tag", nil, nil, lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-1"}) + require.NoError(t, err) + + var confReq confworkflowtypes.ConfidentialWorkflowRequest + require.NoError(t, capturedReq.Payload.UnmarshalTo(&confReq)) + assert.Equal(t, "https://storage.example.com/raw", confReq.Execution.BinaryUrl) + }) + + t.Run("resolver error propagates", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + + resolver := func(_ context.Context, _ string) (string, error) { + return "", errors.New("storage service unavailable") + } + + mod := NewConfidentialModule( + capReg, "https://storage.example.com/raw", []byte("hash"), + "wf-1", "owner", "name", "tag", nil, resolver, lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-1"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to resolve binary URL") + assert.Contains(t, err.Error(), "storage service unavailable") + }) +} + func TestConfidentialModule_InterfaceMethods(t *testing.T) { mod := &ConfidentialModule{} From 5692670a4ea08cb71c3150aab22c15d2fba4ef2f Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 8 Apr 2026 13:46:08 +0200 Subject: [PATCH 12/67] add EmitUserMetric to stubExecutionHelper host.ExecutionHelper gained EmitUserMetric in chainlink-common #1924. --- core/services/workflows/v2/confidential_module_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/services/workflows/v2/confidential_module_test.go b/core/services/workflows/v2/confidential_module_test.go index e72d7164b29..96a58cafa13 100644 --- a/core/services/workflows/v2/confidential_module_test.go +++ b/core/services/workflows/v2/confidential_module_test.go @@ -24,6 +24,7 @@ import ( sdkpb "github.com/smartcontractkit/chainlink-protos/cre/go/sdk" valuespb "github.com/smartcontractkit/chainlink-protos/cre/go/values/pb" + wfpb "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" ) // stubExecutionHelper implements host.ExecutionHelper for testing. @@ -43,6 +44,9 @@ func (s *stubExecutionHelper) GetWorkflowExecutionID() string { return s.executi func (s *stubExecutionHelper) GetNodeTime() time.Time { return time.Time{} } func (s *stubExecutionHelper) GetDONTime() (time.Time, error) { return time.Time{}, nil } func (s *stubExecutionHelper) EmitUserLog(string) error { return nil } +func (s *stubExecutionHelper) EmitUserMetric(context.Context, *wfpb.WorkflowUserMetric) error { + return nil +} func TestParseWorkflowAttributes(t *testing.T) { t.Run("valid JSON with all fields", func(t *testing.T) { From 737f336252e461512d373cc0badc0bb54b07a195 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 8 Apr 2026 14:23:57 +0200 Subject: [PATCH 13/67] fix struct field alignment in ConfidentialModule --- .../workflows/v2/confidential_module.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/core/services/workflows/v2/confidential_module.go b/core/services/workflows/v2/confidential_module.go index 7a1983d2a27..246886bcebf 100644 --- a/core/services/workflows/v2/confidential_module.go +++ b/core/services/workflows/v2/confidential_module.go @@ -67,16 +67,16 @@ type BinaryURLResolver func(ctx context.Context, workflowID string) (string, err // Instead of running WASM locally, it delegates execution to the // confidential-workflows capability via the CapabilitiesRegistry. type ConfidentialModule struct { - capRegistry core.CapabilitiesRegistry - binaryURL string - binaryHash []byte - workflowID string - workflowOwner string - workflowName string - workflowTag string - vaultDonSecrets []SecretIdentifier - binaryURLResolver BinaryURLResolver - lggr logger.Logger + capRegistry core.CapabilitiesRegistry + binaryURL string + binaryHash []byte + workflowID string + workflowOwner string + workflowName string + workflowTag string + vaultDonSecrets []SecretIdentifier + binaryURLResolver BinaryURLResolver + lggr logger.Logger } var _ host.ModuleV2 = (*ConfidentialModule)(nil) From 7a33bf345b66fcb8e1ff6a7455251ced52b91c63 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Mon, 23 Mar 2026 20:55:35 +0100 Subject: [PATCH 14/67] [CRE] [2/5] Relay DON node handler for confidential relay Add GatewayConnectorHandler that validates Nitro attestation and proxies enclave requests to VaultDON (secrets_get) and capability DONs (capability_exec). Supports multi-PCR validation for enclave pools. Part of #21635 --- .../capabilities/confidentialrelay/handler.go | 189 +++++------------- .../confidentialrelay/handler_test.go | 69 ++----- .../capabilities/confidentialrelay/service.go | 8 +- go.md | 8 + go.mod | 2 + go.sum | 5 + 6 files changed, 83 insertions(+), 198 deletions(-) diff --git a/core/capabilities/confidentialrelay/handler.go b/core/capabilities/confidentialrelay/handler.go index d16d5a440cc..e70157f4a23 100644 --- a/core/capabilities/confidentialrelay/handler.go +++ b/core/capabilities/confidentialrelay/handler.go @@ -7,7 +7,6 @@ import ( "encoding/json" "errors" "fmt" - "time" "github.com/ethereum/go-ethereum/common" "go.opentelemetry.io/otel/attribute" @@ -35,43 +34,14 @@ import ( var _ core.GatewayConnectorHandler = (*Handler)(nil) -const ( - HandlerName = "EnclaveRelayHandler" - internalErrorMessage = "internal error" - - // confidentialWorkflowsCapID is the capability ID for the confidential - // workflows enclave pool. The relay handler uses it to look up trusted - // enclave measurements from the capabilities registry. - confidentialWorkflowsCapID = "confidential-workflows@1.0.0-alpha" -) - -// enclaveEntry mirrors the enclave config shape stored in the capabilities -// registry. Only the fields needed for attestation validation are included. -type enclaveEntry struct { - TrustedValues []json.RawMessage `json:"trustedValues"` - CARootsPEM string `json:"caRootsPEM,omitempty"` -} - -type enclavesList struct { - Enclaves []enclaveEntry -} +const HandlerName = "EnclaveRelayHandler" type handlerMetrics struct { - requestCount metric.Int64Counter - requestLatency metric.Int64Histogram requestInternalError metric.Int64Counter requestSuccess metric.Int64Counter } func newMetrics() (*handlerMetrics, error) { - requestCount, err := beholder.GetMeter().Int64Counter("enclave_relay_request_count") - if err != nil { - return nil, fmt.Errorf("failed to register request count counter: %w", err) - } - requestLatency, err := beholder.GetMeter().Int64Histogram("enclave_relay_request_latency_ms", metric.WithUnit("ms")) - if err != nil { - return nil, fmt.Errorf("failed to register request latency histogram: %w", err) - } requestInternalError, err := beholder.GetMeter().Int64Counter("enclave_relay_request_internal_error") if err != nil { return nil, fmt.Errorf("failed to register internal error counter: %w", err) @@ -81,44 +51,59 @@ func newMetrics() (*handlerMetrics, error) { return nil, fmt.Errorf("failed to register success counter: %w", err) } return &handlerMetrics{ - requestCount: requestCount, - requestLatency: requestLatency, requestInternalError: requestInternalError, requestSuccess: requestSuccess, }, nil } -// attestationValidatorFunc validates a TEE attestation document. -type attestationValidatorFunc func(attestation []byte, expectedUserData []byte, trustedMeasurements []byte) error +type gatewayConnector interface { + SendToGateway(ctx context.Context, gatewayID string, resp *jsonrpc.Response[json.RawMessage]) error + AddHandler(ctx context.Context, methods []string, handler core.GatewayConnectorHandler) error + RemoveHandler(ctx context.Context, methods []string) error +} + +// attestationValidatorFunc validates a Nitro attestation document. +type attestationValidatorFunc func(attestation []byte, expectedUserData []byte, trustedMeasurements []byte, caRootsPEM string) error // Handler processes enclave relay requests from the gateway. -// It validates attestations and proxies requests to VaultDON or capability DONs. +// It validates Nitro attestations and proxies requests to VaultDON or capability DONs. type Handler struct { services.Service eng *services.Engine capRegistry core.CapabilitiesRegistry - gatewayConnector core.GatewayConnector + gatewayConnector gatewayConnector + trustedPCRs []byte lggr logger.Logger metrics *handlerMetrics - // validateAttestation validates TEE attestation documents. - // Defaults to the Nitro validator; overridden in tests. + // validateAttestation validates Nitro attestation documents. + // Defaults to the real nitro validator; overridden in tests. validateAttestation attestationValidatorFunc + + // caRootsPEM is an optional PEM-encoded CA root certificate for attestation + // validation. Empty string uses DefaultCARoots (production default). + caRootsPEM string } -func NewHandler(capRegistry core.CapabilitiesRegistry, conn core.GatewayConnector, lggr logger.Logger) (*Handler, error) { +func NewHandler(capRegistry core.CapabilitiesRegistry, conn gatewayConnector, trustedPCRs []byte, lggr logger.Logger, caRootsPEM ...string) (*Handler, error) { m, err := newMetrics() if err != nil { return nil, fmt.Errorf("failed to create metrics: %w", err) } + var roots string + if len(caRootsPEM) > 0 { + roots = caRootsPEM[0] + } h := &Handler{ capRegistry: capRegistry, gatewayConnector: conn, + trustedPCRs: trustedPCRs, lggr: logger.Named(lggr, HandlerName), metrics: m, validateAttestation: nitro.ValidateAttestation, + caRootsPEM: roots, } h.Service, h.eng = services.Config{ Name: HandlerName, @@ -152,21 +137,6 @@ func (h *Handler) Methods() []string { func (h *Handler) HandleGatewayMessage(ctx context.Context, gatewayID string, req *jsonrpc.Request[json.RawMessage]) error { h.lggr.Debugw("received message from gateway", "gatewayID", gatewayID, "requestID", req.ID) - startTime := time.Now() - outcome := "success" - var errorCode int64 - defer func() { - attrs := []attribute.KeyValue{ - attribute.String("gateway_id", gatewayID), - attribute.String("method", req.Method), - attribute.String("outcome", outcome), - } - if errorCode != 0 { - attrs = append(attrs, attribute.Int64("error_code", errorCode)) - } - h.metrics.requestCount.Add(ctx, 1, metric.WithAttributes(attrs...)) - h.metrics.requestLatency.Record(ctx, time.Since(startTime).Milliseconds(), metric.WithAttributes(attrs...)) - }() var response *jsonrpc.Response[json.RawMessage] switch req.Method { @@ -177,13 +147,8 @@ func (h *Handler) HandleGatewayMessage(ctx context.Context, gatewayID string, re default: response = h.errorResponse(ctx, gatewayID, req, jsonrpc.ErrMethodNotFound, errors.New("unsupported method: "+req.Method)) } - if response != nil && response.Error != nil { - outcome = "error" - errorCode = response.Error.Code - } if err := h.gatewayConnector.SendToGateway(ctx, gatewayID, response); err != nil { - outcome = "send_error" h.lggr.Errorw("failed to send message to gateway", "gatewayID", gatewayID, "err", err) return err } @@ -208,7 +173,7 @@ func (h *Handler) handleSecretsGet(ctx context.Context, gatewayID string, req *j att := params.Attestation params.Attestation = "" - if err := h.verifyAttestationHash(ctx, att, params, confidentialrelaytypes.DomainSecretsGet); err != nil { + if err := h.verifyAttestationHash(att, params, confidentialrelaytypes.DomainSecretsGet); err != nil { return h.errorResponse(ctx, gatewayID, req, jsonrpc.ErrInternal, err) } @@ -363,7 +328,7 @@ func (h *Handler) handleCapabilityExecute(ctx context.Context, gatewayID string, att := params.Attestation params.Attestation = "" - if err := h.verifyAttestationHash(ctx, att, params, confidentialrelaytypes.DomainCapabilityExec); err != nil { + if err := h.verifyAttestationHash(att, params, confidentialrelaytypes.DomainCapabilityExec); err != nil { return h.errorResponse(ctx, gatewayID, req, jsonrpc.ErrInternal, err) } @@ -382,20 +347,12 @@ func (h *Handler) handleCapabilityExecute(ctx context.Context, gatewayID string, return h.errorResponse(ctx, gatewayID, req, jsonrpc.ErrInvalidParams, fmt.Errorf("failed to unmarshal capability request: %w", err)) } - referenceID := params.ReferenceID - if referenceID == "" { - referenceID = req.ID - } - capReq := capabilities.CapabilityRequest{ Payload: sdkReq.Payload, Method: sdkReq.Method, CapabilityId: params.CapabilityID, Metadata: capabilities.RequestMetadata{ - WorkflowID: params.WorkflowID, - WorkflowOwner: params.Owner, - WorkflowExecutionID: params.ExecutionID, - ReferenceID: referenceID, + WorkflowID: params.WorkflowID, }, } @@ -432,45 +389,7 @@ func (h *Handler) handleCapabilityExecute(ctx context.Context, gatewayID string, return h.jsonResponse(req, result) } -// getEnclaveAttestationConfig reads the enclave pool configuration from the -// capabilities registry and returns trusted measurement sets and CA roots -// for attestation validation. Called per-request so the config stays fresh -// (same pattern as CC's EnsureFreshEnclaves). -func (h *Handler) getEnclaveAttestationConfig(ctx context.Context) ([]json.RawMessage, string, error) { - dons, err := h.capRegistry.DONsForCapability(ctx, confidentialWorkflowsCapID) - if err != nil { - return nil, "", fmt.Errorf("failed to find DON for %s: %w", confidentialWorkflowsCapID, err) - } - if len(dons) == 0 { - return nil, "", fmt.Errorf("no DON found hosting %s", confidentialWorkflowsCapID) - } - - capConfig, err := h.capRegistry.ConfigForCapability(ctx, confidentialWorkflowsCapID, dons[0].DON.ID) - if err != nil { - return nil, "", fmt.Errorf("failed to get config for %s: %w", confidentialWorkflowsCapID, err) - } - - if capConfig.DefaultConfig == nil { - return nil, "", fmt.Errorf("no default config for %s", confidentialWorkflowsCapID) - } - - var enclaves enclavesList - if err := capConfig.DefaultConfig.UnwrapTo(&enclaves); err != nil { - return nil, "", fmt.Errorf("failed to unwrap enclave config: %w", err) - } - - var measurements []json.RawMessage - var caRootsPEM string - for _, e := range enclaves.Enclaves { - measurements = append(measurements, e.TrustedValues...) - if caRootsPEM == "" && e.CARootsPEM != "" { - caRootsPEM = e.CARootsPEM - } - } - return measurements, caRootsPEM, nil -} - -func (h *Handler) verifyAttestationHash(ctx context.Context, attestationB64 string, cleanParams any, domainTag string) error { +func (h *Handler) verifyAttestationHash(attestationB64 string, cleanParams any, domainTag string) error { if attestationB64 == "" { return errors.New("missing attestation") } @@ -487,33 +406,24 @@ func (h *Handler) verifyAttestationHash(ctx context.Context, attestationB64 stri return fmt.Errorf("failed to decode attestation: %w", err) } - // Look up trusted measurements from the capabilities registry. - // Each enclave instance may have different PCR values, so we try each set - // and succeed if any match (same approach as pool.go's - // validateAttestationAgainstMultipleMeasurements). - measurements, caRootsPEM, err := h.getEnclaveAttestationConfig(ctx) - if err != nil { - return fmt.Errorf("failed to get enclave attestation config: %w", err) - } - - if len(measurements) == 0 { - return errors.New("no trusted measurements found in capabilities registry") - } - - var validationErr error - for _, m := range measurements { - var err error - if caRootsPEM != "" { - err = nitro.ValidateAttestationWithRoots(attestationBytes, hash, m, caRootsPEM) - } else { - err = h.validateAttestation(attestationBytes, hash, m) - } - if err == nil { - return nil + // Each enclave instance has different PCR values (WireGuard keys baked + // per CID), so trustedPCRs may be a JSON array of PCR objects. Try each + // set and succeed if any match, same as pool.go's + // validateAttestationAgainstMultipleMeasurements. + var pcrArray []json.RawMessage + if json.Unmarshal(h.trustedPCRs, &pcrArray) == nil && len(pcrArray) > 0 { + var validationErr error + for _, pcrs := range pcrArray { + err := h.validateAttestation(attestationBytes, hash, pcrs, h.caRootsPEM) + if err == nil { + return nil + } + validationErr = errors.Join(validationErr, err) } - validationErr = errors.Join(validationErr, err) + return fmt.Errorf("no trusted PCR set matched: %w", validationErr) } - return fmt.Errorf("no trusted measurement set matched: %w", validationErr) + + return h.validateAttestation(attestationBytes, hash, h.trustedPCRs, h.caRootsPEM) } func toSDKCapabilityResponse(capResp capabilities.CapabilityResponse) (*sdkpb.CapabilityResponse, error) { @@ -547,7 +457,7 @@ func (h *Handler) jsonResponse(req *jsonrpc.Request[json.RawMessage], result any Method: req.Method, Error: &jsonrpc.WireError{ Code: jsonrpc.ErrInternal, - Message: internalErrorMessage, + Message: err.Error(), }, } } @@ -573,18 +483,13 @@ func (h *Handler) errorResponse( attribute.Int64("error_code", errorCode), )) - message := err.Error() - if errorCode == jsonrpc.ErrInternal { - message = internalErrorMessage - } - return &jsonrpc.Response[json.RawMessage]{ Version: jsonrpc.JsonRpcVersion, ID: req.ID, Method: req.Method, Error: &jsonrpc.WireError{ Code: errorCode, - Message: message, + Message: err.Error(), }, } } diff --git a/core/capabilities/confidentialrelay/handler_test.go b/core/capabilities/confidentialrelay/handler_test.go index 42a1f478951..a0c494a32f0 100644 --- a/core/capabilities/confidentialrelay/handler_test.go +++ b/core/capabilities/confidentialrelay/handler_test.go @@ -41,10 +41,9 @@ func makeCapabilityPayload(t *testing.T, inputs map[string]any) string { const testAttestationB64 = "ZHVtbXktYXR0ZXN0YXRpb24=" // base64("dummy-attestation") -func noopValidator(_ []byte, _, _ []byte) error { return nil } +func noopValidator(_ []byte, _, _ []byte, _ string) error { return nil } type mockGatewayConnector struct { - core.UnimplementedGatewayConnector lastResp *jsonrpc.Response[json.RawMessage] addedMethods []string removed bool @@ -89,7 +88,6 @@ type mockCapRegistry struct { core.UnimplementedCapabilitiesRegistry executables map[string]*mockExecutable configs map[string]capabilities.CapabilityConfiguration - dons map[string][]capabilities.DONWithNodes localNode capabilities.Node } @@ -105,48 +103,20 @@ func (m *mockCapRegistry) ConfigForCapability(_ context.Context, capID string, _ } return capabilities.CapabilityConfiguration{}, fmt.Errorf("config not found: %s", capID) } -func (m *mockCapRegistry) DONsForCapability(_ context.Context, capID string) ([]capabilities.DONWithNodes, error) { - if dons, ok := m.dons[capID]; ok { - return dons, nil - } - return nil, fmt.Errorf("no DONs found for: %s", capID) -} func (m *mockCapRegistry) LocalNode(_ context.Context) (capabilities.Node, error) { return m.localNode, nil } -func newTestHandler(t *testing.T, registry core.CapabilitiesRegistry, gwConn core.GatewayConnector) *Handler { +func newTestHandler(t *testing.T, registry core.CapabilitiesRegistry, gwConn gatewayConnector) *Handler { t.Helper() lggr, err := logger.New() require.NoError(t, err) - h, err := NewHandler(registry, gwConn, lggr) + h, err := NewHandler(registry, gwConn, []byte(`{}`), lggr) require.NoError(t, err) h.validateAttestation = noopValidator return h } -// withEnclaveConfig adds the default confidential-workflows enclave config -// to a mock registry so getEnclaveAttestationConfig succeeds during tests. -func withEnclaveConfig(reg *mockCapRegistry) *mockCapRegistry { - enclaveConfig := enclavesList{ - Enclaves: []enclaveEntry{{TrustedValues: []json.RawMessage{json.RawMessage(`{}`)}}}, - } - wrapped, _ := values.WrapMap(enclaveConfig) - if reg.configs == nil { - reg.configs = map[string]capabilities.CapabilityConfiguration{} - } - reg.configs[confidentialWorkflowsCapID] = capabilities.CapabilityConfiguration{ - DefaultConfig: wrapped, - } - if reg.dons == nil { - reg.dons = map[string][]capabilities.DONWithNodes{} - } - reg.dons[confidentialWorkflowsCapID] = []capabilities.DONWithNodes{ - {DON: capabilities.DON{ID: 1}}, - } - return reg -} - func makeRequest(t *testing.T, method string, params any) *jsonrpc.Request[json.RawMessage] { t.Helper() b, err := json.Marshal(params) @@ -170,7 +140,7 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { { name: "capability execute success", registry: func(_ *testing.T) *mockCapRegistry { - return withEnclaveConfig(&mockCapRegistry{ + return &mockCapRegistry{ executables: map[string]*mockExecutable{ "my-cap@1.0.0": { execResult: capabilities.CapabilityResponse{ @@ -178,14 +148,11 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { }, }, }, - }) + } }, req: func(t *testing.T) *jsonrpc.Request[json.RawMessage] { return makeRequest(t, confidentialrelaytypes.MethodCapabilityExec, confidentialrelaytypes.CapabilityRequestParams{ WorkflowID: "wf-1", - Owner: "0xowner", - ExecutionID: "32c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce1", - ReferenceID: "17", CapabilityID: "my-cap@1.0.0", Payload: makeCapabilityPayload(t, map[string]any{"key": "val"}), Attestation: testAttestationB64, @@ -203,25 +170,17 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { assert.Equal(t, "result-proto-bytes", string(capResp.GetPayload().GetValue())) assert.Empty(t, result.Error) }, - checkExecutable: func(t *testing.T, reg *mockCapRegistry) { - exec := reg.executables["my-cap@1.0.0"] - require.NotNil(t, exec.lastRequest, "Execute should have been called") - assert.Equal(t, "wf-1", exec.lastRequest.Metadata.WorkflowID) - assert.Equal(t, "0xowner", exec.lastRequest.Metadata.WorkflowOwner) - assert.Equal(t, "32c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce1", exec.lastRequest.Metadata.WorkflowExecutionID) - assert.Equal(t, "17", exec.lastRequest.Metadata.ReferenceID) - }, }, { name: "capability execute sets Inputs from Payload for backward compat", registry: func(_ *testing.T) *mockCapRegistry { - return withEnclaveConfig(&mockCapRegistry{ + return &mockCapRegistry{ executables: map[string]*mockExecutable{ "my-cap@1.0.0": { execResult: capabilities.CapabilityResponse{}, }, }, - }) + } }, req: func(t *testing.T) *jsonrpc.Request[json.RawMessage] { return makeRequest(t, confidentialrelaytypes.MethodCapabilityExec, confidentialrelaytypes.CapabilityRequestParams{ @@ -251,7 +210,7 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { { name: "capability execute attestation failure", registry: func(_ *testing.T) *mockCapRegistry { - return withEnclaveConfig(&mockCapRegistry{}) + return &mockCapRegistry{} }, req: func(t *testing.T) *jsonrpc.Request[json.RawMessage] { return makeRequest(t, confidentialrelaytypes.MethodCapabilityExec, confidentialrelaytypes.CapabilityRequestParams{ @@ -268,7 +227,7 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { { name: "capability execute not found", registry: func(_ *testing.T) *mockCapRegistry { - return withEnclaveConfig(&mockCapRegistry{executables: map[string]*mockExecutable{}}) + return &mockCapRegistry{executables: map[string]*mockExecutable{}} }, req: func(t *testing.T) *jsonrpc.Request[json.RawMessage] { return makeRequest(t, confidentialrelaytypes.MethodCapabilityExec, confidentialrelaytypes.CapabilityRequestParams{ @@ -281,17 +240,17 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { checkResp: func(t *testing.T, resp *jsonrpc.Response[json.RawMessage]) { require.NotNil(t, resp.Error) assert.Equal(t, jsonrpc.ErrInternal, resp.Error.Code) - assert.Equal(t, internalErrorMessage, resp.Error.Message) + assert.Contains(t, resp.Error.Message, "capability not found") }, }, { name: "capability execute error returned in result", registry: func(_ *testing.T) *mockCapRegistry { - return withEnclaveConfig(&mockCapRegistry{ + return &mockCapRegistry{ executables: map[string]*mockExecutable{ "fail-cap@1.0.0": {execErr: errors.New("execution failed")}, }, - }) + } }, req: func(t *testing.T) *jsonrpc.Request[json.RawMessage] { sdkReq := &sdkpb.CapabilityRequest{Id: "fail-cap@1.0.0", Method: "Execute"} @@ -315,7 +274,7 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { { name: "unsupported method", registry: func(_ *testing.T) *mockCapRegistry { - return withEnclaveConfig(&mockCapRegistry{}) + return &mockCapRegistry{} }, req: func(t *testing.T) *jsonrpc.Request[json.RawMessage] { return makeRequest(t, "unknown.method", nil) @@ -328,7 +287,7 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { { name: "invalid params JSON", registry: func(_ *testing.T) *mockCapRegistry { - return withEnclaveConfig(&mockCapRegistry{}) + return &mockCapRegistry{} }, req: func(_ *testing.T) *jsonrpc.Request[json.RawMessage] { raw := json.RawMessage([]byte(`{invalid json`)) diff --git a/core/capabilities/confidentialrelay/service.go b/core/capabilities/confidentialrelay/service.go index b00ca76eb95..b9ae214848f 100644 --- a/core/capabilities/confidentialrelay/service.go +++ b/core/capabilities/confidentialrelay/service.go @@ -20,6 +20,8 @@ type Service struct { wrapper *gatewayconnector.ServiceWrapper capRegistry core.CapabilitiesRegistry + trustedPCRs []byte + caRootsPEM string lggr logger.Logger handler *Handler @@ -28,11 +30,15 @@ type Service struct { func NewService( wrapper *gatewayconnector.ServiceWrapper, capRegistry core.CapabilitiesRegistry, + trustedPCRs []byte, + caRootsPEM string, lggr logger.Logger, ) *Service { s := &Service{ wrapper: wrapper, capRegistry: capRegistry, + trustedPCRs: trustedPCRs, + caRootsPEM: caRootsPEM, lggr: lggr, } s.Service, s.eng = services.Config{ @@ -48,7 +54,7 @@ func (s *Service) start(ctx context.Context) error { if conn == nil { return errors.New("gateway connector not available") } - h, err := NewHandler(s.capRegistry, conn, s.lggr) + h, err := NewHandler(s.capRegistry, conn, s.trustedPCRs, s.lggr, s.caRootsPEM) if err != nil { return err } diff --git a/go.md b/go.md index 3bedafbab43..58e13c7b65b 100644 --- a/go.md +++ b/go.md @@ -66,6 +66,8 @@ flowchart LR click chainlink-common/pkg/chipingress href "https://github.com/smartcontractkit/chainlink-common" chainlink-common/pkg/monitoring click chainlink-common/pkg/monitoring href "https://github.com/smartcontractkit/chainlink-common" + chainlink-common/pkg/teeattestation + click chainlink-common/pkg/teeattestation href "https://github.com/smartcontractkit/chainlink-common" chainlink-data-streams --> chainlink-common/keystore click chainlink-data-streams href "https://github.com/smartcontractkit/chainlink-data-streams" chainlink-evm --> chainlink-data-streams @@ -138,6 +140,7 @@ flowchart LR click chainlink-tron/relayer href "https://github.com/smartcontractkit/chainlink-tron" chainlink/v2 --> chainlink-automation chainlink/v2 --> chainlink-ccv + chainlink/v2 --> chainlink-common/pkg/teeattestation chainlink/v2 --> chainlink-evm/contracts/cre/gobindings chainlink/v2 --> chainlink-feeds chainlink/v2 --> chainlink-protos/ring/go @@ -188,6 +191,7 @@ flowchart LR chainlink-common/keystore chainlink-common/pkg/chipingress chainlink-common/pkg/monitoring + chainlink-common/pkg/teeattestation end click chainlink-common-repo href "https://github.com/smartcontractkit/chainlink-common" @@ -318,6 +322,8 @@ flowchart LR click chainlink-common/pkg/chipingress href "https://github.com/smartcontractkit/chainlink-common" chainlink-common/pkg/monitoring click chainlink-common/pkg/monitoring href "https://github.com/smartcontractkit/chainlink-common" + chainlink-common/pkg/teeattestation + click chainlink-common/pkg/teeattestation href "https://github.com/smartcontractkit/chainlink-common" chainlink-common/pkg/values click chainlink-common/pkg/values href "https://github.com/smartcontractkit/chainlink-common" chainlink-common/pkg/workflows/sdk/v2/pb --> chainlink-common/pkg/values @@ -530,6 +536,7 @@ flowchart LR click chainlink/system-tests/tests/smoke/cre/vaultsecret href "https://github.com/smartcontractkit/chainlink" chainlink/v2 --> chainlink-automation chainlink/v2 --> chainlink-ccv + chainlink/v2 --> chainlink-common/pkg/teeattestation chainlink/v2 --> chainlink-evm/contracts/cre/gobindings chainlink/v2 --> chainlink-feeds chainlink/v2 --> chainlink-protos/ring/go @@ -625,6 +632,7 @@ flowchart LR chainlink-common/keystore chainlink-common/pkg/chipingress chainlink-common/pkg/monitoring + chainlink-common/pkg/teeattestation chainlink-common/pkg/values chainlink-common/pkg/workflows/sdk/v2/pb end diff --git a/go.mod b/go.mod index fe53621739e..e3a4cd99488 100644 --- a/go.mod +++ b/go.mod @@ -88,6 +88,7 @@ require ( github.com/smartcontractkit/chainlink-common v0.11.2-0.20260406055916-9aa6b6c0ae81 github.com/smartcontractkit/chainlink-common/keystore v1.0.2 github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10 + github.com/smartcontractkit/chainlink-common/pkg/teeattestation v0.0.0-20260326144312-af5248caa880 github.com/smartcontractkit/chainlink-data-streams v0.1.13 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260330133421-5151ea0c3b05 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 @@ -292,6 +293,7 @@ require ( github.com/hashicorp/yamux v0.1.2 // indirect github.com/hasura/go-graphql-client v0.15.1 // indirect github.com/hdevalence/ed25519consensus v0.2.0 // indirect + github.com/hf/nitrite v0.0.0-20241225144000-c2d5d3c4f303 // indirect github.com/holiman/billy v0.0.0-20250707135307-f2f9b9aae7db // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect github.com/huin/goupnp v1.3.0 // indirect diff --git a/go.sum b/go.sum index 4c02975ac41..354d18b0136 100644 --- a/go.sum +++ b/go.sum @@ -425,6 +425,7 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.2.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gabriel-vasile/mimetype v1.4.10 h1:zyueNbySn/z8mJZHLt6IPw0KoZsiQNszIpU+bX4+ZK0= @@ -743,6 +744,8 @@ github.com/hasura/go-graphql-client v0.15.1 h1:mCb5I+8Bk3FU3GKWvf/zDXkTh7FbGlqJm github.com/hasura/go-graphql-client v0.15.1/go.mod h1:jfSZtBER3or+88Q9vFhWHiFMPppfYILRyl+0zsgPIIw= github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU= github.com/hdevalence/ed25519consensus v0.2.0/go.mod h1:w3BHWjwJbFU29IRHL1Iqkw3sus+7FctEyM4RqDxYNzo= +github.com/hf/nitrite v0.0.0-20241225144000-c2d5d3c4f303 h1:XBSq4rXFUgD8ic6Mr7dBwJN/47yg87XpZQhiknfr4Cg= +github.com/hf/nitrite v0.0.0-20241225144000-c2d5d3c4f303/go.mod h1:ycRhVmo6wegyEl6WN+zXOHUTJvB0J2tiuH88q/McTK8= github.com/holiman/billy v0.0.0-20250707135307-f2f9b9aae7db h1:IZUYC/xb3giYwBLMnr8d0TGTzPKFGNTCGgGLoyeX330= github.com/holiman/billy v0.0.0-20250707135307-f2f9b9aae7db/go.mod h1:xTEYN9KCHxuYHs+NmrmzFcnvHMzLLNiGFafCb1n3Mfg= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= @@ -1243,6 +1246,8 @@ github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10 h1:FJAFgXS9 github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10/go.mod h1:oiDa54M0FwxevWwyAX773lwdWvFYYlYHHQV1LQ5HpWY= github.com/smartcontractkit/chainlink-common/pkg/monitoring v0.0.0-20251215152504-b1e41f508340 h1:PsjEI+5jZIz9AS4eOsLS5VpSWJINf38clXV3wryPyMk= github.com/smartcontractkit/chainlink-common/pkg/monitoring v0.0.0-20251215152504-b1e41f508340/go.mod h1:P/0OSXUlFaxxD4B/P6HWbxYtIRmmWGDJAvanq19879c= +github.com/smartcontractkit/chainlink-common/pkg/teeattestation v0.0.0-20260326144312-af5248caa880 h1:qLrZSgc7e/exxIE4M020A52ep6xN0tXOJESiots9DvE= +github.com/smartcontractkit/chainlink-common/pkg/teeattestation v0.0.0-20260326144312-af5248caa880/go.mod h1:+X7Cb8ysHfNnPd74htGIInzZMHfUrpdDrjlr6+VW0gU= github.com/smartcontractkit/chainlink-data-streams v0.1.13 h1:YOmt545DW6U0SyaqBf+NTGDLm1yMurVI7yOvxP5hlJk= github.com/smartcontractkit/chainlink-data-streams v0.1.13/go.mod h1:00aL7OK0BJdF9gn/4t4f/pctUu2VLwwfA8G/tl9rCrM= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260330133421-5151ea0c3b05 h1:mUaNoTKE3lqlANSOlQmiYDIAjJoqHiShw+r5rl1xh7g= From 559b906d1f16ad0cb2229a5d2264f1ed8084409e Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 13:31:18 +0100 Subject: [PATCH 15/67] Add ConfidentialRelay config for relay DON handler --- core/config/cre_config.go | 2 ++ core/config/toml/types.go | 12 ++++++++++-- core/services/chainlink/config_cre.go | 18 +++++++++++++++--- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/core/config/cre_config.go b/core/config/cre_config.go index 30aa3ce3c93..78a600d1490 100644 --- a/core/config/cre_config.go +++ b/core/config/cre_config.go @@ -25,6 +25,8 @@ type WorkflowFetcher interface { // CREConfidentialRelay defines configuration for the confidential relay handler. type CREConfidentialRelay interface { Enabled() bool + TrustedPCRs() string + CARootsPEM() string } // CRELinking defines configuration for connecting to the CRE linking service diff --git a/core/config/toml/types.go b/core/config/toml/types.go index cd1b765f269..87da02a1eb8 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -1985,7 +1985,7 @@ type CreConfig struct { // Requires [Tracing].Enabled = true for traces to be exported (trace export is gated by // Tracing.Enabled in initGlobals; Telemetry.Enabled is optional—traces work with or without it). // WARNING: This is not suitable for production use due to performance overhead. - DebugMode *bool `toml:",omitempty"` + DebugMode *bool `toml:",omitempty"` ConfidentialRelay *ConfidentialRelayConfig `toml:",omitempty"` } @@ -1998,7 +1998,9 @@ type WorkflowFetcherConfig struct { // When Enabled is true, the node participates in the confidential relay DON, // validating enclave attestations and proxying capability requests. type ConfidentialRelayConfig struct { - Enabled *bool `toml:",omitempty"` + Enabled *bool `toml:",omitempty"` + TrustedPCRs *string `toml:",omitempty"` + CARootsPEM *string `toml:",omitempty"` } // LinkingConfig holds the configuration for connecting to the CRE linking service @@ -2062,6 +2064,12 @@ func (c *CreConfig) setFrom(f *CreConfig) { if v := f.ConfidentialRelay.Enabled; v != nil { c.ConfidentialRelay.Enabled = v } + if v := f.ConfidentialRelay.TrustedPCRs; v != nil { + c.ConfidentialRelay.TrustedPCRs = v + } + if v := f.ConfidentialRelay.CARootsPEM; v != nil { + c.ConfidentialRelay.CARootsPEM = v + } } } diff --git a/core/services/chainlink/config_cre.go b/core/services/chainlink/config_cre.go index 00d2878d00c..dbb9c404f21 100644 --- a/core/services/chainlink/config_cre.go +++ b/core/services/chainlink/config_cre.go @@ -106,10 +106,14 @@ func (c *creConfig) Linking() config.CRELinking { } type confidentialRelayConfig struct { - enabled bool + enabled bool + trustedPCRs string + caRootsPEM string } -func (cr *confidentialRelayConfig) Enabled() bool { return cr.enabled } +func (cr *confidentialRelayConfig) Enabled() bool { return cr.enabled } +func (cr *confidentialRelayConfig) TrustedPCRs() string { return cr.trustedPCRs } +func (cr *confidentialRelayConfig) CARootsPEM() string { return cr.caRootsPEM } func (c *creConfig) ConfidentialRelay() config.CREConfidentialRelay { if c.c.ConfidentialRelay == nil { @@ -119,7 +123,15 @@ func (c *creConfig) ConfidentialRelay() config.CREConfidentialRelay { if c.c.ConfidentialRelay.Enabled != nil { enabled = *c.c.ConfidentialRelay.Enabled } - return &confidentialRelayConfig{enabled: enabled} + trustedPCRs := "" + if c.c.ConfidentialRelay.TrustedPCRs != nil { + trustedPCRs = *c.c.ConfidentialRelay.TrustedPCRs + } + caRootsPEM := "" + if c.c.ConfidentialRelay.CARootsPEM != nil { + caRootsPEM = *c.c.ConfidentialRelay.CARootsPEM + } + return &confidentialRelayConfig{enabled: enabled, trustedPCRs: trustedPCRs, caRootsPEM: caRootsPEM} } func (c *creConfig) LocalSecrets() map[string]string { From 74476acc01e08c440e265c224a58f85069756c6f Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 25 Mar 2026 12:15:47 +0100 Subject: [PATCH 16/67] Remove Nitro-specific wording from attestation comments --- core/capabilities/confidentialrelay/handler.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/capabilities/confidentialrelay/handler.go b/core/capabilities/confidentialrelay/handler.go index e70157f4a23..454075f13b7 100644 --- a/core/capabilities/confidentialrelay/handler.go +++ b/core/capabilities/confidentialrelay/handler.go @@ -62,11 +62,11 @@ type gatewayConnector interface { RemoveHandler(ctx context.Context, methods []string) error } -// attestationValidatorFunc validates a Nitro attestation document. +// attestationValidatorFunc validates a TEE attestation document. type attestationValidatorFunc func(attestation []byte, expectedUserData []byte, trustedMeasurements []byte, caRootsPEM string) error // Handler processes enclave relay requests from the gateway. -// It validates Nitro attestations and proxies requests to VaultDON or capability DONs. +// It validates attestations and proxies requests to VaultDON or capability DONs. type Handler struct { services.Service eng *services.Engine @@ -77,8 +77,8 @@ type Handler struct { lggr logger.Logger metrics *handlerMetrics - // validateAttestation validates Nitro attestation documents. - // Defaults to the real nitro validator; overridden in tests. + // validateAttestation validates TEE attestation documents. + // Defaults to the Nitro validator; overridden in tests. validateAttestation attestationValidatorFunc // caRootsPEM is an optional PEM-encoded CA root certificate for attestation From 13bcba5ea57b337064199eddd5682ece2dc31e48 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 25 Mar 2026 12:38:22 +0100 Subject: [PATCH 17/67] Look up trusted measurements from cap registry instead of TOML config Replace the static trustedPCRs TOML parameter with a per-request lookup from the capabilities registry. The handler reads the confidential-workflows capability config to extract enclave TrustedValues, same pattern as CC's EnsureFreshEnclaves. Remove TrustedPCRs from CREConfidentialRelay config interface, ConfidentialRelayConfig TOML struct, and config_cre.go. --- .../capabilities/confidentialrelay/handler.go | 98 ++++++++++++++----- .../confidentialrelay/handler_test.go | 51 +++++++--- .../capabilities/confidentialrelay/service.go | 5 +- core/config/cre_config.go | 1 - core/config/toml/types.go | 8 +- core/services/chainlink/config_cre.go | 16 +-- 6 files changed, 123 insertions(+), 56 deletions(-) diff --git a/core/capabilities/confidentialrelay/handler.go b/core/capabilities/confidentialrelay/handler.go index 454075f13b7..7f717d653c5 100644 --- a/core/capabilities/confidentialrelay/handler.go +++ b/core/capabilities/confidentialrelay/handler.go @@ -34,7 +34,24 @@ import ( var _ core.GatewayConnectorHandler = (*Handler)(nil) -const HandlerName = "EnclaveRelayHandler" +const ( + HandlerName = "EnclaveRelayHandler" + + // confidentialWorkflowsCapID is the capability ID for the confidential + // workflows enclave pool. The relay handler uses it to look up trusted + // enclave measurements from the capabilities registry. + confidentialWorkflowsCapID = "confidential-workflows@1.0.0-alpha" +) + +// enclaveEntry mirrors the enclave config shape stored in the capabilities +// registry. Only the fields needed for attestation validation are included. +type enclaveEntry struct { + TrustedValues []json.RawMessage `json:"trustedValues"` +} + +type enclavesList struct { + Enclaves []enclaveEntry +} type handlerMetrics struct { requestInternalError metric.Int64Counter @@ -73,7 +90,6 @@ type Handler struct { capRegistry core.CapabilitiesRegistry gatewayConnector gatewayConnector - trustedPCRs []byte lggr logger.Logger metrics *handlerMetrics @@ -86,7 +102,7 @@ type Handler struct { caRootsPEM string } -func NewHandler(capRegistry core.CapabilitiesRegistry, conn gatewayConnector, trustedPCRs []byte, lggr logger.Logger, caRootsPEM ...string) (*Handler, error) { +func NewHandler(capRegistry core.CapabilitiesRegistry, conn gatewayConnector, lggr logger.Logger, caRootsPEM ...string) (*Handler, error) { m, err := newMetrics() if err != nil { return nil, fmt.Errorf("failed to create metrics: %w", err) @@ -99,7 +115,6 @@ func NewHandler(capRegistry core.CapabilitiesRegistry, conn gatewayConnector, tr h := &Handler{ capRegistry: capRegistry, gatewayConnector: conn, - trustedPCRs: trustedPCRs, lggr: logger.Named(lggr, HandlerName), metrics: m, validateAttestation: nitro.ValidateAttestation, @@ -173,7 +188,7 @@ func (h *Handler) handleSecretsGet(ctx context.Context, gatewayID string, req *j att := params.Attestation params.Attestation = "" - if err := h.verifyAttestationHash(att, params, confidentialrelaytypes.DomainSecretsGet); err != nil { + if err := h.verifyAttestationHash(ctx, att, params, confidentialrelaytypes.DomainSecretsGet); err != nil { return h.errorResponse(ctx, gatewayID, req, jsonrpc.ErrInternal, err) } @@ -328,7 +343,7 @@ func (h *Handler) handleCapabilityExecute(ctx context.Context, gatewayID string, att := params.Attestation params.Attestation = "" - if err := h.verifyAttestationHash(att, params, confidentialrelaytypes.DomainCapabilityExec); err != nil { + if err := h.verifyAttestationHash(ctx, att, params, confidentialrelaytypes.DomainCapabilityExec); err != nil { return h.errorResponse(ctx, gatewayID, req, jsonrpc.ErrInternal, err) } @@ -389,7 +404,41 @@ func (h *Handler) handleCapabilityExecute(ctx context.Context, gatewayID string, return h.jsonResponse(req, result) } -func (h *Handler) verifyAttestationHash(attestationB64 string, cleanParams any, domainTag string) error { +// getTrustedMeasurements reads the enclave pool configuration from the +// capabilities registry and returns all trusted measurement sets. Called +// per-request so measurements stay fresh (same pattern as CC's +// EnsureFreshEnclaves). +func (h *Handler) getTrustedMeasurements(ctx context.Context) ([]json.RawMessage, error) { + dons, err := h.capRegistry.DONsForCapability(ctx, confidentialWorkflowsCapID) + if err != nil { + return nil, fmt.Errorf("failed to find DON for %s: %w", confidentialWorkflowsCapID, err) + } + if len(dons) == 0 { + return nil, fmt.Errorf("no DON found hosting %s", confidentialWorkflowsCapID) + } + + capConfig, err := h.capRegistry.ConfigForCapability(ctx, confidentialWorkflowsCapID, dons[0].DON.ID) + if err != nil { + return nil, fmt.Errorf("failed to get config for %s: %w", confidentialWorkflowsCapID, err) + } + + if capConfig.DefaultConfig == nil { + return nil, fmt.Errorf("no default config for %s", confidentialWorkflowsCapID) + } + + var enclaves enclavesList + if err := capConfig.DefaultConfig.UnwrapTo(&enclaves); err != nil { + return nil, fmt.Errorf("failed to unwrap enclave config: %w", err) + } + + var measurements []json.RawMessage + for _, e := range enclaves.Enclaves { + measurements = append(measurements, e.TrustedValues...) + } + return measurements, nil +} + +func (h *Handler) verifyAttestationHash(ctx context.Context, attestationB64 string, cleanParams any, domainTag string) error { if attestationB64 == "" { return errors.New("missing attestation") } @@ -406,24 +455,27 @@ func (h *Handler) verifyAttestationHash(attestationB64 string, cleanParams any, return fmt.Errorf("failed to decode attestation: %w", err) } - // Each enclave instance has different PCR values (WireGuard keys baked - // per CID), so trustedPCRs may be a JSON array of PCR objects. Try each - // set and succeed if any match, same as pool.go's - // validateAttestationAgainstMultipleMeasurements. - var pcrArray []json.RawMessage - if json.Unmarshal(h.trustedPCRs, &pcrArray) == nil && len(pcrArray) > 0 { - var validationErr error - for _, pcrs := range pcrArray { - err := h.validateAttestation(attestationBytes, hash, pcrs, h.caRootsPEM) - if err == nil { - return nil - } - validationErr = errors.Join(validationErr, err) - } - return fmt.Errorf("no trusted PCR set matched: %w", validationErr) + // Look up trusted measurements from the capabilities registry. Each enclave + // instance may have different PCR values, so we try each set and succeed if + // any match (same approach as pool.go's validateAttestationAgainstMultipleMeasurements). + measurements, err := h.getTrustedMeasurements(ctx) + if err != nil { + return fmt.Errorf("failed to get trusted measurements: %w", err) + } + + if len(measurements) == 0 { + return errors.New("no trusted measurements found in capabilities registry") } - return h.validateAttestation(attestationBytes, hash, h.trustedPCRs, h.caRootsPEM) + var validationErr error + for _, m := range measurements { + err := h.validateAttestation(attestationBytes, hash, m, h.caRootsPEM) + if err == nil { + return nil + } + validationErr = errors.Join(validationErr, err) + } + return fmt.Errorf("no trusted measurement set matched: %w", validationErr) } func toSDKCapabilityResponse(capResp capabilities.CapabilityResponse) (*sdkpb.CapabilityResponse, error) { diff --git a/core/capabilities/confidentialrelay/handler_test.go b/core/capabilities/confidentialrelay/handler_test.go index a0c494a32f0..bcf90b77f57 100644 --- a/core/capabilities/confidentialrelay/handler_test.go +++ b/core/capabilities/confidentialrelay/handler_test.go @@ -88,6 +88,7 @@ type mockCapRegistry struct { core.UnimplementedCapabilitiesRegistry executables map[string]*mockExecutable configs map[string]capabilities.CapabilityConfiguration + dons map[string][]capabilities.DONWithNodes localNode capabilities.Node } @@ -103,6 +104,12 @@ func (m *mockCapRegistry) ConfigForCapability(_ context.Context, capID string, _ } return capabilities.CapabilityConfiguration{}, fmt.Errorf("config not found: %s", capID) } +func (m *mockCapRegistry) DONsForCapability(_ context.Context, capID string) ([]capabilities.DONWithNodes, error) { + if dons, ok := m.dons[capID]; ok { + return dons, nil + } + return nil, fmt.Errorf("no DONs found for: %s", capID) +} func (m *mockCapRegistry) LocalNode(_ context.Context) (capabilities.Node, error) { return m.localNode, nil } @@ -111,12 +118,34 @@ func newTestHandler(t *testing.T, registry core.CapabilitiesRegistry, gwConn gat t.Helper() lggr, err := logger.New() require.NoError(t, err) - h, err := NewHandler(registry, gwConn, []byte(`{}`), lggr) + h, err := NewHandler(registry, gwConn, lggr) require.NoError(t, err) h.validateAttestation = noopValidator return h } +// withEnclaveConfig adds the default confidential-workflows enclave config +// to a mock registry so getTrustedMeasurements succeeds during tests. +func withEnclaveConfig(reg *mockCapRegistry) *mockCapRegistry { + enclaveConfig := enclavesList{ + Enclaves: []enclaveEntry{{TrustedValues: []json.RawMessage{json.RawMessage(`{}`)}}}, + } + wrapped, _ := values.WrapMap(enclaveConfig) + if reg.configs == nil { + reg.configs = map[string]capabilities.CapabilityConfiguration{} + } + reg.configs[confidentialWorkflowsCapID] = capabilities.CapabilityConfiguration{ + DefaultConfig: wrapped, + } + if reg.dons == nil { + reg.dons = map[string][]capabilities.DONWithNodes{} + } + reg.dons[confidentialWorkflowsCapID] = []capabilities.DONWithNodes{ + {DON: capabilities.DON{ID: 1}}, + } + return reg +} + func makeRequest(t *testing.T, method string, params any) *jsonrpc.Request[json.RawMessage] { t.Helper() b, err := json.Marshal(params) @@ -140,7 +169,7 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { { name: "capability execute success", registry: func(_ *testing.T) *mockCapRegistry { - return &mockCapRegistry{ + return withEnclaveConfig(&mockCapRegistry{ executables: map[string]*mockExecutable{ "my-cap@1.0.0": { execResult: capabilities.CapabilityResponse{ @@ -148,7 +177,7 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { }, }, }, - } + }) }, req: func(t *testing.T) *jsonrpc.Request[json.RawMessage] { return makeRequest(t, confidentialrelaytypes.MethodCapabilityExec, confidentialrelaytypes.CapabilityRequestParams{ @@ -174,13 +203,13 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { { name: "capability execute sets Inputs from Payload for backward compat", registry: func(_ *testing.T) *mockCapRegistry { - return &mockCapRegistry{ + return withEnclaveConfig(&mockCapRegistry{ executables: map[string]*mockExecutable{ "my-cap@1.0.0": { execResult: capabilities.CapabilityResponse{}, }, }, - } + }) }, req: func(t *testing.T) *jsonrpc.Request[json.RawMessage] { return makeRequest(t, confidentialrelaytypes.MethodCapabilityExec, confidentialrelaytypes.CapabilityRequestParams{ @@ -210,7 +239,7 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { { name: "capability execute attestation failure", registry: func(_ *testing.T) *mockCapRegistry { - return &mockCapRegistry{} + return withEnclaveConfig(&mockCapRegistry{}) }, req: func(t *testing.T) *jsonrpc.Request[json.RawMessage] { return makeRequest(t, confidentialrelaytypes.MethodCapabilityExec, confidentialrelaytypes.CapabilityRequestParams{ @@ -227,7 +256,7 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { { name: "capability execute not found", registry: func(_ *testing.T) *mockCapRegistry { - return &mockCapRegistry{executables: map[string]*mockExecutable{}} + return withEnclaveConfig(&mockCapRegistry{executables: map[string]*mockExecutable{}}) }, req: func(t *testing.T) *jsonrpc.Request[json.RawMessage] { return makeRequest(t, confidentialrelaytypes.MethodCapabilityExec, confidentialrelaytypes.CapabilityRequestParams{ @@ -246,11 +275,11 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { { name: "capability execute error returned in result", registry: func(_ *testing.T) *mockCapRegistry { - return &mockCapRegistry{ + return withEnclaveConfig(&mockCapRegistry{ executables: map[string]*mockExecutable{ "fail-cap@1.0.0": {execErr: errors.New("execution failed")}, }, - } + }) }, req: func(t *testing.T) *jsonrpc.Request[json.RawMessage] { sdkReq := &sdkpb.CapabilityRequest{Id: "fail-cap@1.0.0", Method: "Execute"} @@ -274,7 +303,7 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { { name: "unsupported method", registry: func(_ *testing.T) *mockCapRegistry { - return &mockCapRegistry{} + return withEnclaveConfig(&mockCapRegistry{}) }, req: func(t *testing.T) *jsonrpc.Request[json.RawMessage] { return makeRequest(t, "unknown.method", nil) @@ -287,7 +316,7 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { { name: "invalid params JSON", registry: func(_ *testing.T) *mockCapRegistry { - return &mockCapRegistry{} + return withEnclaveConfig(&mockCapRegistry{}) }, req: func(_ *testing.T) *jsonrpc.Request[json.RawMessage] { raw := json.RawMessage([]byte(`{invalid json`)) diff --git a/core/capabilities/confidentialrelay/service.go b/core/capabilities/confidentialrelay/service.go index b9ae214848f..236e4442f8c 100644 --- a/core/capabilities/confidentialrelay/service.go +++ b/core/capabilities/confidentialrelay/service.go @@ -20,7 +20,6 @@ type Service struct { wrapper *gatewayconnector.ServiceWrapper capRegistry core.CapabilitiesRegistry - trustedPCRs []byte caRootsPEM string lggr logger.Logger @@ -30,14 +29,12 @@ type Service struct { func NewService( wrapper *gatewayconnector.ServiceWrapper, capRegistry core.CapabilitiesRegistry, - trustedPCRs []byte, caRootsPEM string, lggr logger.Logger, ) *Service { s := &Service{ wrapper: wrapper, capRegistry: capRegistry, - trustedPCRs: trustedPCRs, caRootsPEM: caRootsPEM, lggr: lggr, } @@ -54,7 +51,7 @@ func (s *Service) start(ctx context.Context) error { if conn == nil { return errors.New("gateway connector not available") } - h, err := NewHandler(s.capRegistry, conn, s.trustedPCRs, s.lggr, s.caRootsPEM) + h, err := NewHandler(s.capRegistry, conn, s.lggr, s.caRootsPEM) if err != nil { return err } diff --git a/core/config/cre_config.go b/core/config/cre_config.go index 78a600d1490..880793147fa 100644 --- a/core/config/cre_config.go +++ b/core/config/cre_config.go @@ -25,7 +25,6 @@ type WorkflowFetcher interface { // CREConfidentialRelay defines configuration for the confidential relay handler. type CREConfidentialRelay interface { Enabled() bool - TrustedPCRs() string CARootsPEM() string } diff --git a/core/config/toml/types.go b/core/config/toml/types.go index 87da02a1eb8..a445e89d3ac 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -1998,9 +1998,8 @@ type WorkflowFetcherConfig struct { // When Enabled is true, the node participates in the confidential relay DON, // validating enclave attestations and proxying capability requests. type ConfidentialRelayConfig struct { - Enabled *bool `toml:",omitempty"` - TrustedPCRs *string `toml:",omitempty"` - CARootsPEM *string `toml:",omitempty"` + Enabled *bool `toml:",omitempty"` + CARootsPEM *string `toml:",omitempty"` } // LinkingConfig holds the configuration for connecting to the CRE linking service @@ -2064,9 +2063,6 @@ func (c *CreConfig) setFrom(f *CreConfig) { if v := f.ConfidentialRelay.Enabled; v != nil { c.ConfidentialRelay.Enabled = v } - if v := f.ConfidentialRelay.TrustedPCRs; v != nil { - c.ConfidentialRelay.TrustedPCRs = v - } if v := f.ConfidentialRelay.CARootsPEM; v != nil { c.ConfidentialRelay.CARootsPEM = v } diff --git a/core/services/chainlink/config_cre.go b/core/services/chainlink/config_cre.go index dbb9c404f21..fc14b3448c5 100644 --- a/core/services/chainlink/config_cre.go +++ b/core/services/chainlink/config_cre.go @@ -106,14 +106,12 @@ func (c *creConfig) Linking() config.CRELinking { } type confidentialRelayConfig struct { - enabled bool - trustedPCRs string - caRootsPEM string + enabled bool + caRootsPEM string } -func (cr *confidentialRelayConfig) Enabled() bool { return cr.enabled } -func (cr *confidentialRelayConfig) TrustedPCRs() string { return cr.trustedPCRs } -func (cr *confidentialRelayConfig) CARootsPEM() string { return cr.caRootsPEM } +func (cr *confidentialRelayConfig) Enabled() bool { return cr.enabled } +func (cr *confidentialRelayConfig) CARootsPEM() string { return cr.caRootsPEM } func (c *creConfig) ConfidentialRelay() config.CREConfidentialRelay { if c.c.ConfidentialRelay == nil { @@ -123,15 +121,11 @@ func (c *creConfig) ConfidentialRelay() config.CREConfidentialRelay { if c.c.ConfidentialRelay.Enabled != nil { enabled = *c.c.ConfidentialRelay.Enabled } - trustedPCRs := "" - if c.c.ConfidentialRelay.TrustedPCRs != nil { - trustedPCRs = *c.c.ConfidentialRelay.TrustedPCRs - } caRootsPEM := "" if c.c.ConfidentialRelay.CARootsPEM != nil { caRootsPEM = *c.c.ConfidentialRelay.CARootsPEM } - return &confidentialRelayConfig{enabled: enabled, trustedPCRs: trustedPCRs, caRootsPEM: caRootsPEM} + return &confidentialRelayConfig{enabled: enabled, caRootsPEM: caRootsPEM} } func (c *creConfig) LocalSecrets() map[string]string { From 009f21e692fc550a34ab79b2968128dc70d604b4 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 25 Mar 2026 12:45:10 +0100 Subject: [PATCH 18/67] Also read CARootsPEM from cap registry, remove from TOML config CARootsPEM is stored per-enclave in the same cap registry config as TrustedValues. Read both in getEnclaveAttestationConfig, eliminating the last TOML field besides Enabled from CRE.ConfidentialRelay. --- .../capabilities/confidentialrelay/handler.go | 51 +++++++++---------- .../confidentialrelay/handler_test.go | 2 +- .../capabilities/confidentialrelay/service.go | 5 +- core/config/cre_config.go | 1 - core/config/toml/types.go | 6 +-- core/services/chainlink/config_cre.go | 12 ++--- 6 files changed, 29 insertions(+), 48 deletions(-) diff --git a/core/capabilities/confidentialrelay/handler.go b/core/capabilities/confidentialrelay/handler.go index 7f717d653c5..e824dc14974 100644 --- a/core/capabilities/confidentialrelay/handler.go +++ b/core/capabilities/confidentialrelay/handler.go @@ -47,6 +47,7 @@ const ( // registry. Only the fields needed for attestation validation are included. type enclaveEntry struct { TrustedValues []json.RawMessage `json:"trustedValues"` + CARootsPEM string `json:"caRootsPEM,omitempty"` } type enclavesList struct { @@ -96,29 +97,20 @@ type Handler struct { // validateAttestation validates TEE attestation documents. // Defaults to the Nitro validator; overridden in tests. validateAttestation attestationValidatorFunc - - // caRootsPEM is an optional PEM-encoded CA root certificate for attestation - // validation. Empty string uses DefaultCARoots (production default). - caRootsPEM string } -func NewHandler(capRegistry core.CapabilitiesRegistry, conn gatewayConnector, lggr logger.Logger, caRootsPEM ...string) (*Handler, error) { +func NewHandler(capRegistry core.CapabilitiesRegistry, conn gatewayConnector, lggr logger.Logger) (*Handler, error) { m, err := newMetrics() if err != nil { return nil, fmt.Errorf("failed to create metrics: %w", err) } - var roots string - if len(caRootsPEM) > 0 { - roots = caRootsPEM[0] - } h := &Handler{ capRegistry: capRegistry, gatewayConnector: conn, lggr: logger.Named(lggr, HandlerName), metrics: m, validateAttestation: nitro.ValidateAttestation, - caRootsPEM: roots, } h.Service, h.eng = services.Config{ Name: HandlerName, @@ -404,38 +396,40 @@ func (h *Handler) handleCapabilityExecute(ctx context.Context, gatewayID string, return h.jsonResponse(req, result) } -// getTrustedMeasurements reads the enclave pool configuration from the -// capabilities registry and returns all trusted measurement sets. Called -// per-request so measurements stay fresh (same pattern as CC's -// EnsureFreshEnclaves). -func (h *Handler) getTrustedMeasurements(ctx context.Context) ([]json.RawMessage, error) { +// getEnclaveAttestationConfig reads the enclave pool configuration from the +// capabilities registry and returns trusted measurement sets and CA roots +// for attestation validation. Called per-request so the config stays fresh +// (same pattern as CC's EnsureFreshEnclaves). +func (h *Handler) getEnclaveAttestationConfig(ctx context.Context) (measurements []json.RawMessage, caRootsPEM string, err error) { dons, err := h.capRegistry.DONsForCapability(ctx, confidentialWorkflowsCapID) if err != nil { - return nil, fmt.Errorf("failed to find DON for %s: %w", confidentialWorkflowsCapID, err) + return nil, "", fmt.Errorf("failed to find DON for %s: %w", confidentialWorkflowsCapID, err) } if len(dons) == 0 { - return nil, fmt.Errorf("no DON found hosting %s", confidentialWorkflowsCapID) + return nil, "", fmt.Errorf("no DON found hosting %s", confidentialWorkflowsCapID) } capConfig, err := h.capRegistry.ConfigForCapability(ctx, confidentialWorkflowsCapID, dons[0].DON.ID) if err != nil { - return nil, fmt.Errorf("failed to get config for %s: %w", confidentialWorkflowsCapID, err) + return nil, "", fmt.Errorf("failed to get config for %s: %w", confidentialWorkflowsCapID, err) } if capConfig.DefaultConfig == nil { - return nil, fmt.Errorf("no default config for %s", confidentialWorkflowsCapID) + return nil, "", fmt.Errorf("no default config for %s", confidentialWorkflowsCapID) } var enclaves enclavesList if err := capConfig.DefaultConfig.UnwrapTo(&enclaves); err != nil { - return nil, fmt.Errorf("failed to unwrap enclave config: %w", err) + return nil, "", fmt.Errorf("failed to unwrap enclave config: %w", err) } - var measurements []json.RawMessage for _, e := range enclaves.Enclaves { measurements = append(measurements, e.TrustedValues...) + if caRootsPEM == "" && e.CARootsPEM != "" { + caRootsPEM = e.CARootsPEM + } } - return measurements, nil + return measurements, caRootsPEM, nil } func (h *Handler) verifyAttestationHash(ctx context.Context, attestationB64 string, cleanParams any, domainTag string) error { @@ -455,12 +449,13 @@ func (h *Handler) verifyAttestationHash(ctx context.Context, attestationB64 stri return fmt.Errorf("failed to decode attestation: %w", err) } - // Look up trusted measurements from the capabilities registry. Each enclave - // instance may have different PCR values, so we try each set and succeed if - // any match (same approach as pool.go's validateAttestationAgainstMultipleMeasurements). - measurements, err := h.getTrustedMeasurements(ctx) + // Look up trusted measurements and CA roots from the capabilities registry. + // Each enclave instance may have different PCR values, so we try each set + // and succeed if any match (same approach as pool.go's + // validateAttestationAgainstMultipleMeasurements). + measurements, caRootsPEM, err := h.getEnclaveAttestationConfig(ctx) if err != nil { - return fmt.Errorf("failed to get trusted measurements: %w", err) + return fmt.Errorf("failed to get enclave attestation config: %w", err) } if len(measurements) == 0 { @@ -469,7 +464,7 @@ func (h *Handler) verifyAttestationHash(ctx context.Context, attestationB64 stri var validationErr error for _, m := range measurements { - err := h.validateAttestation(attestationBytes, hash, m, h.caRootsPEM) + err := h.validateAttestation(attestationBytes, hash, m, caRootsPEM) if err == nil { return nil } diff --git a/core/capabilities/confidentialrelay/handler_test.go b/core/capabilities/confidentialrelay/handler_test.go index bcf90b77f57..42d2f4eb222 100644 --- a/core/capabilities/confidentialrelay/handler_test.go +++ b/core/capabilities/confidentialrelay/handler_test.go @@ -125,7 +125,7 @@ func newTestHandler(t *testing.T, registry core.CapabilitiesRegistry, gwConn gat } // withEnclaveConfig adds the default confidential-workflows enclave config -// to a mock registry so getTrustedMeasurements succeeds during tests. +// to a mock registry so getEnclaveAttestationConfig succeeds during tests. func withEnclaveConfig(reg *mockCapRegistry) *mockCapRegistry { enclaveConfig := enclavesList{ Enclaves: []enclaveEntry{{TrustedValues: []json.RawMessage{json.RawMessage(`{}`)}}}, diff --git a/core/capabilities/confidentialrelay/service.go b/core/capabilities/confidentialrelay/service.go index 236e4442f8c..b00ca76eb95 100644 --- a/core/capabilities/confidentialrelay/service.go +++ b/core/capabilities/confidentialrelay/service.go @@ -20,7 +20,6 @@ type Service struct { wrapper *gatewayconnector.ServiceWrapper capRegistry core.CapabilitiesRegistry - caRootsPEM string lggr logger.Logger handler *Handler @@ -29,13 +28,11 @@ type Service struct { func NewService( wrapper *gatewayconnector.ServiceWrapper, capRegistry core.CapabilitiesRegistry, - caRootsPEM string, lggr logger.Logger, ) *Service { s := &Service{ wrapper: wrapper, capRegistry: capRegistry, - caRootsPEM: caRootsPEM, lggr: lggr, } s.Service, s.eng = services.Config{ @@ -51,7 +48,7 @@ func (s *Service) start(ctx context.Context) error { if conn == nil { return errors.New("gateway connector not available") } - h, err := NewHandler(s.capRegistry, conn, s.lggr, s.caRootsPEM) + h, err := NewHandler(s.capRegistry, conn, s.lggr) if err != nil { return err } diff --git a/core/config/cre_config.go b/core/config/cre_config.go index 880793147fa..30aa3ce3c93 100644 --- a/core/config/cre_config.go +++ b/core/config/cre_config.go @@ -25,7 +25,6 @@ type WorkflowFetcher interface { // CREConfidentialRelay defines configuration for the confidential relay handler. type CREConfidentialRelay interface { Enabled() bool - CARootsPEM() string } // CRELinking defines configuration for connecting to the CRE linking service diff --git a/core/config/toml/types.go b/core/config/toml/types.go index a445e89d3ac..64f5980aeb9 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -1998,8 +1998,7 @@ type WorkflowFetcherConfig struct { // When Enabled is true, the node participates in the confidential relay DON, // validating enclave attestations and proxying capability requests. type ConfidentialRelayConfig struct { - Enabled *bool `toml:",omitempty"` - CARootsPEM *string `toml:",omitempty"` + Enabled *bool `toml:",omitempty"` } // LinkingConfig holds the configuration for connecting to the CRE linking service @@ -2063,9 +2062,6 @@ func (c *CreConfig) setFrom(f *CreConfig) { if v := f.ConfidentialRelay.Enabled; v != nil { c.ConfidentialRelay.Enabled = v } - if v := f.ConfidentialRelay.CARootsPEM; v != nil { - c.ConfidentialRelay.CARootsPEM = v - } } } diff --git a/core/services/chainlink/config_cre.go b/core/services/chainlink/config_cre.go index fc14b3448c5..00d2878d00c 100644 --- a/core/services/chainlink/config_cre.go +++ b/core/services/chainlink/config_cre.go @@ -106,12 +106,10 @@ func (c *creConfig) Linking() config.CRELinking { } type confidentialRelayConfig struct { - enabled bool - caRootsPEM string + enabled bool } -func (cr *confidentialRelayConfig) Enabled() bool { return cr.enabled } -func (cr *confidentialRelayConfig) CARootsPEM() string { return cr.caRootsPEM } +func (cr *confidentialRelayConfig) Enabled() bool { return cr.enabled } func (c *creConfig) ConfidentialRelay() config.CREConfidentialRelay { if c.c.ConfidentialRelay == nil { @@ -121,11 +119,7 @@ func (c *creConfig) ConfidentialRelay() config.CREConfidentialRelay { if c.c.ConfidentialRelay.Enabled != nil { enabled = *c.c.ConfidentialRelay.Enabled } - caRootsPEM := "" - if c.c.ConfidentialRelay.CARootsPEM != nil { - caRootsPEM = *c.c.ConfidentialRelay.CARootsPEM - } - return &confidentialRelayConfig{enabled: enabled, caRootsPEM: caRootsPEM} + return &confidentialRelayConfig{enabled: enabled} } func (c *creConfig) LocalSecrets() map[string]string { From 8f0d5a2b2f56f9e204d90d0331a959ca06202f73 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 26 Mar 2026 13:13:31 +0100 Subject: [PATCH 19/67] Remove CARootsPEM from relay handler ValidateAttestation in chainlink-common now always uses the hardcoded AWS Nitro root cert (c7b8c137). Drop the caRootsPEM parameter from the handler's attestation validator func, getEnclaveAttestationConfig, and the enclaveEntry struct. --- .../capabilities/confidentialrelay/handler.go | 27 +++++++++---------- .../confidentialrelay/handler_test.go | 2 +- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/core/capabilities/confidentialrelay/handler.go b/core/capabilities/confidentialrelay/handler.go index e824dc14974..3e743888717 100644 --- a/core/capabilities/confidentialrelay/handler.go +++ b/core/capabilities/confidentialrelay/handler.go @@ -47,7 +47,6 @@ const ( // registry. Only the fields needed for attestation validation are included. type enclaveEntry struct { TrustedValues []json.RawMessage `json:"trustedValues"` - CARootsPEM string `json:"caRootsPEM,omitempty"` } type enclavesList struct { @@ -81,7 +80,7 @@ type gatewayConnector interface { } // attestationValidatorFunc validates a TEE attestation document. -type attestationValidatorFunc func(attestation []byte, expectedUserData []byte, trustedMeasurements []byte, caRootsPEM string) error +type attestationValidatorFunc func(attestation []byte, expectedUserData []byte, trustedMeasurements []byte) error // Handler processes enclave relay requests from the gateway. // It validates attestations and proxies requests to VaultDON or capability DONs. @@ -400,36 +399,34 @@ func (h *Handler) handleCapabilityExecute(ctx context.Context, gatewayID string, // capabilities registry and returns trusted measurement sets and CA roots // for attestation validation. Called per-request so the config stays fresh // (same pattern as CC's EnsureFreshEnclaves). -func (h *Handler) getEnclaveAttestationConfig(ctx context.Context) (measurements []json.RawMessage, caRootsPEM string, err error) { +func (h *Handler) getEnclaveAttestationConfig(ctx context.Context) ([]json.RawMessage, error) { dons, err := h.capRegistry.DONsForCapability(ctx, confidentialWorkflowsCapID) if err != nil { - return nil, "", fmt.Errorf("failed to find DON for %s: %w", confidentialWorkflowsCapID, err) + return nil, fmt.Errorf("failed to find DON for %s: %w", confidentialWorkflowsCapID, err) } if len(dons) == 0 { - return nil, "", fmt.Errorf("no DON found hosting %s", confidentialWorkflowsCapID) + return nil, fmt.Errorf("no DON found hosting %s", confidentialWorkflowsCapID) } capConfig, err := h.capRegistry.ConfigForCapability(ctx, confidentialWorkflowsCapID, dons[0].DON.ID) if err != nil { - return nil, "", fmt.Errorf("failed to get config for %s: %w", confidentialWorkflowsCapID, err) + return nil, fmt.Errorf("failed to get config for %s: %w", confidentialWorkflowsCapID, err) } if capConfig.DefaultConfig == nil { - return nil, "", fmt.Errorf("no default config for %s", confidentialWorkflowsCapID) + return nil, fmt.Errorf("no default config for %s", confidentialWorkflowsCapID) } var enclaves enclavesList if err := capConfig.DefaultConfig.UnwrapTo(&enclaves); err != nil { - return nil, "", fmt.Errorf("failed to unwrap enclave config: %w", err) + return nil, fmt.Errorf("failed to unwrap enclave config: %w", err) } + var measurements []json.RawMessage for _, e := range enclaves.Enclaves { measurements = append(measurements, e.TrustedValues...) - if caRootsPEM == "" && e.CARootsPEM != "" { - caRootsPEM = e.CARootsPEM - } } - return measurements, caRootsPEM, nil + return measurements, nil } func (h *Handler) verifyAttestationHash(ctx context.Context, attestationB64 string, cleanParams any, domainTag string) error { @@ -449,11 +446,11 @@ func (h *Handler) verifyAttestationHash(ctx context.Context, attestationB64 stri return fmt.Errorf("failed to decode attestation: %w", err) } - // Look up trusted measurements and CA roots from the capabilities registry. + // Look up trusted measurements from the capabilities registry. // Each enclave instance may have different PCR values, so we try each set // and succeed if any match (same approach as pool.go's // validateAttestationAgainstMultipleMeasurements). - measurements, caRootsPEM, err := h.getEnclaveAttestationConfig(ctx) + measurements, err := h.getEnclaveAttestationConfig(ctx) if err != nil { return fmt.Errorf("failed to get enclave attestation config: %w", err) } @@ -464,7 +461,7 @@ func (h *Handler) verifyAttestationHash(ctx context.Context, attestationB64 stri var validationErr error for _, m := range measurements { - err := h.validateAttestation(attestationBytes, hash, m, caRootsPEM) + err := h.validateAttestation(attestationBytes, hash, m) if err == nil { return nil } diff --git a/core/capabilities/confidentialrelay/handler_test.go b/core/capabilities/confidentialrelay/handler_test.go index 42d2f4eb222..7de6e246b5e 100644 --- a/core/capabilities/confidentialrelay/handler_test.go +++ b/core/capabilities/confidentialrelay/handler_test.go @@ -41,7 +41,7 @@ func makeCapabilityPayload(t *testing.T, inputs map[string]any) string { const testAttestationB64 = "ZHVtbXktYXR0ZXN0YXRpb24=" // base64("dummy-attestation") -func noopValidator(_ []byte, _, _ []byte, _ string) error { return nil } +func noopValidator(_ []byte, _, _ []byte) error { return nil } type mockGatewayConnector struct { lastResp *jsonrpc.Response[json.RawMessage] From 82126a957a27021a1cd95da600ecaaed58af6e82 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Fri, 27 Mar 2026 14:58:11 +0100 Subject: [PATCH 20/67] Relay handler: use CARootsPEM from registry for fake enclave attestation --- .../capabilities/confidentialrelay/handler.go | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/core/capabilities/confidentialrelay/handler.go b/core/capabilities/confidentialrelay/handler.go index 3e743888717..5b6b54905c9 100644 --- a/core/capabilities/confidentialrelay/handler.go +++ b/core/capabilities/confidentialrelay/handler.go @@ -47,6 +47,7 @@ const ( // registry. Only the fields needed for attestation validation are included. type enclaveEntry struct { TrustedValues []json.RawMessage `json:"trustedValues"` + CARootsPEM string `json:"caRootsPEM,omitempty"` } type enclavesList struct { @@ -399,34 +400,38 @@ func (h *Handler) handleCapabilityExecute(ctx context.Context, gatewayID string, // capabilities registry and returns trusted measurement sets and CA roots // for attestation validation. Called per-request so the config stays fresh // (same pattern as CC's EnsureFreshEnclaves). -func (h *Handler) getEnclaveAttestationConfig(ctx context.Context) ([]json.RawMessage, error) { +func (h *Handler) getEnclaveAttestationConfig(ctx context.Context) ([]json.RawMessage, string, error) { dons, err := h.capRegistry.DONsForCapability(ctx, confidentialWorkflowsCapID) if err != nil { - return nil, fmt.Errorf("failed to find DON for %s: %w", confidentialWorkflowsCapID, err) + return nil, "", fmt.Errorf("failed to find DON for %s: %w", confidentialWorkflowsCapID, err) } if len(dons) == 0 { - return nil, fmt.Errorf("no DON found hosting %s", confidentialWorkflowsCapID) + return nil, "", fmt.Errorf("no DON found hosting %s", confidentialWorkflowsCapID) } capConfig, err := h.capRegistry.ConfigForCapability(ctx, confidentialWorkflowsCapID, dons[0].DON.ID) if err != nil { - return nil, fmt.Errorf("failed to get config for %s: %w", confidentialWorkflowsCapID, err) + return nil, "", fmt.Errorf("failed to get config for %s: %w", confidentialWorkflowsCapID, err) } if capConfig.DefaultConfig == nil { - return nil, fmt.Errorf("no default config for %s", confidentialWorkflowsCapID) + return nil, "", fmt.Errorf("no default config for %s", confidentialWorkflowsCapID) } var enclaves enclavesList if err := capConfig.DefaultConfig.UnwrapTo(&enclaves); err != nil { - return nil, fmt.Errorf("failed to unwrap enclave config: %w", err) + return nil, "", fmt.Errorf("failed to unwrap enclave config: %w", err) } var measurements []json.RawMessage + var caRootsPEM string for _, e := range enclaves.Enclaves { measurements = append(measurements, e.TrustedValues...) + if caRootsPEM == "" && e.CARootsPEM != "" { + caRootsPEM = e.CARootsPEM + } } - return measurements, nil + return measurements, caRootsPEM, nil } func (h *Handler) verifyAttestationHash(ctx context.Context, attestationB64 string, cleanParams any, domainTag string) error { @@ -450,7 +455,7 @@ func (h *Handler) verifyAttestationHash(ctx context.Context, attestationB64 stri // Each enclave instance may have different PCR values, so we try each set // and succeed if any match (same approach as pool.go's // validateAttestationAgainstMultipleMeasurements). - measurements, err := h.getEnclaveAttestationConfig(ctx) + measurements, caRootsPEM, err := h.getEnclaveAttestationConfig(ctx) if err != nil { return fmt.Errorf("failed to get enclave attestation config: %w", err) } @@ -461,7 +466,12 @@ func (h *Handler) verifyAttestationHash(ctx context.Context, attestationB64 stri var validationErr error for _, m := range measurements { - err := h.validateAttestation(attestationBytes, hash, m) + var err error + if caRootsPEM != "" { + err = nitro.ValidateAttestationWithRoots(attestationBytes, hash, m, caRootsPEM) + } else { + err = h.validateAttestation(attestationBytes, hash, m) + } if err == nil { return nil } From 745a181b681159c182d2d5682732998117bd8105 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 31 Mar 2026 18:08:47 +0200 Subject: [PATCH 21/67] forward relay metadata for remote capability execution (cherry picked from commit 16e4fc9417aefeba5740b5aac4c4198e6ff7c60d) --- core/capabilities/confidentialrelay/handler.go | 10 +++++++++- core/capabilities/confidentialrelay/handler_test.go | 11 +++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/core/capabilities/confidentialrelay/handler.go b/core/capabilities/confidentialrelay/handler.go index 5b6b54905c9..f3cd1f1cbb0 100644 --- a/core/capabilities/confidentialrelay/handler.go +++ b/core/capabilities/confidentialrelay/handler.go @@ -354,12 +354,20 @@ func (h *Handler) handleCapabilityExecute(ctx context.Context, gatewayID string, return h.errorResponse(ctx, gatewayID, req, jsonrpc.ErrInvalidParams, fmt.Errorf("failed to unmarshal capability request: %w", err)) } + referenceID := params.ReferenceID + if referenceID == "" { + referenceID = req.ID + } + capReq := capabilities.CapabilityRequest{ Payload: sdkReq.Payload, Method: sdkReq.Method, CapabilityId: params.CapabilityID, Metadata: capabilities.RequestMetadata{ - WorkflowID: params.WorkflowID, + WorkflowID: params.WorkflowID, + WorkflowOwner: params.Owner, + WorkflowExecutionID: params.ExecutionID, + ReferenceID: referenceID, }, } diff --git a/core/capabilities/confidentialrelay/handler_test.go b/core/capabilities/confidentialrelay/handler_test.go index 7de6e246b5e..1135242ae2c 100644 --- a/core/capabilities/confidentialrelay/handler_test.go +++ b/core/capabilities/confidentialrelay/handler_test.go @@ -182,6 +182,9 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { req: func(t *testing.T) *jsonrpc.Request[json.RawMessage] { return makeRequest(t, confidentialrelaytypes.MethodCapabilityExec, confidentialrelaytypes.CapabilityRequestParams{ WorkflowID: "wf-1", + Owner: "0xowner", + ExecutionID: "32c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce1", + ReferenceID: "17", CapabilityID: "my-cap@1.0.0", Payload: makeCapabilityPayload(t, map[string]any{"key": "val"}), Attestation: testAttestationB64, @@ -199,6 +202,14 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { assert.Equal(t, "result-proto-bytes", string(capResp.GetPayload().GetValue())) assert.Empty(t, result.Error) }, + checkExecutable: func(t *testing.T, reg *mockCapRegistry) { + exec := reg.executables["my-cap@1.0.0"] + require.NotNil(t, exec.lastRequest, "Execute should have been called") + assert.Equal(t, "wf-1", exec.lastRequest.Metadata.WorkflowID) + assert.Equal(t, "0xowner", exec.lastRequest.Metadata.WorkflowOwner) + assert.Equal(t, "32c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce1", exec.lastRequest.Metadata.WorkflowExecutionID) + assert.Equal(t, "17", exec.lastRequest.Metadata.ReferenceID) + }, }, { name: "capability execute sets Inputs from Payload for backward compat", From 1fd4841c5f1d824945f3b756d2845ddb0716ddb7 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 1 Apr 2026 14:02:52 +0200 Subject: [PATCH 22/67] add relay request metrics and use exported connector type --- .../capabilities/confidentialrelay/handler.go | 43 +++++++++++++++---- .../confidentialrelay/handler_test.go | 3 +- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/core/capabilities/confidentialrelay/handler.go b/core/capabilities/confidentialrelay/handler.go index f3cd1f1cbb0..c5fba968151 100644 --- a/core/capabilities/confidentialrelay/handler.go +++ b/core/capabilities/confidentialrelay/handler.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "time" "github.com/ethereum/go-ethereum/common" "go.opentelemetry.io/otel/attribute" @@ -55,11 +56,21 @@ type enclavesList struct { } type handlerMetrics struct { + requestCount metric.Int64Counter + requestLatency metric.Int64Histogram requestInternalError metric.Int64Counter requestSuccess metric.Int64Counter } func newMetrics() (*handlerMetrics, error) { + requestCount, err := beholder.GetMeter().Int64Counter("enclave_relay_request_count") + if err != nil { + return nil, fmt.Errorf("failed to register request count counter: %w", err) + } + requestLatency, err := beholder.GetMeter().Int64Histogram("enclave_relay_request_latency_ms", metric.WithUnit("ms")) + if err != nil { + return nil, fmt.Errorf("failed to register request latency histogram: %w", err) + } requestInternalError, err := beholder.GetMeter().Int64Counter("enclave_relay_request_internal_error") if err != nil { return nil, fmt.Errorf("failed to register internal error counter: %w", err) @@ -69,17 +80,13 @@ func newMetrics() (*handlerMetrics, error) { return nil, fmt.Errorf("failed to register success counter: %w", err) } return &handlerMetrics{ + requestCount: requestCount, + requestLatency: requestLatency, requestInternalError: requestInternalError, requestSuccess: requestSuccess, }, nil } -type gatewayConnector interface { - SendToGateway(ctx context.Context, gatewayID string, resp *jsonrpc.Response[json.RawMessage]) error - AddHandler(ctx context.Context, methods []string, handler core.GatewayConnectorHandler) error - RemoveHandler(ctx context.Context, methods []string) error -} - // attestationValidatorFunc validates a TEE attestation document. type attestationValidatorFunc func(attestation []byte, expectedUserData []byte, trustedMeasurements []byte) error @@ -90,7 +97,7 @@ type Handler struct { eng *services.Engine capRegistry core.CapabilitiesRegistry - gatewayConnector gatewayConnector + gatewayConnector core.GatewayConnector lggr logger.Logger metrics *handlerMetrics @@ -99,7 +106,7 @@ type Handler struct { validateAttestation attestationValidatorFunc } -func NewHandler(capRegistry core.CapabilitiesRegistry, conn gatewayConnector, lggr logger.Logger) (*Handler, error) { +func NewHandler(capRegistry core.CapabilitiesRegistry, conn core.GatewayConnector, lggr logger.Logger) (*Handler, error) { m, err := newMetrics() if err != nil { return nil, fmt.Errorf("failed to create metrics: %w", err) @@ -144,6 +151,21 @@ func (h *Handler) Methods() []string { func (h *Handler) HandleGatewayMessage(ctx context.Context, gatewayID string, req *jsonrpc.Request[json.RawMessage]) error { h.lggr.Debugw("received message from gateway", "gatewayID", gatewayID, "requestID", req.ID) + startTime := time.Now() + outcome := "success" + var errorCode int64 + defer func() { + attrs := []attribute.KeyValue{ + attribute.String("gateway_id", gatewayID), + attribute.String("method", req.Method), + attribute.String("outcome", outcome), + } + if errorCode != 0 { + attrs = append(attrs, attribute.Int64("error_code", errorCode)) + } + h.metrics.requestCount.Add(ctx, 1, metric.WithAttributes(attrs...)) + h.metrics.requestLatency.Record(ctx, time.Since(startTime).Milliseconds(), metric.WithAttributes(attrs...)) + }() var response *jsonrpc.Response[json.RawMessage] switch req.Method { @@ -154,8 +176,13 @@ func (h *Handler) HandleGatewayMessage(ctx context.Context, gatewayID string, re default: response = h.errorResponse(ctx, gatewayID, req, jsonrpc.ErrMethodNotFound, errors.New("unsupported method: "+req.Method)) } + if response != nil && response.Error != nil { + outcome = "error" + errorCode = response.Error.Code + } if err := h.gatewayConnector.SendToGateway(ctx, gatewayID, response); err != nil { + outcome = "send_error" h.lggr.Errorw("failed to send message to gateway", "gatewayID", gatewayID, "err", err) return err } diff --git a/core/capabilities/confidentialrelay/handler_test.go b/core/capabilities/confidentialrelay/handler_test.go index 1135242ae2c..1dcb0dd79ee 100644 --- a/core/capabilities/confidentialrelay/handler_test.go +++ b/core/capabilities/confidentialrelay/handler_test.go @@ -44,6 +44,7 @@ const testAttestationB64 = "ZHVtbXktYXR0ZXN0YXRpb24=" // base64("dummy-attestati func noopValidator(_ []byte, _, _ []byte) error { return nil } type mockGatewayConnector struct { + core.UnimplementedGatewayConnector lastResp *jsonrpc.Response[json.RawMessage] addedMethods []string removed bool @@ -114,7 +115,7 @@ func (m *mockCapRegistry) LocalNode(_ context.Context) (capabilities.Node, error return m.localNode, nil } -func newTestHandler(t *testing.T, registry core.CapabilitiesRegistry, gwConn gatewayConnector) *Handler { +func newTestHandler(t *testing.T, registry core.CapabilitiesRegistry, gwConn core.GatewayConnector) *Handler { t.Helper() lggr, err := logger.New() require.NoError(t, err) From 86d574032771bc7d0f0c51d3b81c9dc482bab74f Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 1 Apr 2026 14:08:15 +0200 Subject: [PATCH 23/67] sanitize internal relay error messages --- core/capabilities/confidentialrelay/handler.go | 12 +++++++++--- core/capabilities/confidentialrelay/handler_test.go | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/core/capabilities/confidentialrelay/handler.go b/core/capabilities/confidentialrelay/handler.go index c5fba968151..d16d5a440cc 100644 --- a/core/capabilities/confidentialrelay/handler.go +++ b/core/capabilities/confidentialrelay/handler.go @@ -36,7 +36,8 @@ import ( var _ core.GatewayConnectorHandler = (*Handler)(nil) const ( - HandlerName = "EnclaveRelayHandler" + HandlerName = "EnclaveRelayHandler" + internalErrorMessage = "internal error" // confidentialWorkflowsCapID is the capability ID for the confidential // workflows enclave pool. The relay handler uses it to look up trusted @@ -546,7 +547,7 @@ func (h *Handler) jsonResponse(req *jsonrpc.Request[json.RawMessage], result any Method: req.Method, Error: &jsonrpc.WireError{ Code: jsonrpc.ErrInternal, - Message: err.Error(), + Message: internalErrorMessage, }, } } @@ -572,13 +573,18 @@ func (h *Handler) errorResponse( attribute.Int64("error_code", errorCode), )) + message := err.Error() + if errorCode == jsonrpc.ErrInternal { + message = internalErrorMessage + } + return &jsonrpc.Response[json.RawMessage]{ Version: jsonrpc.JsonRpcVersion, ID: req.ID, Method: req.Method, Error: &jsonrpc.WireError{ Code: errorCode, - Message: err.Error(), + Message: message, }, } } diff --git a/core/capabilities/confidentialrelay/handler_test.go b/core/capabilities/confidentialrelay/handler_test.go index 1dcb0dd79ee..42a1f478951 100644 --- a/core/capabilities/confidentialrelay/handler_test.go +++ b/core/capabilities/confidentialrelay/handler_test.go @@ -281,7 +281,7 @@ func TestHandler_HandleGatewayMessage(t *testing.T) { checkResp: func(t *testing.T, resp *jsonrpc.Response[json.RawMessage]) { require.NotNil(t, resp.Error) assert.Equal(t, jsonrpc.ErrInternal, resp.Error.Code) - assert.Contains(t, resp.Error.Message, "capability not found") + assert.Equal(t, internalErrorMessage, resp.Error.Message) }, }, { From 31c3a1c5eaa14bbfcb5325828e8d64121d14e03f Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 2 Apr 2026 15:59:57 +0200 Subject: [PATCH 24/67] bump chainlink-common and sync relay config snapshots --- go.mod | 4 +--- go.sum | 9 ++------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index e3a4cd99488..a37e892c17d 100644 --- a/go.mod +++ b/go.mod @@ -85,10 +85,9 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260224214816-cb23ec38649f github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20250912190424-fd2e35d7deb5 github.com/smartcontractkit/chainlink-ccv v0.0.0-20260324000441-d4cfddc9f7d2 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260406055916-9aa6b6c0ae81 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260402120824-48154c0c65a6 github.com/smartcontractkit/chainlink-common/keystore v1.0.2 github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10 - github.com/smartcontractkit/chainlink-common/pkg/teeattestation v0.0.0-20260326144312-af5248caa880 github.com/smartcontractkit/chainlink-data-streams v0.1.13 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260330133421-5151ea0c3b05 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 @@ -293,7 +292,6 @@ require ( github.com/hashicorp/yamux v0.1.2 // indirect github.com/hasura/go-graphql-client v0.15.1 // indirect github.com/hdevalence/ed25519consensus v0.2.0 // indirect - github.com/hf/nitrite v0.0.0-20241225144000-c2d5d3c4f303 // indirect github.com/holiman/billy v0.0.0-20250707135307-f2f9b9aae7db // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect github.com/huin/goupnp v1.3.0 // indirect diff --git a/go.sum b/go.sum index 354d18b0136..71c35208109 100644 --- a/go.sum +++ b/go.sum @@ -425,7 +425,6 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/fxamacker/cbor/v2 v2.2.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gabriel-vasile/mimetype v1.4.10 h1:zyueNbySn/z8mJZHLt6IPw0KoZsiQNszIpU+bX4+ZK0= @@ -744,8 +743,6 @@ github.com/hasura/go-graphql-client v0.15.1 h1:mCb5I+8Bk3FU3GKWvf/zDXkTh7FbGlqJm github.com/hasura/go-graphql-client v0.15.1/go.mod h1:jfSZtBER3or+88Q9vFhWHiFMPppfYILRyl+0zsgPIIw= github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU= github.com/hdevalence/ed25519consensus v0.2.0/go.mod h1:w3BHWjwJbFU29IRHL1Iqkw3sus+7FctEyM4RqDxYNzo= -github.com/hf/nitrite v0.0.0-20241225144000-c2d5d3c4f303 h1:XBSq4rXFUgD8ic6Mr7dBwJN/47yg87XpZQhiknfr4Cg= -github.com/hf/nitrite v0.0.0-20241225144000-c2d5d3c4f303/go.mod h1:ycRhVmo6wegyEl6WN+zXOHUTJvB0J2tiuH88q/McTK8= github.com/holiman/billy v0.0.0-20250707135307-f2f9b9aae7db h1:IZUYC/xb3giYwBLMnr8d0TGTzPKFGNTCGgGLoyeX330= github.com/holiman/billy v0.0.0-20250707135307-f2f9b9aae7db/go.mod h1:xTEYN9KCHxuYHs+NmrmzFcnvHMzLLNiGFafCb1n3Mfg= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= @@ -1238,16 +1235,14 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20250 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20250912190424-fd2e35d7deb5/go.mod h1:xtZNi6pOKdC3sLvokDvXOhgHzT+cyBqH/gWwvxTxqrg= github.com/smartcontractkit/chainlink-ccv v0.0.0-20260324000441-d4cfddc9f7d2 h1:5HdH/A6yn8INZAltYDLb7UkUi5IKemhJzJkDW4Bgxyg= github.com/smartcontractkit/chainlink-ccv v0.0.0-20260324000441-d4cfddc9f7d2/go.mod h1:wDHq2E0KwUWG0lQ9f5frW1a7CKVW17MJLPuvKmtSRDg= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260406055916-9aa6b6c0ae81 h1:qBQxh/dndRMJX41xWEihr8FkvPL21luWwTFn/0Nl3RU= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260406055916-9aa6b6c0ae81/go.mod h1:Ob7ZRLEvPkDwGUjKdDIiHy0Mxu4+UG6oMBkR7Jv/U6o= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260402120824-48154c0c65a6 h1:oaXslIvcy5HD3zkWhx3nu8vRGdWGedYJ+XCsBD8mYkA= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260402120824-48154c0c65a6/go.mod h1:Ea94/OgfFPRTByGO2Qo+uZ7/4sWhwE2HKu7dDwdojME= github.com/smartcontractkit/chainlink-common/keystore v1.0.2 h1:AWisx4JT3QV8tcgh6J5NCrex+wAgTYpWyHsyNPSXzsQ= github.com/smartcontractkit/chainlink-common/keystore v1.0.2/go.mod h1:rSkIHdomyak3YnUtXLenl6poIq8q0V3UZPiiyYqPdGA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10 h1:FJAFgXS9oqASnkS03RE1HQwYQQxrO4l46O5JSzxqLgg= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10/go.mod h1:oiDa54M0FwxevWwyAX773lwdWvFYYlYHHQV1LQ5HpWY= github.com/smartcontractkit/chainlink-common/pkg/monitoring v0.0.0-20251215152504-b1e41f508340 h1:PsjEI+5jZIz9AS4eOsLS5VpSWJINf38clXV3wryPyMk= github.com/smartcontractkit/chainlink-common/pkg/monitoring v0.0.0-20251215152504-b1e41f508340/go.mod h1:P/0OSXUlFaxxD4B/P6HWbxYtIRmmWGDJAvanq19879c= -github.com/smartcontractkit/chainlink-common/pkg/teeattestation v0.0.0-20260326144312-af5248caa880 h1:qLrZSgc7e/exxIE4M020A52ep6xN0tXOJESiots9DvE= -github.com/smartcontractkit/chainlink-common/pkg/teeattestation v0.0.0-20260326144312-af5248caa880/go.mod h1:+X7Cb8ysHfNnPd74htGIInzZMHfUrpdDrjlr6+VW0gU= github.com/smartcontractkit/chainlink-data-streams v0.1.13 h1:YOmt545DW6U0SyaqBf+NTGDLm1yMurVI7yOvxP5hlJk= github.com/smartcontractkit/chainlink-data-streams v0.1.13/go.mod h1:00aL7OK0BJdF9gn/4t4f/pctUu2VLwwfA8G/tl9rCrM= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260330133421-5151ea0c3b05 h1:mUaNoTKE3lqlANSOlQmiYDIAjJoqHiShw+r5rl1xh7g= From cacc2cc0a3d96ff856411b63728dc90c7547e830 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 1 Apr 2026 12:57:31 +0200 Subject: [PATCH 25/67] [CRE] [5/5] Wire confidential workflow execution into CRE Wires PRs 1-4 together: - cre.go: start confidential relay service when gateway connector exists - system tests: ConfidentialRelay feature plugin, workflow registration with attributes, CompileAndDeployConfidentialWorkflow helper This branch includes code from PRs 1-4 for compilation. As those merge into develop, rebasing this branch will shrink the diff to just the wiring and system test changes. Part of #21635 --- .changeset/confidential-workflow-execution.md | 5 + core/capabilities/launcher.go | 7 + core/capabilities/launcher_test.go | 75 ++++++ .../cre/environment/environment/workflow.go | 2 +- core/services/cre/cre.go | 8 + core/services/gateway/handler_factory.go | 5 +- .../handlers/confidentialrelay/aggregator.go | 25 +- .../handlers/confidentialrelay/handler.go | 234 +++++++++--------- .../confidentialrelay/handler_test.go | 179 ++------------ core/services/workflows/syncer/v2/handler.go | 28 ++- .../workflows/v2/confidential_module.go | 64 ++--- .../workflows/v2/confidential_module_test.go | 115 +-------- deployment/cre/jobs/pkg/gateway_job.go | 9 +- .../confidential_relay/confidential_relay.go | 102 ++++++++ system-tests/lib/cre/types.go | 35 +-- system-tests/lib/cre/workflow/workflow.go | 6 +- system-tests/tests/test-helpers/t_helpers.go | 213 ++++++---------- 17 files changed, 490 insertions(+), 622 deletions(-) create mode 100644 .changeset/confidential-workflow-execution.md create mode 100644 system-tests/lib/cre/features/confidential_relay/confidential_relay.go diff --git a/.changeset/confidential-workflow-execution.md b/.changeset/confidential-workflow-execution.md new file mode 100644 index 00000000000..952e5207d47 --- /dev/null +++ b/.changeset/confidential-workflow-execution.md @@ -0,0 +1,5 @@ +--- +"chainlink": minor +--- + +Add confidential workflow execution: ConfidentialModule, relay handler, gateway wiring, single-DON capability fix #added #db_update diff --git a/core/capabilities/launcher.go b/core/capabilities/launcher.go index 32bda8c9e08..427bbc64671 100644 --- a/core/capabilities/launcher.go +++ b/core/capabilities/launcher.go @@ -410,6 +410,13 @@ func (w *launcher) OnNewRegistry(ctx context.Context, localRegistry *registrysyn if belongsToACapabilityDON { for _, myDON := range myCapabilityDONs { w.serveCapabilities(ctx, w.myPeerID, myDON, localRegistry, remoteWorkflowDONs) + + // Capability DONs may need to call capabilities on other DONs + // (e.g. relay DON calling vault DON for secret fetching). + // Without this, only workflow DONs discover cross-DON capabilities. + for _, rcd := range remoteCapabilityDONs { + w.addRemoteCapabilities(ctx, myDON, rcd, localRegistry) + } } } diff --git a/core/capabilities/launcher_test.go b/core/capabilities/launcher_test.go index 29ec22e2fe3..001fd29b3a3 100644 --- a/core/capabilities/launcher_test.go +++ b/core/capabilities/launcher_test.go @@ -2,6 +2,8 @@ package capabilities import ( "context" + "fmt" + "strings" "testing" "time" @@ -234,6 +236,79 @@ func TestLauncher(t *testing.T) { assert.Equal(t, 1, observedLogs.FilterMessage("failed to serve capability").Len()) }) + t.Run("OK-capability_DON_discovers_remote_capabilities", func(t *testing.T) { + // Regression test: a capability DON (e.g. relay DON) that needs to call + // another capability DON (e.g. vault DON) must discover it via + // addRemoteCapabilities. Previously, only workflow DONs discovered + // remote capabilities, so the relay DON couldn't find vault. + lggr, observedLogs := logger.TestObserved(t, zapcore.DebugLevel) + registry := NewRegistry(lggr) + dispatcher := remoteMocks.NewDispatcher(t) + + nodes := newNodes(4) + remoteNodes := newNodes(4) + peer := mocks.NewPeer(t) + peer.On("UpdateConnections", mock.Anything).Return(nil) + peer.On("ID").Return(nodes[0]) + peer.On("IsBootstrap").Return(false) + wrapper := mocks.NewPeerWrapper(t) + wrapper.On("GetPeer").Return(peer) + + // Local DON: capability DON (relay) with a trigger capability + fullTriggerCapID := "streams-trigger@1.0.0" + mt := newMockTrigger(capabilities.MustNewCapabilityInfo( + fullTriggerCapID, + capabilities.CapabilityTypeTrigger, + "streams trigger", + )) + require.NoError(t, registry.Add(t.Context(), mt)) + + triggerCapID := RandomUTF8BytesWord() + + localDONID := uint32(1) + remoteDONID := uint32(2) + localRegistry := buildLocalRegistry() + + // Local DON: capability DON (isPublic=true, acceptsWorkflows=false) + addDON(localRegistry, localDONID, uint32(0), uint8(1), true, false, nodes, []string{"zone-a"}, 1, [][32]byte{triggerCapID}) + addCapabilityToDON(localRegistry, localDONID, fullTriggerCapID, capabilities.CapabilityTypeTrigger, nil) + + // Remote DON: another capability DON (vault) that the local DON needs to call + remoteTargetCapID := RandomUTF8BytesWord() + fullRemoteTargetID := "vault@1.0.0" + addDON(localRegistry, remoteDONID, uint32(0), uint8(1), true, false, remoteNodes, []string{"zone-a"}, 1, [][32]byte{remoteTargetCapID}) + addCapabilityToDON(localRegistry, remoteDONID, fullRemoteTargetID, capabilities.CapabilityTypeTarget, nil) + + launcher, err := NewLauncher( + lggr, + wrapper, + nil, + nil, + dispatcher, + registry, + &mockDonNotifier{}, + ) + require.NoError(t, err) + require.NoError(t, launcher.Start(t.Context())) + defer launcher.Close() + + dispatcher.On("SetReceiver", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(nil) + + require.NoError(t, launcher.OnNewRegistry(t.Context(), localRegistry)) + // Without the fix, addRemoteCapabilities is never called for capability DONs. + // With the fix, the launcher attempts to add vault@1.0.0 as a remote capability. + // It may fail downstream (signer setup) but the attempt proves discovery works. + allLogs := observedLogs.All() + found := false + for _, entry := range allLogs { + if strings.Contains(entry.Message, "remote capability") && strings.Contains(fmt.Sprintf("%v", entry.ContextMap()), fullRemoteTargetID) { + found = true + break + } + } + assert.True(t, found, "expected capability DON to attempt remote capability discovery for %s", fullRemoteTargetID) + }) + t.Run("start and close with nil peer wrapper", func(t *testing.T) { lggr := logger.Test(t) registry := NewRegistry(lggr) diff --git a/core/scripts/cre/environment/environment/workflow.go b/core/scripts/cre/environment/environment/workflow.go index 407c25f7eb0..631e6e226a1 100644 --- a/core/scripts/cre/environment/environment/workflow.go +++ b/core/scripts/cre/environment/environment/workflow.go @@ -475,7 +475,7 @@ func deployWorkflow( fmt.Printf("\n⚙️ Registering workflow '%s' with the workflow registry\n\n", workflowNameFlag) - workflowID, registerErr := creworkflow.RegisterWithContract(ctx, sethClient, common.HexToAddress(workflowRegistryAddress), workflowRegistryVersion, uint64(donIDFlag), workflowNameFlag, "file://"+wasmWorkflowFilePathFlag, configPath, secretsPath, &containerTargetDirFlag) + workflowID, registerErr := creworkflow.RegisterWithContract(ctx, sethClient, common.HexToAddress(workflowRegistryAddress), workflowRegistryVersion, uint64(donIDFlag), workflowNameFlag, "file://"+wasmWorkflowFilePathFlag, configPath, secretsPath, nil, &containerTargetDirFlag) if registerErr != nil { return errors.Wrapf(registerErr, "❌ failed to register workflow %s", workflowNameFlag) } diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index 1e9dd386c01..143d6d40e29 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -32,6 +32,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/capabilities" "github.com/smartcontractkit/chainlink/v2/core/capabilities/compute" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/confidentialrelay" gatewayconnector "github.com/smartcontractkit/chainlink/v2/core/capabilities/gateway_connector" "github.com/smartcontractkit/chainlink/v2/core/capabilities/localcapmgr" "github.com/smartcontractkit/chainlink/v2/core/capabilities/remote" @@ -171,6 +172,13 @@ func (s *Services) newSubservices( } s.GatewayConnectorWrapper = gatewayConnectorWrapper srvs = append(srvs, gatewayConnectorWrapper) + + relayService := confidentialrelay.NewService( + gatewayConnectorWrapper, + opts.CapabilitiesRegistry, + lggr, + ) + srvs = append(srvs, relayService) } if cfg.CRE().Linking().URL() != "" { diff --git a/core/services/gateway/handler_factory.go b/core/services/gateway/handler_factory.go index 38a1ca0e299..80f3a08fecc 100644 --- a/core/services/gateway/handler_factory.go +++ b/core/services/gateway/handler_factory.go @@ -86,9 +86,10 @@ func (hf *handlerFactory) NewHandler( case HTTPCapabilityType: return v2.NewGatewayHandler(handlerConfig, donConfig, don, hf.httpClient, hf.lggr, hf.lf) case VaultHandlerType: - return vault.NewHandler(handlerConfig, donConfig, don, hf.capabilitiesRegistry, hf.workflowRegistrySyncer, hf.lggr, clockwork.NewRealClock(), hf.lf) + requestAuthorizer := vaultcap.NewRequestAuthorizer(hf.lggr, hf.workflowRegistrySyncer) + return vault.NewHandler(handlerConfig, donConfig, don, hf.capabilitiesRegistry, requestAuthorizer, hf.lggr, clockwork.NewRealClock(), hf.lf) case ConfidentialRelayHandlerType: - return confidentialrelay.NewHandler(handlerConfig, donConfig, don, hf.lggr, clockwork.NewRealClock(), hf.lf) + return confidentialrelay.NewHandler(handlerConfig, donConfig, don, hf.lggr, clockwork.NewRealClock()) default: return nil, fmt.Errorf("unsupported handler type %s", handlerType) } diff --git a/core/services/gateway/handlers/confidentialrelay/aggregator.go b/core/services/gateway/handlers/confidentialrelay/aggregator.go index 657cc78c639..5eb00664571 100644 --- a/core/services/gateway/handlers/confidentialrelay/aggregator.go +++ b/core/services/gateway/handlers/confidentialrelay/aggregator.go @@ -4,8 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "maps" - "slices" jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -43,27 +41,8 @@ func (a *aggregator) Aggregate(resps map[string]jsonrpc.Response[json.RawMessage if shaToCount[sha] > maxShaToCount { maxShaToCount = shaToCount[sha] } - } - - var qualifiedDigests []string - for sha, n := range shaToCount { - if n >= requiredQuorum { - qualifiedDigests = append(qualifiedDigests, sha) - } - } - if len(qualifiedDigests) > 0 { - slices.Sort(qualifiedDigests) - want := qualifiedDigests[0] - for _, k := range slices.Sorted(maps.Keys(resps)) { - r := resps[k] - sha, err := r.Digest() - if err != nil { - continue - } - if sha == want { - out := r - return &out, nil - } + if shaToCount[sha] >= requiredQuorum { + return &r, nil } } diff --git a/core/services/gateway/handlers/confidentialrelay/handler.go b/core/services/gateway/handlers/confidentialrelay/handler.go index 3d99174da44..0f746fa456d 100644 --- a/core/services/gateway/handlers/confidentialrelay/handler.go +++ b/core/services/gateway/handlers/confidentialrelay/handler.go @@ -5,23 +5,21 @@ import ( "encoding/json" "errors" "fmt" + "maps" "strconv" "sync" - "sync/atomic" "time" "github.com/jonboulle/clockwork" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" - "golang.org/x/sync/errgroup" "github.com/smartcontractkit/chainlink-common/pkg/beholder" relaytypes "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/actions/confidentialrelay" jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/ratelimit" "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" - "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/api" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" gwhandlers "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers" @@ -114,7 +112,8 @@ type relayAggregator interface { } type Config struct { - RequestTimeoutSec int `json:"requestTimeoutSec"` + NodeRateLimiter ratelimit.RateLimiterConfig `json:"nodeRateLimiter"` + RequestTimeoutSec int `json:"requestTimeoutSec"` } type handler struct { @@ -126,9 +125,8 @@ type handler struct { mu sync.RWMutex stopCh services.StopChan - globalNodeRateLimiter limits.RateLimiter - perNodeRateLimiters map[string]limits.RateLimiter - requestTimeout time.Duration + nodeRateLimiter *ratelimit.RateLimiter + requestTimeout time.Duration activeRequests map[string]*activeRequest metrics *metrics @@ -146,7 +144,7 @@ func (h *handler) Name() string { return h.lggr.Name() } -func NewHandler(methodConfig json.RawMessage, donConfig *config.DONConfig, don gwhandlers.DON, lggr logger.Logger, clock clockwork.Clock, limitsFactory limits.Factory) (*handler, error) { +func NewHandler(methodConfig json.RawMessage, donConfig *config.DONConfig, don gwhandlers.DON, lggr logger.Logger, clock clockwork.Clock) (*handler, error) { var cfg Config if err := json.Unmarshal(methodConfig, &cfg); err != nil { return nil, fmt.Errorf("failed to unmarshal method config: %w", err) @@ -156,18 +154,9 @@ func NewHandler(methodConfig json.RawMessage, donConfig *config.DONConfig, don g cfg.RequestTimeoutSec = 30 } - globalNodeRateLimiter, err := limitsFactory.MakeRateLimiter(cresettings.Default.GatewayConfidentialRelayGlobalRate) + nodeRateLimiter, err := ratelimit.NewRateLimiter(cfg.NodeRateLimiter) if err != nil { - return nil, fmt.Errorf("failed to create global node rate limiter: %w", err) - } - - perNodeRateLimiters := make(map[string]limits.RateLimiter, len(donConfig.Members)) - for _, member := range donConfig.Members { - rl, makeErr := limitsFactory.MakeRateLimiter(cresettings.Default.GatewayConfidentialRelayPerNodeRate) - if makeErr != nil { - return nil, fmt.Errorf("failed to create per-node rate limiter for %s: %w", member.Address, makeErr) - } - perNodeRateLimiters[member.Address] = rl + return nil, fmt.Errorf("failed to create node rate limiter: %w", err) } metrics, err := newMetrics() @@ -176,18 +165,17 @@ func NewHandler(methodConfig json.RawMessage, donConfig *config.DONConfig, don g } return &handler{ - donConfig: donConfig, - don: don, - lggr: logger.Named(lggr, "ConfidentialRelayHandler:"+donConfig.DonId), - requestTimeout: time.Duration(cfg.RequestTimeoutSec) * time.Second, - globalNodeRateLimiter: globalNodeRateLimiter, - perNodeRateLimiters: perNodeRateLimiters, - activeRequests: make(map[string]*activeRequest), - mu: sync.RWMutex{}, - stopCh: make(services.StopChan), - metrics: metrics, - aggregator: &aggregator{}, - clock: clock, + donConfig: donConfig, + don: don, + lggr: logger.Named(lggr, "ConfidentialRelayHandler:"+donConfig.DonId), + requestTimeout: time.Duration(cfg.RequestTimeoutSec) * time.Second, + nodeRateLimiter: nodeRateLimiter, + activeRequests: make(map[string]*activeRequest), + mu: sync.RWMutex{}, + stopCh: make(services.StopChan), + metrics: metrics, + aggregator: &aggregator{}, + clock: clock, }, nil } @@ -216,14 +204,7 @@ func (h *handler) Close() error { return h.StopOnce("ConfidentialRelayHandler", func() error { h.lggr.Info("closing confidential relay handler") close(h.stopCh) - var err error - if h.globalNodeRateLimiter != nil { - err = errors.Join(err, h.globalNodeRateLimiter.Close()) - } - for _, rl := range h.perNodeRateLimiters { - err = errors.Join(err, rl.Close()) - } - return err + return nil }) } @@ -241,7 +222,8 @@ func (h *handler) removeExpiredRequests(ctx context.Context) { for _, er := range expiredRequests { responses := er.copiedResponses() h.lggr.Debugw("request expired without quorum", "requestID", er.req.ID, "responseCount", len(responses), "required", h.donConfig.F+1) - err := h.sendResponseAndCleanup(ctx, er, h.constructErrorResponse(er.req, api.RequestTimeoutError, fmt.Errorf("request expired: got %d/%d responses", len(responses), h.donConfig.F+1))) + // sendResponse deletes the request from activeRequests after sending. + err := h.sendResponse(ctx, er, h.errorResponse(er.req, api.RequestTimeoutError, fmt.Errorf("request expired: got %d/%d responses", len(responses), h.donConfig.F+1), nil)) if err != nil { h.lggr.Errorw("error sending response to user", "requestID", er.req.ID, "error", err) } @@ -302,18 +284,10 @@ func (h *handler) HandleNodeMessage(ctx context.Context, resp *jsonrpc.Response[ l := logger.With(h.lggr, "method", resp.Method, "requestID", resp.ID, "nodeAddr", nodeAddr) l.Debugw("handling node response") - nodeRateLimiter, ok := h.perNodeRateLimiters[nodeAddr] - if !ok { - return fmt.Errorf("received message from unexpected node %s", nodeAddr) - } - if !nodeRateLimiter.Allow(ctx) { + if !h.nodeRateLimiter.Allow(nodeAddr) { l.Debugw("node is rate limited", "nodeAddr", nodeAddr) return nil } - if !h.globalNodeRateLimiter.Allow(ctx) { - l.Debug("global relay rate limit exceeded") - return nil - } ar := h.getActiveRequest(resp.ID) if ar == nil { @@ -321,8 +295,8 @@ func (h *handler) HandleNodeMessage(ctx context.Context, resp *jsonrpc.Response[ return nil } - added := ar.addResponseForNode(nodeAddr, resp) - if !added { + ok := ar.addResponseForNode(nodeAddr, resp) + if !ok { l.Errorw("duplicate response from node, ignoring", "nodeAddr", nodeAddr) return nil } @@ -333,113 +307,129 @@ func (h *handler) HandleNodeMessage(ctx context.Context, resp *jsonrpc.Response[ case errors.Is(err, errInsufficientResponsesForQuorum): l.Debugw("aggregating responses, waiting for other nodes...", "error", err) return nil - case errors.Is(err, errQuorumUnobtainable): - l.Errorw("quorum unobtainable, returning error to user", "error", err) - return h.sendResponseAndCleanup(ctx, ar, h.constructErrorResponse(ar.req, api.FatalError, err)) case err != nil: - l.Errorw("unexpected aggregation error", "error", err) - return h.sendResponseAndCleanup(ctx, ar, h.constructErrorResponse(ar.req, api.FatalError, err)) + l.Error("quorum unobtainable, returning response to user...", "error", err, "responses", maps.Values(ar.responses)) + return h.sendResponse(ctx, ar, h.errorResponse(ar.req, api.FatalError, err, nil)) } - rawResponse, err := jsonrpc.EncodeResponse(aggregatedResp) - if err != nil { - h.lggr.Errorw("failed to encode response", "requestID", ar.req.ID, "error", err) - return h.sendResponseAndCleanup(ctx, ar, h.constructErrorResponse(ar.req, api.NodeReponseEncodingError, err)) - } - return h.sendResponseAndCleanup(ctx, ar, gwhandlers.UserCallbackPayload{ - RawResponse: rawResponse, - ErrorCode: api.NoError, - }) + return h.sendSuccessResponse(ctx, l, ar, aggregatedResp) } func (h *handler) fanOutToNodes(ctx context.Context, l logger.Logger, ar *activeRequest) error { - var ( - group errgroup.Group - nodeErrors atomic.Uint32 - ) - + var nodeErrors []error for _, node := range h.donConfig.Members { - group.Go(func() error { - err := h.don.SendToNode(ctx, node.Address, &ar.req) - if err != nil { - nodeErrors.Add(1) - l.Errorw("error sending request to node", "node", node.Address, "error", err) - } - return nil - }) + err := h.don.SendToNode(ctx, node.Address, &ar.req) + if err != nil { + nodeErrors = append(nodeErrors, err) + l.Errorw("error sending request to node", "node", node.Address, "error", err) + } } - _ = group.Wait() - - numNodeErrors := nodeErrors.Load() - remainingPossibleResponses := len(h.donConfig.Members) - int(numNodeErrors) - if remainingPossibleResponses < h.donConfig.F+1 && numNodeErrors > 0 { - return h.sendResponseAndCleanup(ctx, ar, h.constructErrorResponse(ar.req, api.FatalError, errors.New("failed to forward user request to nodes"))) + if len(nodeErrors) == len(h.donConfig.Members) && len(nodeErrors) > 0 { + return h.sendResponse(ctx, ar, h.errorResponse(ar.req, api.FatalError, errors.New("failed to forward user request to nodes"), nil)) } l.Debugw("successfully forwarded request to relay nodes") return nil } -// sendResponseAndCleanup sends payload. -// The request is always removed from activeRequests -// regardless of whether the send succeeds, since a failed callback cannot -// be retried. -func (h *handler) sendResponseAndCleanup(ctx context.Context, ar *activeRequest, payload gwhandlers.UserCallbackPayload) error { - h.recordMetrics(ctx, payload.ErrorCode) - sendErr := ar.SendResponse(payload) - - h.mu.Lock() - delete(h.activeRequests, ar.req.ID) - h.mu.Unlock() - - if sendErr != nil { - h.lggr.Errorw("error sending response to user", "requestID", ar.req.ID, "error", sendErr) - return sendErr +func (h *handler) sendSuccessResponse(ctx context.Context, l logger.Logger, ar *activeRequest, resp *jsonrpc.Response[json.RawMessage]) error { + rawResponse, err := jsonrpc.EncodeResponse(resp) + if err != nil { + l.Errorw("failed to encode response", "error", err) + return h.sendResponse(ctx, ar, h.errorResponse(ar.req, api.NodeReponseEncodingError, fmt.Errorf("failed to marshal response: %w", err), nil)) } - h.lggr.Debugw("response sent to user", "requestID", ar.req.ID, "errorCode", payload.ErrorCode) - return nil -} + var errorCode api.ErrorCode + if resp.Error != nil { + errorCode = api.FromJSONRPCErrorCode(resp.Error.Code) + } else { + errorCode = api.NoError + } -func (h *handler) recordMetrics(ctx context.Context, errorCode api.ErrorCode) { - //nolint:exhaustive // do not record other errors - switch errorCode { - case api.HandlerError: - h.metrics.requestInternalError.Add(ctx, 1, metric.WithAttributes( - attribute.String("don_id", h.donConfig.DonId), - attribute.String("error", errorCode.String()), - )) - case api.UnsupportedDONIdError: - h.metrics.requestUserError.Add(ctx, 1, metric.WithAttributes( - attribute.String("don_id", h.donConfig.DonId), - )) - case api.NoError: - h.metrics.requestSuccess.Add(ctx, 1, metric.WithAttributes( - attribute.String("don_id", h.donConfig.DonId), - )) + l.Debugw("issued user callback", "errorCode", errorCode) + successResp := gwhandlers.UserCallbackPayload{ + RawResponse: rawResponse, + ErrorCode: errorCode, } + return h.sendResponse(ctx, ar, successResp) } -func (h *handler) constructErrorResponse(req jsonrpc.Request[json.RawMessage], errorCode api.ErrorCode, err error) gwhandlers.UserCallbackPayload { - //nolint:exhaustive // do not modify other error codes +func (h *handler) errorResponse( + req jsonrpc.Request[json.RawMessage], + errorCode api.ErrorCode, + err error, + data []byte, +) gwhandlers.UserCallbackPayload { switch errorCode { + case api.FatalError: case api.NodeReponseEncodingError: + h.lggr.Errorw(err.Error(), "requestID", req.ID) err = errors.New(errorCode.String()) case api.InvalidParamsError: + h.lggr.Errorw("invalid params", "requestID", req.ID, "params", string(*req.Params)) err = fmt.Errorf("invalid params error: %w", err) case api.UnsupportedMethodError: + h.lggr.Errorw("unsupported method", "requestID", req.ID, "method", req.Method, "error", err.Error()) err = fmt.Errorf("unsupported method(%s): %w", req.Method, err) case api.UserMessageParseError: + h.lggr.Errorw("user message parse error", "requestID", req.ID, "error", err.Error()) err = fmt.Errorf("user message parse error: %w", err) + case api.NoError: + case api.UnsupportedDONIdError: + case api.HandlerError: + case api.RequestTimeoutError: + case api.StaleNodeResponseError: + case api.ConflictError: + case api.LimitExceededError: } + return gwhandlers.UserCallbackPayload{ RawResponse: h.codec.EncodeNewErrorResponse( req.ID, api.ToJSONRPCErrorCode(errorCode), err.Error(), - nil, + data, ), ErrorCode: errorCode, } } + +func (h *handler) sendResponse(ctx context.Context, userRequest *activeRequest, resp gwhandlers.UserCallbackPayload) error { + switch resp.ErrorCode { + case api.StaleNodeResponseError: + case api.FatalError: + case api.NodeReponseEncodingError: + case api.RequestTimeoutError: + case api.HandlerError: + h.metrics.requestInternalError.Add(ctx, 1, metric.WithAttributes( + attribute.String("don_id", h.donConfig.DonId), + attribute.String("error", resp.ErrorCode.String()), + )) + case api.InvalidParamsError: + case api.UnsupportedMethodError: + case api.UserMessageParseError: + case api.UnsupportedDONIdError: + h.metrics.requestUserError.Add(ctx, 1, metric.WithAttributes( + attribute.String("don_id", h.donConfig.DonId), + )) + case api.NoError: + h.metrics.requestSuccess.Add(ctx, 1, metric.WithAttributes( + attribute.String("don_id", h.donConfig.DonId), + )) + case api.ConflictError: + case api.LimitExceededError: + } + + err := userRequest.SendResponse(resp) + if err != nil { + h.lggr.Errorw("error sending response to user", "requestID", userRequest.req.ID, "error", err) + return err + } + + h.mu.Lock() + defer h.mu.Unlock() + delete(h.activeRequests, userRequest.req.ID) + h.lggr.Debugw("response sent to user", "requestID", userRequest.req.ID, "errorCode", resp.ErrorCode) + return nil +} diff --git a/core/services/gateway/handlers/confidentialrelay/handler_test.go b/core/services/gateway/handlers/confidentialrelay/handler_test.go index 0cf3537405d..efc6e2c1689 100644 --- a/core/services/gateway/handlers/confidentialrelay/handler_test.go +++ b/core/services/gateway/handlers/confidentialrelay/handler_test.go @@ -14,12 +14,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "golang.org/x/time/rate" jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" - "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + "github.com/smartcontractkit/chainlink-common/pkg/ratelimit" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/api" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" @@ -27,38 +25,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/mocks" ) -type barrierDON struct { - total int - mu sync.Mutex - started int - allStarted chan struct{} - releaseOnce sync.Once -} - -func newBarrierDON(total int) *barrierDON { - return &barrierDON{ - total: total, - allStarted: make(chan struct{}), - } -} - -func (d *barrierDON) SendToNode(ctx context.Context, _ string, _ *jsonrpc.Request[json.RawMessage]) error { - d.mu.Lock() - d.started++ - if d.started == d.total { - d.releaseOnce.Do(func() { close(d.allStarted) }) - } - ch := d.allStarted - d.mu.Unlock() - - select { - case <-ch: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} - var nodeOne = config.NodeConfig{ Name: "node1", Address: "0x1234", @@ -84,13 +50,18 @@ func setupHandler(t *testing.T, numNodes int) (*handler, *common.Callback, *mock } handlerConfig := Config{ RequestTimeoutSec: 30, + NodeRateLimiter: ratelimit.RateLimiterConfig{ + GlobalRPS: 100, + GlobalBurst: 100, + PerSenderRPS: 10, + PerSenderBurst: 10, + }, } methodConfig, err := json.Marshal(handlerConfig) require.NoError(t, err) clock := clockwork.NewFakeClock() - limitsFactory := limits.Factory{Settings: cresettings.DefaultGetter, Logger: lggr} - h, err := NewHandler(methodConfig, donConfig, don, lggr, clock, limitsFactory) + h, err := NewHandler(methodConfig, donConfig, don, lggr, clock) require.NoError(t, err) h.aggregator = &mockAggregator{} cb := common.NewCallback() @@ -391,6 +362,12 @@ func TestConfidentialRelayHandler_DuplicateRequestID(t *testing.T) { func TestConfidentialRelayHandler_RateLimitedNode(t *testing.T) { handlerConfig := Config{ RequestTimeoutSec: 30, + NodeRateLimiter: ratelimit.RateLimiterConfig{ + GlobalRPS: 100, + GlobalBurst: 100, + PerSenderRPS: 0.001, // Effectively zero + PerSenderBurst: 1, + }, } methodConfig, err := json.Marshal(handlerConfig) require.NoError(t, err) @@ -403,12 +380,9 @@ func TestConfidentialRelayHandler_RateLimitedNode(t *testing.T) { Members: []config.NodeConfig{nodeOne}, } clock := clockwork.NewFakeClock() - limitsFactory := limits.Factory{Settings: cresettings.DefaultGetter, Logger: lggr} - h, err := NewHandler(methodConfig, donConfig, don, lggr, clock, limitsFactory) + h, err := NewHandler(methodConfig, donConfig, don, lggr, clock) require.NoError(t, err) h.aggregator = &respondingMockAggregator{} - h.globalNodeRateLimiter = limits.GlobalRateLimiter(rate.Limit(100), 100) - h.perNodeRateLimiters[nodeOne.Address] = limits.GlobalRateLimiter(rate.Limit(0.001), 1) don.On("SendToNode", mock.Anything, mock.Anything, mock.Anything).Return(nil) @@ -521,129 +495,6 @@ func TestConfidentialRelayHandler_AllNodesFanOutFail(t *testing.T) { wg.Wait() } -func TestConfidentialRelayHandler_FanOutWaitsWhileQuorumStillPossible(t *testing.T) { - h, cb, don, _ := setupHandler(t, 4) - don.On("SendToNode", mock.Anything, mock.Anything, mock.Anything).Return( - func(_ context.Context, nodeAddress string, _ *jsonrpc.Request[json.RawMessage]) error { - switch nodeAddress { - case "0x0000", "0x0001": - return errors.New("connection refused") - default: - return nil - } - }, - ) - - params := json.RawMessage(`{"workflow_id":"wf1"}`) - req := jsonrpc.Request[json.RawMessage]{ - ID: "req-still-possible", - Method: MethodCapabilityExec, - Params: ¶ms, - } - - err := h.HandleJSONRPCUserMessage(t.Context(), req, cb) - require.NoError(t, err) - - require.NotNil(t, h.getActiveRequest(req.ID), "request should remain active while quorum is still possible") - - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond) - defer cancel() - _, err = cb.Wait(ctx) - require.Error(t, err) -} - -func TestConfidentialRelayHandler_FanOutFailsWhenQuorumBecomesImpossible(t *testing.T) { - h, cb, don, _ := setupHandler(t, 4) - don.On("SendToNode", mock.Anything, mock.Anything, mock.Anything).Return( - func(_ context.Context, nodeAddress string, _ *jsonrpc.Request[json.RawMessage]) error { - switch nodeAddress { - case "0x0000", "0x0001", "0x0002": - return errors.New("connection refused") - default: - return nil - } - }, - ) - - params := json.RawMessage(`{"workflow_id":"wf1"}`) - req := jsonrpc.Request[json.RawMessage]{ - ID: "req-quorum-impossible", - Method: MethodCapabilityExec, - Params: ¶ms, - } - - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - resp, err := cb.Wait(t.Context()) - assert.NoError(t, err) - assert.Equal(t, api.FatalError, resp.ErrorCode) - var jsonResp jsonrpc.Response[json.RawMessage] - err = json.Unmarshal(resp.RawResponse, &jsonResp) - assert.NoError(t, err) - assert.Contains(t, jsonResp.Error.Message, "failed to forward user request to nodes") - }() - - err := h.HandleJSONRPCUserMessage(t.Context(), req, cb) - require.NoError(t, err) - wg.Wait() - - require.Nil(t, h.getActiveRequest(req.ID), "request should be cleaned up once quorum is impossible") -} - -func TestConfidentialRelayHandler_FanOutToNodes_IsConcurrent(t *testing.T) { - lggr := logger.Test(t) - don := newBarrierDON(2) - donConfig := &config.DONConfig{ - DonId: "test_relay_don", - F: 1, - Members: []config.NodeConfig{ - {Name: "node0", Address: "0x0000"}, - {Name: "node1", Address: "0x0001"}, - }, - } - - methodConfig, err := json.Marshal(Config{ - RequestTimeoutSec: 30, - }) - require.NoError(t, err) - - limitsFactory := limits.Factory{Settings: cresettings.DefaultGetter, Logger: lggr} - h, err := NewHandler(methodConfig, donConfig, don, lggr, clockwork.NewFakeClock(), limitsFactory) - require.NoError(t, err) - - cb := common.NewCallback() - params := json.RawMessage(`{"workflow_id":"wf1"}`) - req := jsonrpc.Request[json.RawMessage]{ - ID: "req-concurrent-fanout", - Method: MethodCapabilityExec, - Params: ¶ms, - } - - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - - done := make(chan error, 1) - go func() { - done <- h.HandleJSONRPCUserMessage(ctx, req, cb) - }() - - select { - case err := <-done: - require.NoError(t, err) - case <-time.After(100 * time.Millisecond): - cancel() - <-done - t.Fatal("HandleJSONRPCUserMessage did not fan out to nodes concurrently") - } - - don.mu.Lock() - started := don.started - don.mu.Unlock() - assert.Equal(t, 2, started) -} - func TestConfidentialRelayHandler_CapabilityExecMethod(t *testing.T) { h, cb, don, _ := setupHandler(t, 4) h.aggregator = &respondingMockAggregator{} diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 0f357ba996e..2bda9999383 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -784,7 +784,19 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp h.lggr.Infow("routing workflow to confidential execution", "workflowID", spec.WorkflowID) engine, err = h.confidentialEngineFactory(spec, workflowName, decodedBinary, initDone) } else { - engine, err = h.engineFactory(ctx, spec.WorkflowID, spec.WorkflowOwner, workflowName, spec.WorkflowTag, configBytes, decodedBinary, initDone) + // Scope the engineFactory call so that decodedBinary goes out of scope immediately after the factory returns + engine, err = func() (services.Service, error) { + return h.engineFactory( + ctx, + spec.WorkflowID, + spec.WorkflowOwner, + workflowName, + spec.WorkflowTag, + configBytes, + decodedBinary, + initDone, + ) + }() } if err != nil { return fmt.Errorf("failed to create workflow engine: %w", err) @@ -883,11 +895,6 @@ func (h *eventHandler) newV2EngineConfig( SecretsFetcher: h.secretsFetcher, DebugMode: h.debugMode, SdkName: sdkName, - - ShardOrchestratorClient: h.shardOrchestratorClient, - ShardingEnabled: h.shardingEnabled, - MyShardID: h.myShardID, - ShardRoutingSteady: h.shardRoutingSteady, } } @@ -928,16 +935,15 @@ func (h *eventHandler) confidentialEngineFactory( lggr := logger.Named(h.lggr, "WorkflowEngine.ConfidentialModule") lggr = logger.With(lggr, "workflowID", spec.WorkflowID, "workflowName", spec.WorkflowName, "workflowOwner", spec.WorkflowOwner) - // nil resolver: raw binaryURL is passed to the enclave as-is. - // PR 5/5 (#21642) wires this to the storage service retriever - // so the enclave receives a presigned URL. module := v2.NewConfidentialModule( h.capRegistry, spec.BinaryURL, binaryHash, - spec.WorkflowID, spec.WorkflowOwner, workflowName.String(), spec.WorkflowTag, + spec.WorkflowID, + spec.WorkflowOwner, + workflowName.String(), + spec.WorkflowTag, attrs.VaultDonSecrets, - nil, lggr, ) diff --git a/core/services/workflows/v2/confidential_module.go b/core/services/workflows/v2/confidential_module.go index 246886bcebf..4cc3761e18a 100644 --- a/core/services/workflows/v2/confidential_module.go +++ b/core/services/workflows/v2/confidential_module.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha256" "encoding/json" - "errors" "fmt" "google.golang.org/protobuf/proto" @@ -57,26 +56,19 @@ func IsConfidential(data []byte) (bool, error) { return attrs.Confidential, nil } -// BinaryURLResolver resolves a raw binary URL into an ephemeral/presigned -// URL that the enclave can fetch without authentication. In production this -// calls the CRE storage service; nil means the raw URL is used as-is. -// PR 5/5 (#21642) wires this to the storage service retriever. -type BinaryURLResolver func(ctx context.Context, workflowID string) (string, error) - // ConfidentialModule implements host.ModuleV2 for confidential workflows. // Instead of running WASM locally, it delegates execution to the // confidential-workflows capability via the CapabilitiesRegistry. type ConfidentialModule struct { - capRegistry core.CapabilitiesRegistry - binaryURL string - binaryHash []byte - workflowID string - workflowOwner string - workflowName string - workflowTag string - vaultDonSecrets []SecretIdentifier - binaryURLResolver BinaryURLResolver - lggr logger.Logger + capRegistry core.CapabilitiesRegistry + binaryURL string + binaryHash []byte + workflowID string + workflowOwner string + workflowName string + workflowTag string + vaultDonSecrets []SecretIdentifier + lggr logger.Logger } var _ host.ModuleV2 = (*ConfidentialModule)(nil) @@ -85,22 +77,23 @@ func NewConfidentialModule( capRegistry core.CapabilitiesRegistry, binaryURL string, binaryHash []byte, - workflowID, workflowOwner, workflowName, workflowTag string, + workflowID string, + workflowOwner string, + workflowName string, + workflowTag string, vaultDonSecrets []SecretIdentifier, - binaryURLResolver BinaryURLResolver, lggr logger.Logger, ) *ConfidentialModule { return &ConfidentialModule{ - capRegistry: capRegistry, - binaryURL: binaryURL, - binaryHash: binaryHash, - workflowID: workflowID, - workflowOwner: workflowOwner, - workflowName: workflowName, - workflowTag: workflowTag, - vaultDonSecrets: vaultDonSecrets, - binaryURLResolver: binaryURLResolver, - lggr: lggr, + capRegistry: capRegistry, + binaryURL: binaryURL, + binaryHash: binaryHash, + workflowID: workflowID, + workflowOwner: workflowOwner, + workflowName: workflowName, + workflowTag: workflowTag, + vaultDonSecrets: vaultDonSecrets, + lggr: lggr, } } @@ -131,20 +124,11 @@ func (m *ConfidentialModule) Execute( } } - binaryURL := m.binaryURL - if m.binaryURLResolver != nil { - resolved, resolveErr := m.binaryURLResolver(ctx, m.workflowID) - if resolveErr != nil { - return nil, fmt.Errorf("failed to resolve binary URL: %w", resolveErr) - } - binaryURL = resolved - } - capInput := &confworkflowtypes.ConfidentialWorkflowRequest{ VaultDonSecrets: protoSecrets, Execution: &confworkflowtypes.WorkflowExecution{ WorkflowId: m.workflowID, - BinaryUrl: binaryURL, + BinaryUrl: m.binaryURL, BinaryHash: m.binaryHash, ExecuteRequest: execReqBytes, Owner: m.workflowOwner, @@ -181,7 +165,7 @@ func (m *ConfidentialModule) Execute( } if capResp.Payload == nil { - return nil, errors.New("confidential-workflows capability returned nil payload") + return nil, fmt.Errorf("confidential-workflows capability returned nil payload") } var confResp confworkflowtypes.ConfidentialWorkflowResponse diff --git a/core/services/workflows/v2/confidential_module_test.go b/core/services/workflows/v2/confidential_module_test.go index 96a58cafa13..87dab426022 100644 --- a/core/services/workflows/v2/confidential_module_test.go +++ b/core/services/workflows/v2/confidential_module_test.go @@ -3,7 +3,7 @@ package v2 import ( "context" "crypto/sha256" - "errors" + "fmt" "testing" "time" @@ -24,7 +24,6 @@ import ( sdkpb "github.com/smartcontractkit/chainlink-protos/cre/go/sdk" valuespb "github.com/smartcontractkit/chainlink-protos/cre/go/values/pb" - wfpb "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" ) // stubExecutionHelper implements host.ExecutionHelper for testing. @@ -44,9 +43,6 @@ func (s *stubExecutionHelper) GetWorkflowExecutionID() string { return s.executi func (s *stubExecutionHelper) GetNodeTime() time.Time { return time.Time{} } func (s *stubExecutionHelper) GetDONTime() (time.Time, error) { return time.Time{}, nil } func (s *stubExecutionHelper) EmitUserLog(string) error { return nil } -func (s *stubExecutionHelper) EmitUserMetric(context.Context, *wfpb.WorkflowUserMetric) error { - return nil -} func TestParseWorkflowAttributes(t *testing.T) { t.Run("valid JSON with all fields", func(t *testing.T) { @@ -56,7 +52,7 @@ func TestParseWorkflowAttributes(t *testing.T) { assert.True(t, attrs.Confidential) require.Len(t, attrs.VaultDonSecrets, 2) assert.Equal(t, "API_KEY", attrs.VaultDonSecrets[0].Key) - assert.Empty(t, attrs.VaultDonSecrets[0].Namespace) + assert.Equal(t, "", attrs.VaultDonSecrets[0].Namespace) assert.Equal(t, "SIGNING_KEY", attrs.VaultDonSecrets[1].Key) assert.Equal(t, "custom-ns", attrs.VaultDonSecrets[1].Namespace) }) @@ -176,7 +172,6 @@ func TestConfidentialModule_Execute(t *testing.T) { {Key: "API_KEY"}, {Key: "SIGNING_KEY", Namespace: "custom-ns"}, }, - nil, lggr, ) @@ -210,7 +205,6 @@ func TestConfidentialModule_Execute(t *testing.T) { []byte("hash"), "wf-1", "owner", "name", "tag", []SecretIdentifier{{Key: "SECRET_A"}}, // no namespace - nil, lggr, ) @@ -228,10 +222,10 @@ func TestConfidentialModule_Execute(t *testing.T) { t.Run("GetExecutable error", func(t *testing.T) { capReg := regmocks.NewCapabilitiesRegistry(t) capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). - Return(nil, errors.New("capability not found")).Once() + Return(nil, fmt.Errorf("capability not found")).Once() mod := NewConfidentialModule( - capReg, "", nil, "wf", "owner", "name", "tag", nil, nil, lggr, + capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, ) _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) @@ -246,10 +240,10 @@ func TestConfidentialModule_Execute(t *testing.T) { capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). Return(execCap, nil).Once() execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). - Return(capabilities.CapabilityResponse{}, errors.New("enclave unavailable")).Once() + Return(capabilities.CapabilityResponse{}, fmt.Errorf("enclave unavailable")).Once() mod := NewConfidentialModule( - capReg, "", nil, "wf", "owner", "name", "tag", nil, nil, lggr, + capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, ) _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) @@ -267,7 +261,7 @@ func TestConfidentialModule_Execute(t *testing.T) { Return(capabilities.CapabilityResponse{Payload: nil}, nil).Once() mod := NewConfidentialModule( - capReg, "", nil, "wf", "owner", "name", "tag", nil, nil, lggr, + capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, ) _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) @@ -302,7 +296,6 @@ func TestConfidentialModule_Execute(t *testing.T) { {Key: "K1", Namespace: "ns1"}, {Key: "K2"}, }, - nil, lggr, ) @@ -340,100 +333,6 @@ func TestConfidentialModule_Execute(t *testing.T) { }) } -func TestConfidentialModule_BinaryURLResolver(t *testing.T) { - ctx := context.Background() - lggr := logger.Nop() - - execReq := &sdkpb.ExecuteRequest{Config: []byte("cfg")} - - expectedResult := &sdkpb.ExecutionResult{ - Result: &sdkpb.ExecutionResult_Value{ - Value: valuespb.NewStringValue("ok"), - }, - } - resultBytes, err := proto.Marshal(expectedResult) - require.NoError(t, err) - confResp := &confworkflowtypes.ConfidentialWorkflowResponse{ExecutionResult: resultBytes} - respPayload, err := anypb.New(confResp) - require.NoError(t, err) - - t.Run("resolver replaces binary URL", func(t *testing.T) { - capReg := regmocks.NewCapabilitiesRegistry(t) - execCap := capmocks.NewExecutableCapability(t) - - capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). - Return(execCap, nil).Once() - - var capturedReq capabilities.CapabilityRequest - execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). - Run(func(_ context.Context, req capabilities.CapabilityRequest) { - capturedReq = req - }). - Return(capabilities.CapabilityResponse{Payload: respPayload}, nil).Once() - - resolver := func(_ context.Context, wfID string) (string, error) { - return "https://presigned.example.com/" + wfID + "?token=abc", nil - } - - mod := NewConfidentialModule( - capReg, "https://storage.example.com/raw", []byte("hash"), - "wf-1", "owner", "name", "tag", nil, resolver, lggr, - ) - - _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-1"}) - require.NoError(t, err) - - var confReq confworkflowtypes.ConfidentialWorkflowRequest - require.NoError(t, capturedReq.Payload.UnmarshalTo(&confReq)) - assert.Equal(t, "https://presigned.example.com/wf-1?token=abc", confReq.Execution.BinaryUrl) - }) - - t.Run("nil resolver uses raw URL", func(t *testing.T) { - capReg := regmocks.NewCapabilitiesRegistry(t) - execCap := capmocks.NewExecutableCapability(t) - - capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). - Return(execCap, nil).Once() - - var capturedReq capabilities.CapabilityRequest - execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). - Run(func(_ context.Context, req capabilities.CapabilityRequest) { - capturedReq = req - }). - Return(capabilities.CapabilityResponse{Payload: respPayload}, nil).Once() - - mod := NewConfidentialModule( - capReg, "https://storage.example.com/raw", []byte("hash"), - "wf-1", "owner", "name", "tag", nil, nil, lggr, - ) - - _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-1"}) - require.NoError(t, err) - - var confReq confworkflowtypes.ConfidentialWorkflowRequest - require.NoError(t, capturedReq.Payload.UnmarshalTo(&confReq)) - assert.Equal(t, "https://storage.example.com/raw", confReq.Execution.BinaryUrl) - }) - - t.Run("resolver error propagates", func(t *testing.T) { - capReg := regmocks.NewCapabilitiesRegistry(t) - - resolver := func(_ context.Context, _ string) (string, error) { - return "", errors.New("storage service unavailable") - } - - mod := NewConfidentialModule( - capReg, "https://storage.example.com/raw", []byte("hash"), - "wf-1", "owner", "name", "tag", nil, resolver, lggr, - ) - - _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-1"}) - require.Error(t, err) - assert.Contains(t, err.Error(), "failed to resolve binary URL") - assert.Contains(t, err.Error(), "storage service unavailable") - }) -} - func TestConfidentialModule_InterfaceMethods(t *testing.T) { mod := &ConfidentialModule{} diff --git a/deployment/cre/jobs/pkg/gateway_job.go b/deployment/cre/jobs/pkg/gateway_job.go index 3bc6828cac1..cca265a0bb6 100644 --- a/deployment/cre/jobs/pkg/gateway_job.go +++ b/deployment/cre/jobs/pkg/gateway_job.go @@ -457,7 +457,8 @@ func newDefaultHTTPCapabilitiesHandler() handler { } type confidentialRelayHandlerConfig struct { - RequestTimeoutSec int `toml:"requestTimeoutSec"` + RequestTimeoutSec int `toml:"requestTimeoutSec"` + NodeRateLimiter nodeRateLimiterConfig `toml:"NodeRateLimiter"` } func newDefaultConfidentialRelayHandler(requestTimeoutSec int) handler { @@ -466,6 +467,12 @@ func newDefaultConfidentialRelayHandler(requestTimeoutSec int) handler { ServiceName: ServiceNameConfidential, Config: confidentialRelayHandlerConfig{ RequestTimeoutSec: requestTimeoutSec, + NodeRateLimiter: nodeRateLimiterConfig{ + GlobalBurst: 10, + GlobalRPS: 50, + PerSenderBurst: 10, + PerSenderRPS: 10, + }, }, } } diff --git a/system-tests/lib/cre/features/confidential_relay/confidential_relay.go b/system-tests/lib/cre/features/confidential_relay/confidential_relay.go new file mode 100644 index 00000000000..4cf89d8227e --- /dev/null +++ b/system-tests/lib/cre/features/confidential_relay/confidential_relay.go @@ -0,0 +1,102 @@ +package confidentialrelay + +import ( + "context" + "fmt" + + tomlser "github.com/pelletier/go-toml/v2" + "github.com/pkg/errors" + "github.com/rs/zerolog" + + chainselectors "github.com/smartcontractkit/chain-selectors" + + "github.com/smartcontractkit/chainlink/deployment/cre/jobs/pkg" + "github.com/smartcontractkit/chainlink/system-tests/lib/cre" + coretoml "github.com/smartcontractkit/chainlink/v2/core/config/toml" + corechainlink "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" +) + +const flag = cre.ConfidentialRelayCapability + +type ConfidentialRelay struct{} + +func (o *ConfidentialRelay) Flag() cre.CapabilityFlag { + return flag +} + +func (o *ConfidentialRelay) PreEnvStartup( + ctx context.Context, + testLogger zerolog.Logger, + don *cre.DonMetadata, + topology *cre.Topology, + creEnv *cre.Environment, +) (*cre.PreEnvStartupOutput, error) { + registryChainID, chErr := chainselectors.ChainIdFromSelector(creEnv.RegistryChainSelector) + if chErr != nil { + return nil, errors.Wrapf(chErr, "failed to get chain ID from selector %d", creEnv.RegistryChainSelector) + } + + hErr := topology.AddGatewayHandlers(*don, []string{pkg.GatewayHandlerTypeConfidentialRelay}) + if hErr != nil { + return nil, errors.Wrapf(hErr, "failed to add gateway handlers to gateway config for don %s", don.Name) + } + + cErr := don.ConfigureForGatewayAccess(registryChainID, *topology.GatewayConnectors) + if cErr != nil { + return nil, errors.Wrapf(cErr, "failed to add gateway connectors to node's TOML config for don %s", don.Name) + } + + // Set TOML config to activate the confidential relay handler on DON nodes. + capConfig, ok := don.CapabilityConfigs[flag] + if ok && capConfig.Values != nil { + ns := don.MustNodeSet() + for i := range ns.NodeSpecs { + currentConfig := ns.NodeSpecs[i].Node.TestConfigOverrides + var typedConfig corechainlink.Config + if currentConfig != "" { + if err := tomlser.Unmarshal([]byte(currentConfig), &typedConfig); err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal node TOML config for node %d", i) + } + } + + enabled := true + relayConf := &coretoml.ConfidentialRelayConfig{Enabled: &enabled} + if v, exists := capConfig.Values["trustedPCRs"]; exists { + s, ok := v.(string) + if !ok { + return nil, fmt.Errorf("trustedPCRs must be a string, got %T", v) + } + relayConf.TrustedPCRs = &s + } + if v, exists := capConfig.Values["caRootsPEM"]; exists { + s, ok := v.(string) + if !ok { + return nil, fmt.Errorf("caRootsPEM must be a string, got %T", v) + } + relayConf.CARootsPEM = &s + } + typedConfig.CRE.ConfidentialRelay = relayConf + + out, err := tomlser.Marshal(typedConfig) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal node TOML config for node %d", i) + } + ns.NodeSpecs[i].Node.TestConfigOverrides = string(out) + } + } + + // No on-chain capability registration needed. The relay handler is a CRE subservice, + // not a registered capability. The mock capability that runs on the relay DON is + // registered separately via the mock flag. + return &cre.PreEnvStartupOutput{}, nil +} + +func (o *ConfidentialRelay) PostEnvStartup( + ctx context.Context, + testLogger zerolog.Logger, + don *cre.Don, + dons *cre.Dons, + creEnv *cre.Environment, +) error { + return nil +} diff --git a/system-tests/lib/cre/types.go b/system-tests/lib/cre/types.go index 6402d41e2a8..f1c0dec70eb 100644 --- a/system-tests/lib/cre/types.go +++ b/system-tests/lib/cre/types.go @@ -57,23 +57,24 @@ const ( // Capabilities const ( - ConsensusCapability CapabilityFlag = "ocr3" - DONTimeCapability CapabilityFlag = "don-time" - ConsensusCapabilityV2 CapabilityFlag = "consensus" // v2 - CronCapability CapabilityFlag = "cron" - EVMCapability CapabilityFlag = "evm" - CustomComputeCapability CapabilityFlag = "custom-compute" - WriteEVMCapability CapabilityFlag = "write-evm" - ReadContractCapability CapabilityFlag = "read-contract" - LogEventTriggerCapability CapabilityFlag = "log-event-trigger" - WebAPITargetCapability CapabilityFlag = "web-api-target" - WebAPITriggerCapability CapabilityFlag = "web-api-trigger" - MockCapability CapabilityFlag = "mock" - VaultCapability CapabilityFlag = "vault" - HTTPTriggerCapability CapabilityFlag = "http-trigger" - HTTPActionCapability CapabilityFlag = "http-action" - SolanaCapability CapabilityFlag = "solana" - AptosCapability CapabilityFlag = "aptos" + ConsensusCapability CapabilityFlag = "ocr3" + DONTimeCapability CapabilityFlag = "don-time" + ConsensusCapabilityV2 CapabilityFlag = "consensus" // v2 + CronCapability CapabilityFlag = "cron" + EVMCapability CapabilityFlag = "evm" + CustomComputeCapability CapabilityFlag = "custom-compute" + WriteEVMCapability CapabilityFlag = "write-evm" + ReadContractCapability CapabilityFlag = "read-contract" + LogEventTriggerCapability CapabilityFlag = "log-event-trigger" + WebAPITargetCapability CapabilityFlag = "web-api-target" + WebAPITriggerCapability CapabilityFlag = "web-api-trigger" + MockCapability CapabilityFlag = "mock" + VaultCapability CapabilityFlag = "vault" + HTTPTriggerCapability CapabilityFlag = "http-trigger" + HTTPActionCapability CapabilityFlag = "http-action" + SolanaCapability CapabilityFlag = "solana" + ConfidentialRelayCapability CapabilityFlag = "confidential-relay" + AptosCapability CapabilityFlag = "aptos" // Add more capabilities as needed ) diff --git a/system-tests/lib/cre/workflow/workflow.go b/system-tests/lib/cre/workflow/workflow.go index 2e7df738450..6560a9a4cfe 100644 --- a/system-tests/lib/cre/workflow/workflow.go +++ b/system-tests/lib/cre/workflow/workflow.go @@ -55,6 +55,7 @@ func RegisterWithContract( version *semver.Version, donID uint64, workflowName, binaryURL string, configURL, secretsURL *string, + attributes []byte, artifactsDirInContainer *string, ) (string, error) { // Download and decode workflow binary @@ -92,7 +93,7 @@ func RegisterWithContract( // Register workflow based on version switch version.Major() { case 2: - if err := registerWorkflowV2(sc, workflowRegistryAddr, version, workflowName, workflowID, binaryURLToUse, configURLToUse); err != nil { + if err := registerWorkflowV2(sc, workflowRegistryAddr, version, workflowName, workflowID, binaryURLToUse, configURLToUse, attributes); err != nil { return "", err } default: @@ -227,6 +228,7 @@ func registerWorkflowV2( workflowRegistryAddr common.Address, version *semver.Version, workflowName, workflowID, binaryURL, configURL string, + attributes []byte, ) error { registry, err := getRegistryV2Instance(sc, workflowRegistryAddr, version) if err != nil { @@ -251,7 +253,7 @@ func registerWorkflowV2( contracts.DonFamily, binaryURL, configURL, - nil, + attributes, false, )) if err != nil { diff --git a/system-tests/tests/test-helpers/t_helpers.go b/system-tests/tests/test-helpers/t_helpers.go index 830d071fb21..19fe5eba175 100644 --- a/system-tests/tests/test-helpers/t_helpers.go +++ b/system-tests/tests/test-helpers/t_helpers.go @@ -17,8 +17,6 @@ package helpers import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "math/big" @@ -28,7 +26,6 @@ import ( "slices" "strconv" "strings" - "sync" "testing" "time" @@ -77,9 +74,6 @@ import ( ) const WorkflowEngineInitErrorLog = "Workflow Engine initialization failed" -const maxWorkflowNameLen = 64 - -var deleteWorkflowsMu sync.Mutex ///////////////////////// // ENVIRONMENT HELPERS // @@ -330,12 +324,13 @@ type WorkflowRegistrationConfig struct { ConfigFilePath string CompressedWasmPath string SecretsURL string + Attributes []byte WorkflowRegistryAddr common.Address WorkflowRegistryVersion *semver.Version ChainID uint64 DonID uint64 ContainerTargetDir string - SethClient *seth.Client + Blockchains []blockchains.Blockchain } /* @@ -348,11 +343,11 @@ It returns the paths to: 1. the compressed WASM file; 2. the workflow config file. */ -func createWorkflowArtifacts[T WorkflowConfig](t *testing.T, testLogger zerolog.Logger, workflowName string, workflowDONs []*cre.Don, workflowConfig *T, workflowFileLocation, artifactDir string) (string, string) { +func createWorkflowArtifacts[T WorkflowConfig](t *testing.T, testLogger zerolog.Logger, workflowName string, workflowDONs []*cre.Don, workflowConfig *T, workflowFileLocation string) (string, string) { t.Helper() - workflowConfigFilePath := workflowConfigFactory(t, testLogger, workflowName, workflowConfig, artifactDir) - compressedWorkflowWasmPath, compileErr := creworkflow.CompileWorkflowToDir(t.Context(), workflowFileLocation, workflowName, artifactDir) + workflowConfigFilePath := workflowConfigFactory(t, testLogger, workflowName, workflowConfig) + compressedWorkflowWasmPath, compileErr := creworkflow.CompileWorkflow(t.Context(), workflowFileLocation, workflowName) require.NoError(t, compileErr, "failed to compile workflow '%s'", workflowFileLocation) testLogger.Info().Msg("Workflow compiled successfully.") @@ -373,7 +368,7 @@ Pass `nil` to skip workflow config file creation. Returns the path to the workflow config file. */ -func workflowConfigFactory[T WorkflowConfig](t *testing.T, testLogger zerolog.Logger, workflowName string, workflowConfig *T, outputDir string) (filePath string) { +func workflowConfigFactory[T WorkflowConfig](t *testing.T, testLogger zerolog.Logger, workflowName string, workflowConfig *T) (filePath string) { t.Helper() var workflowConfigFilePath string @@ -386,7 +381,7 @@ func workflowConfigFactory[T WorkflowConfig](t *testing.T, testLogger zerolog.Lo testLogger.Info().Msg("Workflow config file is not requested and will not be created.") case *portypes.WorkflowConfig: - workflowCfgFilePath, configErr := createPoRWorkflowConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := createPoRWorkflowConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create PoR workflow config file") testLogger.Info().Msg("PoR workflow config file created.") @@ -401,7 +396,7 @@ func workflowConfigFactory[T WorkflowConfig](t *testing.T, testLogger zerolog.Lo cleanID = cleanID[:32] } cfg.FeedID = "0x" + cleanID - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create PoR v2 workflow config file") testLogger.Info().Msg("PoR v2 workflow config file created.") @@ -425,72 +420,72 @@ func workflowConfigFactory[T WorkflowConfig](t *testing.T, testLogger zerolog.Lo testLogger.Info().Msg("Aptos write roundtrip workflow config file created.") case *HTTPWorkflowConfig: - workflowCfgFilePath, configErr := createHTTPWorkflowConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := createHTTPWorkflowConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create HTTP workflow config file") testLogger.Info().Msg("HTTP workflow config file created.") case *crontypes.WorkflowConfig: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create Cron workflow config file") testLogger.Info().Msg("Cron workflow config file created.") case *consensus_negative_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create consensus workflow config file") testLogger.Info().Msg("Consensus workflow config file created.") case *evmread_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create evmread workflow config file") testLogger.Info().Msg("EVM Read workflow config file created.") case *logtrigger_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create logtrigger workflow config file") testLogger.Info().Msg("EVM LogTrigger workflow config file created.") case *evmread_negative_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create evmread-negative workflow config file") testLogger.Info().Msg("EVM Read negative workflow config file created.") case *evmwrite_negative_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create evmwrite-negative workflow config file") testLogger.Info().Msg("EVM Write negative workflow config file created.") case *logtrigger_negative_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create logtrigger-negative workflow config file") testLogger.Info().Msg("EVM LogTrigger negative workflow config file created.") case *http_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create http-negative workflow config file") testLogger.Info().Msg("HTTP negative workflow config file created.") case *httpaction_smoke_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create httpaction smoke workflow config file") testLogger.Info().Msg("HTTP Action smoke workflow config file created.") case *httpaction_negative_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create httpaction negative workflow config file") testLogger.Info().Msg("HTTP Action negative workflow config file created.") case *solwrite_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create solwrite workflow config file") testLogger.Info().Msg("Solana write workflow config file created.") @@ -515,14 +510,14 @@ It stores the values used by a workflow (main.go), The values are written to types.WorkflowConfig. The method returns the absolute path to the created config file. */ -func createPoRWorkflowConfigFile(workflowName string, workflowConfig *portypes.WorkflowConfig, outputDir string) (string, error) { +func createPoRWorkflowConfigFile(workflowName string, workflowConfig *portypes.WorkflowConfig) (string, error) { feedIDToUse, fIDerr := validateAndFormatFeedID(workflowConfig) if fIDerr != nil { return "", errors.Wrap(fIDerr, "failed to validate and format feed ID") } workflowConfig.FeedID = feedIDToUse - return CreateWorkflowYamlConfigFile(workflowName, workflowConfig, outputDir) + return CreateWorkflowYamlConfigFile(workflowName, workflowConfig) } func validateAndFormatFeedID(workflowConfig *portypes.WorkflowConfig) (string, error) { @@ -544,7 +539,7 @@ func validateAndFormatFeedID(workflowConfig *portypes.WorkflowConfig) (string, e return feedIDToUse, nil } -func createHTTPWorkflowConfigFile(workflowName string, cfg *HTTPWorkflowConfig, outputDir string) (string, error) { +func createHTTPWorkflowConfigFile(workflowName string, cfg *HTTPWorkflowConfig) (string, error) { testLogger := framework.L mockServerURL := cfg.URL parsedURL, urlErr := url.Parse(mockServerURL) @@ -564,7 +559,7 @@ func createHTTPWorkflowConfigFile(workflowName string, cfg *HTTPWorkflowConfig, } configFileName := fmt.Sprintf("test_http_workflow_config_%s.json", workflowName) - configPath := filepath.Join(outputDir, configFileName) + configPath := filepath.Join(os.TempDir(), configFileName) writeErr := os.WriteFile(configPath, configBytes, 0o644) //nolint:gosec // this is a test file if writeErr != nil { @@ -577,23 +572,21 @@ func createHTTPWorkflowConfigFile(workflowName string, cfg *HTTPWorkflowConfig, /* Creates .yaml workflow configuration file and returns the absolute path to the created config file. */ -func CreateWorkflowYamlConfigFile(workflowName string, workflowConfig any, outputDir string) (string, error) { +func CreateWorkflowYamlConfigFile(workflowName string, workflowConfig any) (string, error) { // Write workflow config to a .yaml file configMarshalled, err := yaml.Marshal(workflowConfig) if err != nil { return "", errors.Wrap(err, "failed to marshal workflow config") } - if mkErr := os.MkdirAll(outputDir, 0o755); mkErr != nil { - return "", errors.Wrap(mkErr, "failed to create output directory") - } + workflowSuffix := "_config.yaml" + workflowConfigOutputFile := workflowName + workflowSuffix - workflowConfigFile, tempErr := os.CreateTemp(outputDir, workflowName+"-*_config.yaml") - if tempErr != nil { - return "", errors.Wrap(tempErr, "failed to create workflow config file") - } - workflowConfigOutputFile := workflowConfigFile.Name() - if closeErr := workflowConfigFile.Close(); closeErr != nil { - return "", errors.Wrap(closeErr, "failed to close workflow config file") + // remove the duplicate if it already exists + _, statErr := os.Stat(workflowConfigOutputFile) + if statErr == nil { + if err := os.Remove(workflowConfigOutputFile); err != nil { + return "", errors.Wrap(err, "failed to remove existing output file") + } } if err := os.WriteFile(workflowConfigOutputFile, configMarshalled, 0o644); err != nil { //nolint:gosec // G306: we want it to be readable by everyone @@ -619,8 +612,8 @@ func registerWorkflow(ctx context.Context, t *testing.T, t.Cleanup(func() { deleteWorkflows(t, wfRegCfg.WorkflowName, wfRegCfg.ConfigFilePath, - wfRegCfg.CompressedWasmPath, - wfRegCfg.WorkflowRegistryAddr, wfRegCfg.WorkflowRegistryVersion, wfRegCfg.SethClient, + wfRegCfg.CompressedWasmPath, wfRegCfg.Blockchains, + wfRegCfg.WorkflowRegistryAddr, wfRegCfg.WorkflowRegistryVersion, ) }) @@ -644,6 +637,7 @@ func registerWorkflow(ctx context.Context, t *testing.T, binaryURL, configURL, nil, // no secrets yet + wfRegCfg.Attributes, containerTargetDir, ) require.NoError(t, registerErr, "failed to register workflow '%s'", wfRegCfg.WorkflowName) @@ -664,9 +658,9 @@ func deleteWorkflows( uniqueWorkflowName string, workflowConfigFilePath string, compressedWorkflowWasmPath string, + blockchains []blockchains.Blockchain, workflowRegistryAddress common.Address, version *semver.Version, - sethClient *seth.Client, ) { t.Helper() @@ -675,10 +669,8 @@ func deleteWorkflows( localEnvErr := creworkflow.RemoveWorkflowArtifactsFromLocalEnv(workflowConfigFilePath, compressedWorkflowWasmPath) require.NoError(t, localEnvErr, "failed to remove workflow artifacts from local environment") - deleteWorkflowsMu.Lock() - defer deleteWorkflowsMu.Unlock() - - deleteErr := creworkflow.DeleteWithContract(t.Context(), sethClient, workflowRegistryAddress, version, uniqueWorkflowName) + require.IsType(t, &evm.Blockchain{}, blockchains[0], "expected EVM blockchain type") + deleteErr := creworkflow.DeleteWithContract(t.Context(), blockchains[0].(*evm.Blockchain).SethClient, workflowRegistryAddress, version, uniqueWorkflowName) require.NoError(t, deleteErr, "failed to delete workflow '%s'. Please delete/unregister it manually.", uniqueWorkflowName) testLogger.Info().Msgf("Workflow '%s' deleted successfully from the registry.", uniqueWorkflowName) } @@ -686,26 +678,24 @@ func deleteWorkflows( func CompileAndDeployWorkflow[T WorkflowConfig](t *testing.T, testEnv *ttypes.TestEnvironment, testLogger zerolog.Logger, workflowName string, workflowConfig *T, workflowFileLocation string, - opts ...CompileAndDeployWorkflowOpt, ) string { t.Helper() - cfg := compileAndDeployWorkflowCfg{ - artifactCopyDONTypes: []cre.CapabilityFlag{cre.WorkflowDON}, - } - for _, opt := range opts { - opt(&cfg) - } testLogger.Info(). Str("workflow_name", workflowName). Str("workflow_file_location", workflowFileLocation). Msgf("compiling and registering workflow '%s'", workflowName) - artifactDir := workflowArtifactsDir(t, testEnv) registryChainSelector := testEnv.CreEnvironment.Blockchains[0].ChainSelector() - workflowDONs := selectArtifactTargetDONs(testEnv, cfg.artifactCopyDONTypes) + workflowDONs := make([]*cre.Don, 0) + for _, don := range testEnv.Dons.List() { + if !don.HasFlag(cre.WorkflowDON) { + continue + } + workflowDONs = append(workflowDONs, don) + } - compressedWorkflowWasmPath, workflowConfigPath := createWorkflowArtifacts(t, testLogger, workflowName, workflowDONs, workflowConfig, workflowFileLocation, artifactDir) + compressedWorkflowWasmPath, workflowConfigPath := createWorkflowArtifacts(t, testLogger, workflowName, workflowDONs, workflowConfig, workflowFileLocation) require.NotEmpty(t, compressedWorkflowWasmPath, "failed to find workflow DON in the topology") workflowRegistryAddress := crecontracts.MustGetAddressRefFromDataStore(testEnv.CreEnvironment.CldfEnvironment.DataStore, testEnv.CreEnvironment.Blockchains[0].ChainSelector(), keystone_changeset.WorkflowRegistry.String(), testEnv.CreEnvironment.ContractVersions[keystone_changeset.WorkflowRegistry.String()], "") @@ -720,93 +710,54 @@ func CompileAndDeployWorkflow[T WorkflowConfig](t *testing.T, ChainID: registryChainSelector, DonID: testEnv.Dons.MustWorkflowDON().ID, ContainerTargetDir: creworkflow.DefaultWorkflowTargetDir, - SethClient: testEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient, + Blockchains: testEnv.CreEnvironment.Blockchains, } require.IsType(t, &evm.Blockchain{}, testEnv.CreEnvironment.Blockchains[0], "expected EVM blockchain type") workflowID := registerWorkflow(t.Context(), t, workflowRegConfig, testEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient, testLogger) return workflowID } -type compileAndDeployWorkflowCfg struct { - artifactCopyDONTypes []cre.CapabilityFlag -} - -// CompileAndDeployWorkflowOpt customizes workflow compilation/deployment behavior. -type CompileAndDeployWorkflowOpt func(*compileAndDeployWorkflowCfg) - -// WithArtifactCopyDONTypes sets DON types where workflow artifacts should be copied. -func WithArtifactCopyDONTypes(donTypes ...cre.CapabilityFlag) CompileAndDeployWorkflowOpt { - return func(cfg *compileAndDeployWorkflowCfg) { - if len(donTypes) == 0 { - return - } - cfg.artifactCopyDONTypes = append([]cre.CapabilityFlag{}, donTypes...) - } -} +// CompileAndDeployConfidentialWorkflow compiles a workflow WASM binary, copies it to Docker +// containers, and registers it with confidential attributes on the workflow registry. +func CompileAndDeployConfidentialWorkflow[T WorkflowConfig](t *testing.T, + testEnv *ttypes.TestEnvironment, testLogger zerolog.Logger, workflowName string, + workflowConfig *T, workflowFileLocation string, attributes []byte, +) string { + t.Helper() -func selectArtifactTargetDONs(testEnv *ttypes.TestEnvironment, donTypes []cre.CapabilityFlag) []*cre.Don { - if len(donTypes) == 0 { - donTypes = []cre.CapabilityFlag{cre.WorkflowDON} - } - allow := make(map[cre.CapabilityFlag]struct{}, len(donTypes)) - for _, donType := range donTypes { - allow[donType] = struct{}{} - } + testLogger.Info(). + Str("workflow_name", workflowName). + Str("workflow_file_location", workflowFileLocation). + Msgf("compiling and registering confidential workflow '%s'", workflowName) + registryChainSelector := testEnv.CreEnvironment.Blockchains[0].ChainSelector() - targetDONs := make([]*cre.Don, 0) + workflowDONs := make([]*cre.Don, 0) for _, don := range testEnv.Dons.List() { - for donType := range allow { - if don.HasFlag(donType) { - targetDONs = append(targetDONs, don) - break - } + if !don.HasFlag(cre.WorkflowDON) { + continue } - } - return targetDONs -} - -func workflowArtifactsDir(t *testing.T, testEnv *ttypes.TestEnvironment) string { - t.Helper() - if testEnv.Execution == nil || testEnv.Execution.TestID == "" { - dir, err := os.MkdirTemp("", "cre-workflow-artifacts-*") - require.NoError(t, err, "failed to create artifacts directory") - return dir + workflowDONs = append(workflowDONs, don) } - dir := filepath.Join(os.TempDir(), "cre-workflow-artifacts", testEnv.Execution.TestID) - require.NoError(t, os.MkdirAll(dir, 0o755), "failed to create artifacts directory") - return dir -} - -func UniqueWorkflowName(testEnv *ttypes.TestEnvironment, baseName string) string { - testID := "" - if testEnv != nil && testEnv.Execution != nil { - testID = testEnv.Execution.TestID - } - if testID == "" { - return truncateWorkflowName(baseName, baseName) - } - return truncateWorkflowName(fmt.Sprintf("%s-%s", baseName, testID), fmt.Sprintf("%s:%s", baseName, testID)) -} + compressedWorkflowWasmPath, workflowConfigPath := createWorkflowArtifacts(t, testLogger, workflowName, workflowDONs, workflowConfig, workflowFileLocation) + require.NotEmpty(t, compressedWorkflowWasmPath, "failed to find workflow DON in the topology") -func truncateWorkflowName(name, uniquenessSeed string) string { - if len(name) <= maxWorkflowNameLen { - return name - } + workflowRegistryAddress := crecontracts.MustGetAddressRefFromDataStore(testEnv.CreEnvironment.CldfEnvironment.DataStore, testEnv.CreEnvironment.Blockchains[0].ChainSelector(), keystone_changeset.WorkflowRegistry.String(), testEnv.CreEnvironment.ContractVersions[keystone_changeset.WorkflowRegistry.String()], "") - sum := sha256.Sum256([]byte(uniquenessSeed)) - suffix := hex.EncodeToString(sum[:])[:8] - prefixLen := maxWorkflowNameLen - len(suffix) - 1 // include hyphen - if prefixLen < 1 { - return suffix[:maxWorkflowNameLen] - } - if prefixLen > len(name) { - prefixLen = len(name) + workflowRegConfig := &WorkflowRegistrationConfig{ + WorkflowName: workflowName, + WorkflowLocation: workflowFileLocation, + ConfigFilePath: workflowConfigPath, + CompressedWasmPath: compressedWorkflowWasmPath, + Attributes: attributes, + WorkflowRegistryAddr: common.HexToAddress(workflowRegistryAddress.Address), + WorkflowRegistryVersion: workflowRegistryAddress.Version, + ChainID: registryChainSelector, + DonID: testEnv.Dons.MustWorkflowDON().ID, + ContainerTargetDir: creworkflow.DefaultWorkflowTargetDir, + Blockchains: testEnv.CreEnvironment.Blockchains, } - return fmt.Sprintf("%s-%s", name[:prefixLen], suffix) -} - -func ParallelEnabled() bool { - v := strings.TrimSpace(strings.ToLower(os.Getenv("CRE_TEST_PARALLEL_ENABLED"))) - return v == "1" || v == "true" || v == "yes" + require.IsType(t, &evm.Blockchain{}, testEnv.CreEnvironment.Blockchains[0], "expected EVM blockchain type") + workflowID := registerWorkflow(t.Context(), t, workflowRegConfig, testEnv.CreEnvironment.Blockchains[0].(*evm.Blockchain).SethClient, testLogger) + return workflowID } From 680065929514d7456cd5923cbc4776a73147f28f Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 26 Mar 2026 12:45:15 +0100 Subject: [PATCH 26/67] Remove launcher fix (relay DON will be configured as workflow DON instead) --- core/capabilities/launcher.go | 7 --- core/capabilities/launcher_test.go | 75 ------------------------------ 2 files changed, 82 deletions(-) diff --git a/core/capabilities/launcher.go b/core/capabilities/launcher.go index 427bbc64671..32bda8c9e08 100644 --- a/core/capabilities/launcher.go +++ b/core/capabilities/launcher.go @@ -410,13 +410,6 @@ func (w *launcher) OnNewRegistry(ctx context.Context, localRegistry *registrysyn if belongsToACapabilityDON { for _, myDON := range myCapabilityDONs { w.serveCapabilities(ctx, w.myPeerID, myDON, localRegistry, remoteWorkflowDONs) - - // Capability DONs may need to call capabilities on other DONs - // (e.g. relay DON calling vault DON for secret fetching). - // Without this, only workflow DONs discover cross-DON capabilities. - for _, rcd := range remoteCapabilityDONs { - w.addRemoteCapabilities(ctx, myDON, rcd, localRegistry) - } } } diff --git a/core/capabilities/launcher_test.go b/core/capabilities/launcher_test.go index 001fd29b3a3..29ec22e2fe3 100644 --- a/core/capabilities/launcher_test.go +++ b/core/capabilities/launcher_test.go @@ -2,8 +2,6 @@ package capabilities import ( "context" - "fmt" - "strings" "testing" "time" @@ -236,79 +234,6 @@ func TestLauncher(t *testing.T) { assert.Equal(t, 1, observedLogs.FilterMessage("failed to serve capability").Len()) }) - t.Run("OK-capability_DON_discovers_remote_capabilities", func(t *testing.T) { - // Regression test: a capability DON (e.g. relay DON) that needs to call - // another capability DON (e.g. vault DON) must discover it via - // addRemoteCapabilities. Previously, only workflow DONs discovered - // remote capabilities, so the relay DON couldn't find vault. - lggr, observedLogs := logger.TestObserved(t, zapcore.DebugLevel) - registry := NewRegistry(lggr) - dispatcher := remoteMocks.NewDispatcher(t) - - nodes := newNodes(4) - remoteNodes := newNodes(4) - peer := mocks.NewPeer(t) - peer.On("UpdateConnections", mock.Anything).Return(nil) - peer.On("ID").Return(nodes[0]) - peer.On("IsBootstrap").Return(false) - wrapper := mocks.NewPeerWrapper(t) - wrapper.On("GetPeer").Return(peer) - - // Local DON: capability DON (relay) with a trigger capability - fullTriggerCapID := "streams-trigger@1.0.0" - mt := newMockTrigger(capabilities.MustNewCapabilityInfo( - fullTriggerCapID, - capabilities.CapabilityTypeTrigger, - "streams trigger", - )) - require.NoError(t, registry.Add(t.Context(), mt)) - - triggerCapID := RandomUTF8BytesWord() - - localDONID := uint32(1) - remoteDONID := uint32(2) - localRegistry := buildLocalRegistry() - - // Local DON: capability DON (isPublic=true, acceptsWorkflows=false) - addDON(localRegistry, localDONID, uint32(0), uint8(1), true, false, nodes, []string{"zone-a"}, 1, [][32]byte{triggerCapID}) - addCapabilityToDON(localRegistry, localDONID, fullTriggerCapID, capabilities.CapabilityTypeTrigger, nil) - - // Remote DON: another capability DON (vault) that the local DON needs to call - remoteTargetCapID := RandomUTF8BytesWord() - fullRemoteTargetID := "vault@1.0.0" - addDON(localRegistry, remoteDONID, uint32(0), uint8(1), true, false, remoteNodes, []string{"zone-a"}, 1, [][32]byte{remoteTargetCapID}) - addCapabilityToDON(localRegistry, remoteDONID, fullRemoteTargetID, capabilities.CapabilityTypeTarget, nil) - - launcher, err := NewLauncher( - lggr, - wrapper, - nil, - nil, - dispatcher, - registry, - &mockDonNotifier{}, - ) - require.NoError(t, err) - require.NoError(t, launcher.Start(t.Context())) - defer launcher.Close() - - dispatcher.On("SetReceiver", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(nil) - - require.NoError(t, launcher.OnNewRegistry(t.Context(), localRegistry)) - // Without the fix, addRemoteCapabilities is never called for capability DONs. - // With the fix, the launcher attempts to add vault@1.0.0 as a remote capability. - // It may fail downstream (signer setup) but the attempt proves discovery works. - allLogs := observedLogs.All() - found := false - for _, entry := range allLogs { - if strings.Contains(entry.Message, "remote capability") && strings.Contains(fmt.Sprintf("%v", entry.ContextMap()), fullRemoteTargetID) { - found = true - break - } - } - assert.True(t, found, "expected capability DON to attempt remote capability discovery for %s", fullRemoteTargetID) - }) - t.Run("start and close with nil peer wrapper", func(t *testing.T) { lggr := logger.Test(t) registry := NewRegistry(lggr) From 1b770065166fc4973084b726d619dbec6f7f9ba1 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 26 Mar 2026 15:38:04 +0100 Subject: [PATCH 27/67] Sync with latest PR 1/2/4 branches for E2E --- .../handlers/confidentialrelay/handler.go | 28 +++++++++++-------- .../workflows/v2/confidential_module.go | 3 +- .../workflows/v2/confidential_module_test.go | 8 +++--- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/core/services/gateway/handlers/confidentialrelay/handler.go b/core/services/gateway/handlers/confidentialrelay/handler.go index 0f746fa456d..1b564949783 100644 --- a/core/services/gateway/handlers/confidentialrelay/handler.go +++ b/core/services/gateway/handlers/confidentialrelay/handler.go @@ -222,8 +222,7 @@ func (h *handler) removeExpiredRequests(ctx context.Context) { for _, er := range expiredRequests { responses := er.copiedResponses() h.lggr.Debugw("request expired without quorum", "requestID", er.req.ID, "responseCount", len(responses), "required", h.donConfig.F+1) - // sendResponse deletes the request from activeRequests after sending. - err := h.sendResponse(ctx, er, h.errorResponse(er.req, api.RequestTimeoutError, fmt.Errorf("request expired: got %d/%d responses", len(responses), h.donConfig.F+1), nil)) + err := h.sendResponseAndCleanup(ctx, er, h.errorResponse(er.req, api.RequestTimeoutError, fmt.Errorf("request expired: got %d/%d responses", len(responses), h.donConfig.F+1), nil)) if err != nil { h.lggr.Errorw("error sending response to user", "requestID", er.req.ID, "error", err) } @@ -309,10 +308,10 @@ func (h *handler) HandleNodeMessage(ctx context.Context, resp *jsonrpc.Response[ return nil case err != nil: l.Error("quorum unobtainable, returning response to user...", "error", err, "responses", maps.Values(ar.responses)) - return h.sendResponse(ctx, ar, h.errorResponse(ar.req, api.FatalError, err, nil)) + return h.sendResponseAndCleanup(ctx, ar, h.errorResponse(ar.req, api.FatalError, err, nil)) } - return h.sendSuccessResponse(ctx, l, ar, aggregatedResp) + return h.sendSuccessResponseAndCleanup(ctx, l, ar, aggregatedResp) } func (h *handler) fanOutToNodes(ctx context.Context, l logger.Logger, ar *activeRequest) error { @@ -326,18 +325,18 @@ func (h *handler) fanOutToNodes(ctx context.Context, l logger.Logger, ar *active } if len(nodeErrors) == len(h.donConfig.Members) && len(nodeErrors) > 0 { - return h.sendResponse(ctx, ar, h.errorResponse(ar.req, api.FatalError, errors.New("failed to forward user request to nodes"), nil)) + return h.sendResponseAndCleanup(ctx, ar, h.errorResponse(ar.req, api.FatalError, errors.New("failed to forward user request to nodes"), nil)) } l.Debugw("successfully forwarded request to relay nodes") return nil } -func (h *handler) sendSuccessResponse(ctx context.Context, l logger.Logger, ar *activeRequest, resp *jsonrpc.Response[json.RawMessage]) error { +func (h *handler) sendSuccessResponseAndCleanup(ctx context.Context, l logger.Logger, ar *activeRequest, resp *jsonrpc.Response[json.RawMessage]) error { rawResponse, err := jsonrpc.EncodeResponse(resp) if err != nil { l.Errorw("failed to encode response", "error", err) - return h.sendResponse(ctx, ar, h.errorResponse(ar.req, api.NodeReponseEncodingError, fmt.Errorf("failed to marshal response: %w", err), nil)) + return h.sendResponseAndCleanup(ctx, ar, h.errorResponse(ar.req, api.NodeReponseEncodingError, fmt.Errorf("failed to marshal response: %w", err), nil)) } var errorCode api.ErrorCode @@ -352,7 +351,7 @@ func (h *handler) sendSuccessResponse(ctx context.Context, l logger.Logger, ar * RawResponse: rawResponse, ErrorCode: errorCode, } - return h.sendResponse(ctx, ar, successResp) + return h.sendResponseAndCleanup(ctx, ar, successResp) } func (h *handler) errorResponse( @@ -395,7 +394,10 @@ func (h *handler) errorResponse( } } -func (h *handler) sendResponse(ctx context.Context, userRequest *activeRequest, resp gwhandlers.UserCallbackPayload) error { +// sendResponseAndCleanup sends the response to the user and removes the +// request from activeRequests. The request is always removed regardless of +// whether the send succeeds, since a failed callback cannot be retried. +func (h *handler) sendResponseAndCleanup(ctx context.Context, userRequest *activeRequest, resp gwhandlers.UserCallbackPayload) error { switch resp.ErrorCode { case api.StaleNodeResponseError: case api.FatalError: @@ -422,14 +424,16 @@ func (h *handler) sendResponse(ctx context.Context, userRequest *activeRequest, } err := userRequest.SendResponse(resp) + + h.mu.Lock() + delete(h.activeRequests, userRequest.req.ID) + h.mu.Unlock() + if err != nil { h.lggr.Errorw("error sending response to user", "requestID", userRequest.req.ID, "error", err) return err } - h.mu.Lock() - defer h.mu.Unlock() - delete(h.activeRequests, userRequest.req.ID) h.lggr.Debugw("response sent to user", "requestID", userRequest.req.ID, "errorCode", resp.ErrorCode) return nil } diff --git a/core/services/workflows/v2/confidential_module.go b/core/services/workflows/v2/confidential_module.go index 4cc3761e18a..16bbb83052c 100644 --- a/core/services/workflows/v2/confidential_module.go +++ b/core/services/workflows/v2/confidential_module.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "google.golang.org/protobuf/proto" @@ -165,7 +166,7 @@ func (m *ConfidentialModule) Execute( } if capResp.Payload == nil { - return nil, fmt.Errorf("confidential-workflows capability returned nil payload") + return nil, errors.New("confidential-workflows capability returned nil payload") } var confResp confworkflowtypes.ConfidentialWorkflowResponse diff --git a/core/services/workflows/v2/confidential_module_test.go b/core/services/workflows/v2/confidential_module_test.go index 87dab426022..598f82c0119 100644 --- a/core/services/workflows/v2/confidential_module_test.go +++ b/core/services/workflows/v2/confidential_module_test.go @@ -3,7 +3,7 @@ package v2 import ( "context" "crypto/sha256" - "fmt" + "errors" "testing" "time" @@ -52,7 +52,7 @@ func TestParseWorkflowAttributes(t *testing.T) { assert.True(t, attrs.Confidential) require.Len(t, attrs.VaultDonSecrets, 2) assert.Equal(t, "API_KEY", attrs.VaultDonSecrets[0].Key) - assert.Equal(t, "", attrs.VaultDonSecrets[0].Namespace) + assert.Empty(t, attrs.VaultDonSecrets[0].Namespace) assert.Equal(t, "SIGNING_KEY", attrs.VaultDonSecrets[1].Key) assert.Equal(t, "custom-ns", attrs.VaultDonSecrets[1].Namespace) }) @@ -222,7 +222,7 @@ func TestConfidentialModule_Execute(t *testing.T) { t.Run("GetExecutable error", func(t *testing.T) { capReg := regmocks.NewCapabilitiesRegistry(t) capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). - Return(nil, fmt.Errorf("capability not found")).Once() + Return(nil, errors.New("capability not found")).Once() mod := NewConfidentialModule( capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, @@ -240,7 +240,7 @@ func TestConfidentialModule_Execute(t *testing.T) { capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). Return(execCap, nil).Once() execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). - Return(capabilities.CapabilityResponse{}, fmt.Errorf("enclave unavailable")).Once() + Return(capabilities.CapabilityResponse{}, errors.New("enclave unavailable")).Once() mod := NewConfidentialModule( capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, From 4943e46dc7d8147fa1cd12b34c71599eaa7ff58f Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 26 Mar 2026 15:49:58 +0100 Subject: [PATCH 28/67] Simplify relay feature plugin: only set Enabled, drop TrustedPCRs/CARootsPEM --- .../confidential_relay/confidential_relay.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/system-tests/lib/cre/features/confidential_relay/confidential_relay.go b/system-tests/lib/cre/features/confidential_relay/confidential_relay.go index 4cf89d8227e..f6f72802ac3 100644 --- a/system-tests/lib/cre/features/confidential_relay/confidential_relay.go +++ b/system-tests/lib/cre/features/confidential_relay/confidential_relay.go @@ -2,7 +2,6 @@ package confidentialrelay import ( "context" - "fmt" tomlser "github.com/pelletier/go-toml/v2" "github.com/pkg/errors" @@ -60,22 +59,7 @@ func (o *ConfidentialRelay) PreEnvStartup( } enabled := true - relayConf := &coretoml.ConfidentialRelayConfig{Enabled: &enabled} - if v, exists := capConfig.Values["trustedPCRs"]; exists { - s, ok := v.(string) - if !ok { - return nil, fmt.Errorf("trustedPCRs must be a string, got %T", v) - } - relayConf.TrustedPCRs = &s - } - if v, exists := capConfig.Values["caRootsPEM"]; exists { - s, ok := v.(string) - if !ok { - return nil, fmt.Errorf("caRootsPEM must be a string, got %T", v) - } - relayConf.CARootsPEM = &s - } - typedConfig.CRE.ConfidentialRelay = relayConf + typedConfig.CRE.ConfidentialRelay = &coretoml.ConfidentialRelayConfig{Enabled: &enabled} out, err := tomlser.Marshal(typedConfig) if err != nil { From 9be708c543b5f854d695bdc09fc762bf46044eea Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 26 Mar 2026 16:43:53 +0100 Subject: [PATCH 29/67] Update CC plugin refs to b7f4350 --- plugins/plugins.private.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/plugins.private.yaml b/plugins/plugins.private.yaml index 01c184d0132..398abfe2bde 100644 --- a/plugins/plugins.private.yaml +++ b/plugins/plugins.private.yaml @@ -56,5 +56,10 @@ plugins: installPath: "." confidential-http: - moduleURI: "github.com/smartcontractkit/confidential-compute/enclave/apps/confidential-http/capability" - gitRef: "187018af88ce265ca70b86222f8587e3fcb7ff32" + gitRef: "b7f43502f174425bfaee79157dc068bd7aadd9b9" installPath: "./cmd/confidential-http" + confidential-workflows: + - moduleURI: "github.com/smartcontractkit/confidential-compute/enclave/apps/confidential-workflows/capability" + gitRef: "b7f43502f174425bfaee79157dc068bd7aadd9b9" + installPath: "./cmd/confidential-workflows" +>>>>>>> fb2ffec52b (Update CC plugin refs to b7f4350) From f19751925d5f32adedfc4c72c0475398bafc0a0f Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 26 Mar 2026 17:23:15 +0100 Subject: [PATCH 30/67] Fix mock capability on-chain type: ACTION not TRIGGER --- system-tests/lib/cre/features/mock/mock.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system-tests/lib/cre/features/mock/mock.go b/system-tests/lib/cre/features/mock/mock.go index 4166243f7ff..a504a04d220 100644 --- a/system-tests/lib/cre/features/mock/mock.go +++ b/system-tests/lib/cre/features/mock/mock.go @@ -43,7 +43,7 @@ func (o *Mock) PreEnvStartup( Capability: kcr.CapabilitiesRegistryCapability{ LabelledName: "mock", Version: "1.0.0", - CapabilityType: 0, // TRIGGER + CapabilityType: 1, // ACTION }, Config: &capabilitiespb.CapabilityConfig{ LocalOnly: don.HasOnlyLocalCapabilities(), From 8f38b3cab1c39336dc53b2d041bb33be3faa6943 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 26 Mar 2026 22:02:31 +0100 Subject: [PATCH 31/67] Make BinaryURLResolver injectable via WithBinaryURLResolver option --- core/services/workflows/syncer/v2/handler.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 2bda9999383..25eb9bbec09 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -78,6 +78,7 @@ type eventHandler struct { billingClient metering.BillingClient orgResolver orgresolver.OrgResolver secretsFetcher v2.SecretsFetcher + binaryURLResolver v2.BinaryURLResolver // WorkflowRegistryAddress is the address of the workflow registry contract workflowRegistryAddress string @@ -179,6 +180,15 @@ func WithSecretsFetcher(sf v2.SecretsFetcher) func(*eventHandler) { } } +// WithBinaryURLResolver sets a resolver that transforms raw binary URLs +// into URLs the enclave can fetch. In production this produces presigned +// storage-service URLs. In tests it maps local filenames to HTTP URLs. +func WithBinaryURLResolver(r v2.BinaryURLResolver) func(*eventHandler) { + return func(e *eventHandler) { + e.binaryURLResolver = r + } +} + func WithLocalSecrets(lggr logger.Logger, secrets map[string]string) func(*eventHandler) { return func(e *eventHandler) { if len(secrets) == 0 { @@ -944,6 +954,7 @@ func (h *eventHandler) confidentialEngineFactory( workflowName.String(), spec.WorkflowTag, attrs.VaultDonSecrets, + h.binaryURLResolver, lggr, ) From 7fb5ae599e0d926070ce0d872b17ab21fe9ada78 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 26 Mar 2026 22:10:07 +0100 Subject: [PATCH 32/67] Revert BinaryURLResolver changes --- core/services/workflows/syncer/v2/handler.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 25eb9bbec09..2bda9999383 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -78,7 +78,6 @@ type eventHandler struct { billingClient metering.BillingClient orgResolver orgresolver.OrgResolver secretsFetcher v2.SecretsFetcher - binaryURLResolver v2.BinaryURLResolver // WorkflowRegistryAddress is the address of the workflow registry contract workflowRegistryAddress string @@ -180,15 +179,6 @@ func WithSecretsFetcher(sf v2.SecretsFetcher) func(*eventHandler) { } } -// WithBinaryURLResolver sets a resolver that transforms raw binary URLs -// into URLs the enclave can fetch. In production this produces presigned -// storage-service URLs. In tests it maps local filenames to HTTP URLs. -func WithBinaryURLResolver(r v2.BinaryURLResolver) func(*eventHandler) { - return func(e *eventHandler) { - e.binaryURLResolver = r - } -} - func WithLocalSecrets(lggr logger.Logger, secrets map[string]string) func(*eventHandler) { return func(e *eventHandler) { if len(secrets) == 0 { @@ -954,7 +944,6 @@ func (h *eventHandler) confidentialEngineFactory( workflowName.String(), spec.WorkflowTag, attrs.VaultDonSecrets, - h.binaryURLResolver, lggr, ) From b099f98f46d498a27370aaa22f8f6550632c3777 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Fri, 27 Mar 2026 14:30:11 +0100 Subject: [PATCH 33/67] File fetcher: extract filename from HTTP URLs for local resolution --- core/services/workflows/syncer/fetcher.go | 3 +++ core/services/workflows/syncer/v2/fetcher.go | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/core/services/workflows/syncer/fetcher.go b/core/services/workflows/syncer/fetcher.go index c959b6b4da7..01de212507a 100644 --- a/core/services/workflows/syncer/fetcher.go +++ b/core/services/workflows/syncer/fetcher.go @@ -177,6 +177,9 @@ func newFileFetcher(basePath string, lggr logger.Logger) types.FetcherFunc { if err != nil { return nil, fmt.Errorf("invalid URL: %w", err) } + if u.Scheme == "http" || u.Scheme == "https" { + u.Path = filepath.Base(u.Path) + } fullPath := filepath.Clean(u.Path) // ensure that the incoming request URL is either relative or absolute but within the basePath diff --git a/core/services/workflows/syncer/v2/fetcher.go b/core/services/workflows/syncer/v2/fetcher.go index e86f998f6bb..cb334ff8fb8 100644 --- a/core/services/workflows/syncer/v2/fetcher.go +++ b/core/services/workflows/syncer/v2/fetcher.go @@ -211,6 +211,11 @@ func newFileFetcher(basePath string, lggr logger.Logger) types.FetcherFunc { if err != nil { return nil, fmt.Errorf("invalid URL: %w", err) } + // Confidential workflows register with HTTP URLs (for the enclave). + // Extract the filename so the file fetcher can find the local copy. + if u.Scheme == "http" || u.Scheme == "https" { + u.Path = filepath.Base(u.Path) + } fullPath := filepath.Clean(u.Path) // ensure that the incoming request URL is either relative or absolute but within the basePath From 50f3262e634f525dcdc4e01c3cbd1bc0445d9dda Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Fri, 27 Mar 2026 16:06:58 +0100 Subject: [PATCH 34/67] Update CC plugin refs to d7848a5 --- plugins/plugins.private.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/plugins.private.yaml b/plugins/plugins.private.yaml index 398abfe2bde..7d2ce682d4a 100644 --- a/plugins/plugins.private.yaml +++ b/plugins/plugins.private.yaml @@ -56,10 +56,10 @@ plugins: installPath: "." confidential-http: - moduleURI: "github.com/smartcontractkit/confidential-compute/enclave/apps/confidential-http/capability" - gitRef: "b7f43502f174425bfaee79157dc068bd7aadd9b9" + gitRef: "d7848a56e1fe47560bf1bd97de0e5aa126f062cb" installPath: "./cmd/confidential-http" confidential-workflows: - moduleURI: "github.com/smartcontractkit/confidential-compute/enclave/apps/confidential-workflows/capability" - gitRef: "b7f43502f174425bfaee79157dc068bd7aadd9b9" + gitRef: "d7848a56e1fe47560bf1bd97de0e5aa126f062cb" installPath: "./cmd/confidential-workflows" >>>>>>> fb2ffec52b (Update CC plugin refs to b7f4350) From 6297bdd179974f9ad523daa8aef094a9508e60f7 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Mon, 30 Mar 2026 18:58:21 +0200 Subject: [PATCH 35/67] Bump chainlink-common to c1870a5e, merge PR 1/5 review fixes, update CC plugin ref - chainlink-common v0.11.2-0.20260330123353-c1870a5eff52 (teeattestation merged into root) - Pavel review fixes: sendResponseAndCleanup, error sanitization, requestTimeoutSec - CC plugin ref updated to 526a251 --- .../handlers/confidentialrelay/handler.go | 84 +++++++++++-------- plugins/plugins.private.yaml | 5 +- 2 files changed, 51 insertions(+), 38 deletions(-) diff --git a/core/services/gateway/handlers/confidentialrelay/handler.go b/core/services/gateway/handlers/confidentialrelay/handler.go index 1b564949783..9026f78c113 100644 --- a/core/services/gateway/handlers/confidentialrelay/handler.go +++ b/core/services/gateway/handlers/confidentialrelay/handler.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "maps" "strconv" "sync" "time" @@ -222,7 +221,7 @@ func (h *handler) removeExpiredRequests(ctx context.Context) { for _, er := range expiredRequests { responses := er.copiedResponses() h.lggr.Debugw("request expired without quorum", "requestID", er.req.ID, "responseCount", len(responses), "required", h.donConfig.F+1) - err := h.sendResponseAndCleanup(ctx, er, h.errorResponse(er.req, api.RequestTimeoutError, fmt.Errorf("request expired: got %d/%d responses", len(responses), h.donConfig.F+1), nil)) + err := h.sendResponseAndCleanup(ctx, er, api.RequestTimeoutError, fmt.Errorf("request expired: got %d/%d responses", len(responses), h.donConfig.F+1), nil) if err != nil { h.lggr.Errorw("error sending response to user", "requestID", er.req.ID, "error", err) } @@ -306,9 +305,12 @@ func (h *handler) HandleNodeMessage(ctx context.Context, resp *jsonrpc.Response[ case errors.Is(err, errInsufficientResponsesForQuorum): l.Debugw("aggregating responses, waiting for other nodes...", "error", err) return nil + case errors.Is(err, errQuorumUnobtainable): + l.Errorw("quorum unobtainable, returning error to user", "error", err) + return h.sendResponseAndCleanup(ctx, ar, api.FatalError, err, nil) case err != nil: - l.Error("quorum unobtainable, returning response to user...", "error", err, "responses", maps.Values(ar.responses)) - return h.sendResponseAndCleanup(ctx, ar, h.errorResponse(ar.req, api.FatalError, err, nil)) + l.Errorw("unexpected aggregation error", "error", err) + return h.sendResponseAndCleanup(ctx, ar, api.FatalError, err, nil) } return h.sendSuccessResponseAndCleanup(ctx, l, ar, aggregatedResp) @@ -325,7 +327,7 @@ func (h *handler) fanOutToNodes(ctx context.Context, l logger.Logger, ar *active } if len(nodeErrors) == len(h.donConfig.Members) && len(nodeErrors) > 0 { - return h.sendResponseAndCleanup(ctx, ar, h.errorResponse(ar.req, api.FatalError, errors.New("failed to forward user request to nodes"), nil)) + return h.sendResponseAndCleanup(ctx, ar, api.FatalError, errors.New("failed to forward user request to nodes"), nil) } l.Debugw("successfully forwarded request to relay nodes") @@ -336,7 +338,7 @@ func (h *handler) sendSuccessResponseAndCleanup(ctx context.Context, l logger.Lo rawResponse, err := jsonrpc.EncodeResponse(resp) if err != nil { l.Errorw("failed to encode response", "error", err) - return h.sendResponseAndCleanup(ctx, ar, h.errorResponse(ar.req, api.NodeReponseEncodingError, fmt.Errorf("failed to marshal response: %w", err), nil)) + return h.sendResponseAndCleanup(ctx, ar, api.NodeReponseEncodingError, fmt.Errorf("failed to marshal response: %w", err), nil) } var errorCode api.ErrorCode @@ -347,19 +349,33 @@ func (h *handler) sendSuccessResponseAndCleanup(ctx context.Context, l logger.Lo } l.Debugw("issued user callback", "errorCode", errorCode) - successResp := gwhandlers.UserCallbackPayload{ + payload := gwhandlers.UserCallbackPayload{ RawResponse: rawResponse, ErrorCode: errorCode, } - return h.sendResponseAndCleanup(ctx, ar, successResp) + + h.recordMetrics(ctx, errorCode) + sendErr := ar.SendResponse(payload) + + h.mu.Lock() + delete(h.activeRequests, ar.req.ID) + h.mu.Unlock() + + if sendErr != nil { + h.lggr.Errorw("error sending response to user", "requestID", ar.req.ID, "error", sendErr) + return sendErr + } + + h.lggr.Debugw("response sent to user", "requestID", ar.req.ID, "errorCode", errorCode) + return nil } -func (h *handler) errorResponse( - req jsonrpc.Request[json.RawMessage], - errorCode api.ErrorCode, - err error, - data []byte, -) gwhandlers.UserCallbackPayload { +// sendResponseAndCleanup sanitizes the error, encodes a JSON-RPC error +// response, records metrics, sends the response to the user, and removes +// the request from activeRequests. The request is always removed regardless +// of whether the send succeeds, since a failed callback cannot be retried. +func (h *handler) sendResponseAndCleanup(ctx context.Context, ar *activeRequest, errorCode api.ErrorCode, err error, data []byte) error { + req := ar.req switch errorCode { case api.FatalError: case api.NodeReponseEncodingError: @@ -383,7 +399,7 @@ func (h *handler) errorResponse( case api.LimitExceededError: } - return gwhandlers.UserCallbackPayload{ + payload := gwhandlers.UserCallbackPayload{ RawResponse: h.codec.EncodeNewErrorResponse( req.ID, api.ToJSONRPCErrorCode(errorCode), @@ -392,13 +408,25 @@ func (h *handler) errorResponse( ), ErrorCode: errorCode, } + + h.recordMetrics(ctx, errorCode) + sendErr := ar.SendResponse(payload) + + h.mu.Lock() + delete(h.activeRequests, ar.req.ID) + h.mu.Unlock() + + if sendErr != nil { + h.lggr.Errorw("error sending response to user", "requestID", ar.req.ID, "error", sendErr) + return sendErr + } + + h.lggr.Debugw("response sent to user", "requestID", ar.req.ID, "errorCode", errorCode) + return nil } -// sendResponseAndCleanup sends the response to the user and removes the -// request from activeRequests. The request is always removed regardless of -// whether the send succeeds, since a failed callback cannot be retried. -func (h *handler) sendResponseAndCleanup(ctx context.Context, userRequest *activeRequest, resp gwhandlers.UserCallbackPayload) error { - switch resp.ErrorCode { +func (h *handler) recordMetrics(ctx context.Context, errorCode api.ErrorCode) { + switch errorCode { case api.StaleNodeResponseError: case api.FatalError: case api.NodeReponseEncodingError: @@ -406,7 +434,7 @@ func (h *handler) sendResponseAndCleanup(ctx context.Context, userRequest *activ case api.HandlerError: h.metrics.requestInternalError.Add(ctx, 1, metric.WithAttributes( attribute.String("don_id", h.donConfig.DonId), - attribute.String("error", resp.ErrorCode.String()), + attribute.String("error", errorCode.String()), )) case api.InvalidParamsError: case api.UnsupportedMethodError: @@ -422,18 +450,4 @@ func (h *handler) sendResponseAndCleanup(ctx context.Context, userRequest *activ case api.ConflictError: case api.LimitExceededError: } - - err := userRequest.SendResponse(resp) - - h.mu.Lock() - delete(h.activeRequests, userRequest.req.ID) - h.mu.Unlock() - - if err != nil { - h.lggr.Errorw("error sending response to user", "requestID", userRequest.req.ID, "error", err) - return err - } - - h.lggr.Debugw("response sent to user", "requestID", userRequest.req.ID, "errorCode", resp.ErrorCode) - return nil } diff --git a/plugins/plugins.private.yaml b/plugins/plugins.private.yaml index 7d2ce682d4a..7276869a68f 100644 --- a/plugins/plugins.private.yaml +++ b/plugins/plugins.private.yaml @@ -56,10 +56,9 @@ plugins: installPath: "." confidential-http: - moduleURI: "github.com/smartcontractkit/confidential-compute/enclave/apps/confidential-http/capability" - gitRef: "d7848a56e1fe47560bf1bd97de0e5aa126f062cb" + gitRef: "526a251" installPath: "./cmd/confidential-http" confidential-workflows: - moduleURI: "github.com/smartcontractkit/confidential-compute/enclave/apps/confidential-workflows/capability" - gitRef: "d7848a56e1fe47560bf1bd97de0e5aa126f062cb" + gitRef: "526a251" installPath: "./cmd/confidential-workflows" ->>>>>>> fb2ffec52b (Update CC plugin refs to b7f4350) From 134c0a6e6c910d7cb02fb11010ed2aa07196be07 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Mon, 30 Mar 2026 19:05:30 +0200 Subject: [PATCH 36/67] Unify sendResponseAndCleanup to handle both success and error paths --- .../handlers/confidentialrelay/handler.go | 139 ++++++++---------- 1 file changed, 64 insertions(+), 75 deletions(-) diff --git a/core/services/gateway/handlers/confidentialrelay/handler.go b/core/services/gateway/handlers/confidentialrelay/handler.go index 9026f78c113..20ad2b018e6 100644 --- a/core/services/gateway/handlers/confidentialrelay/handler.go +++ b/core/services/gateway/handlers/confidentialrelay/handler.go @@ -221,7 +221,7 @@ func (h *handler) removeExpiredRequests(ctx context.Context) { for _, er := range expiredRequests { responses := er.copiedResponses() h.lggr.Debugw("request expired without quorum", "requestID", er.req.ID, "responseCount", len(responses), "required", h.donConfig.F+1) - err := h.sendResponseAndCleanup(ctx, er, api.RequestTimeoutError, fmt.Errorf("request expired: got %d/%d responses", len(responses), h.donConfig.F+1), nil) + err := h.sendResponseAndCleanup(ctx, er, nil, api.RequestTimeoutError, fmt.Errorf("request expired: got %d/%d responses", len(responses), h.donConfig.F+1)) if err != nil { h.lggr.Errorw("error sending response to user", "requestID", er.req.ID, "error", err) } @@ -307,13 +307,13 @@ func (h *handler) HandleNodeMessage(ctx context.Context, resp *jsonrpc.Response[ return nil case errors.Is(err, errQuorumUnobtainable): l.Errorw("quorum unobtainable, returning error to user", "error", err) - return h.sendResponseAndCleanup(ctx, ar, api.FatalError, err, nil) + return h.sendResponseAndCleanup(ctx, ar, nil, api.FatalError, err) case err != nil: l.Errorw("unexpected aggregation error", "error", err) - return h.sendResponseAndCleanup(ctx, ar, api.FatalError, err, nil) + return h.sendResponseAndCleanup(ctx, ar, nil, api.FatalError, err) } - return h.sendSuccessResponseAndCleanup(ctx, l, ar, aggregatedResp) + return h.sendResponseAndCleanup(ctx, ar, aggregatedResp, 0, nil) } func (h *handler) fanOutToNodes(ctx context.Context, l logger.Logger, ar *activeRequest) error { @@ -327,86 +327,75 @@ func (h *handler) fanOutToNodes(ctx context.Context, l logger.Logger, ar *active } if len(nodeErrors) == len(h.donConfig.Members) && len(nodeErrors) > 0 { - return h.sendResponseAndCleanup(ctx, ar, api.FatalError, errors.New("failed to forward user request to nodes"), nil) + return h.sendResponseAndCleanup(ctx, ar, nil, api.FatalError, errors.New("failed to forward user request to nodes")) } l.Debugw("successfully forwarded request to relay nodes") return nil } -func (h *handler) sendSuccessResponseAndCleanup(ctx context.Context, l logger.Logger, ar *activeRequest, resp *jsonrpc.Response[json.RawMessage]) error { - rawResponse, err := jsonrpc.EncodeResponse(resp) - if err != nil { - l.Errorw("failed to encode response", "error", err) - return h.sendResponseAndCleanup(ctx, ar, api.NodeReponseEncodingError, fmt.Errorf("failed to marshal response: %w", err), nil) - } - - var errorCode api.ErrorCode - if resp.Error != nil { - errorCode = api.FromJSONRPCErrorCode(resp.Error.Code) - } else { - errorCode = api.NoError - } - - l.Debugw("issued user callback", "errorCode", errorCode) - payload := gwhandlers.UserCallbackPayload{ - RawResponse: rawResponse, - ErrorCode: errorCode, - } - - h.recordMetrics(ctx, errorCode) - sendErr := ar.SendResponse(payload) - - h.mu.Lock() - delete(h.activeRequests, ar.req.ID) - h.mu.Unlock() - - if sendErr != nil { - h.lggr.Errorw("error sending response to user", "requestID", ar.req.ID, "error", sendErr) - return sendErr +// sendResponseAndCleanup handles both success and error responses. For +// success, pass the aggregated resp; for errors, pass nil resp with an +// errorCode and err. The request is always removed from activeRequests +// regardless of whether the send succeeds, since a failed callback cannot +// be retried. +func (h *handler) sendResponseAndCleanup(ctx context.Context, ar *activeRequest, resp *jsonrpc.Response[json.RawMessage], errorCode api.ErrorCode, err error) error { + var payload gwhandlers.UserCallbackPayload + + if resp != nil { + rawResponse, encErr := jsonrpc.EncodeResponse(resp) + if encErr != nil { + h.lggr.Errorw("failed to encode response", "requestID", ar.req.ID, "error", encErr) + errorCode = api.NodeReponseEncodingError + err = fmt.Errorf("failed to marshal response: %w", encErr) + resp = nil + } else { + if resp.Error != nil { + errorCode = api.FromJSONRPCErrorCode(resp.Error.Code) + } else { + errorCode = api.NoError + } + payload = gwhandlers.UserCallbackPayload{ + RawResponse: rawResponse, + ErrorCode: errorCode, + } + } } - h.lggr.Debugw("response sent to user", "requestID", ar.req.ID, "errorCode", errorCode) - return nil -} - -// sendResponseAndCleanup sanitizes the error, encodes a JSON-RPC error -// response, records metrics, sends the response to the user, and removes -// the request from activeRequests. The request is always removed regardless -// of whether the send succeeds, since a failed callback cannot be retried. -func (h *handler) sendResponseAndCleanup(ctx context.Context, ar *activeRequest, errorCode api.ErrorCode, err error, data []byte) error { - req := ar.req - switch errorCode { - case api.FatalError: - case api.NodeReponseEncodingError: - h.lggr.Errorw(err.Error(), "requestID", req.ID) - err = errors.New(errorCode.String()) - case api.InvalidParamsError: - h.lggr.Errorw("invalid params", "requestID", req.ID, "params", string(*req.Params)) - err = fmt.Errorf("invalid params error: %w", err) - case api.UnsupportedMethodError: - h.lggr.Errorw("unsupported method", "requestID", req.ID, "method", req.Method, "error", err.Error()) - err = fmt.Errorf("unsupported method(%s): %w", req.Method, err) - case api.UserMessageParseError: - h.lggr.Errorw("user message parse error", "requestID", req.ID, "error", err.Error()) - err = fmt.Errorf("user message parse error: %w", err) - case api.NoError: - case api.UnsupportedDONIdError: - case api.HandlerError: - case api.RequestTimeoutError: - case api.StaleNodeResponseError: - case api.ConflictError: - case api.LimitExceededError: - } + if resp == nil { + req := ar.req + switch errorCode { + case api.FatalError: + case api.NodeReponseEncodingError: + h.lggr.Errorw(err.Error(), "requestID", req.ID) + err = errors.New(errorCode.String()) + case api.InvalidParamsError: + h.lggr.Errorw("invalid params", "requestID", req.ID, "params", string(*req.Params)) + err = fmt.Errorf("invalid params error: %w", err) + case api.UnsupportedMethodError: + h.lggr.Errorw("unsupported method", "requestID", req.ID, "method", req.Method, "error", err.Error()) + err = fmt.Errorf("unsupported method(%s): %w", req.Method, err) + case api.UserMessageParseError: + h.lggr.Errorw("user message parse error", "requestID", req.ID, "error", err.Error()) + err = fmt.Errorf("user message parse error: %w", err) + case api.NoError: + case api.UnsupportedDONIdError: + case api.HandlerError: + case api.RequestTimeoutError: + case api.StaleNodeResponseError: + case api.ConflictError: + case api.LimitExceededError: + } - payload := gwhandlers.UserCallbackPayload{ - RawResponse: h.codec.EncodeNewErrorResponse( - req.ID, - api.ToJSONRPCErrorCode(errorCode), - err.Error(), - data, - ), - ErrorCode: errorCode, + payload = gwhandlers.UserCallbackPayload{ + RawResponse: h.codec.EncodeNewErrorResponse( + req.ID, + api.ToJSONRPCErrorCode(errorCode), + err.Error(), + nil, + ), + ErrorCode: errorCode, + } } h.recordMetrics(ctx, errorCode) From bfa245d70243ace953cb861b1ca148e7478b9f56 Mon Sep 17 00:00:00 2001 From: pavel-raykov Date: Mon, 30 Mar 2026 22:25:27 +0200 Subject: [PATCH 37/67] Simplify `sendResponseAndCleanup`. --- .../handlers/confidentialrelay/handler.go | 118 ++++++------------ 1 file changed, 40 insertions(+), 78 deletions(-) diff --git a/core/services/gateway/handlers/confidentialrelay/handler.go b/core/services/gateway/handlers/confidentialrelay/handler.go index 20ad2b018e6..f0739a3cd23 100644 --- a/core/services/gateway/handlers/confidentialrelay/handler.go +++ b/core/services/gateway/handlers/confidentialrelay/handler.go @@ -221,7 +221,7 @@ func (h *handler) removeExpiredRequests(ctx context.Context) { for _, er := range expiredRequests { responses := er.copiedResponses() h.lggr.Debugw("request expired without quorum", "requestID", er.req.ID, "responseCount", len(responses), "required", h.donConfig.F+1) - err := h.sendResponseAndCleanup(ctx, er, nil, api.RequestTimeoutError, fmt.Errorf("request expired: got %d/%d responses", len(responses), h.donConfig.F+1)) + err := h.sendResponseAndCleanup(ctx, er, h.constructErrorResponse(er.req, api.RequestTimeoutError, fmt.Errorf("request expired: got %d/%d responses", len(responses), h.donConfig.F+1))) if err != nil { h.lggr.Errorw("error sending response to user", "requestID", er.req.ID, "error", err) } @@ -307,13 +307,21 @@ func (h *handler) HandleNodeMessage(ctx context.Context, resp *jsonrpc.Response[ return nil case errors.Is(err, errQuorumUnobtainable): l.Errorw("quorum unobtainable, returning error to user", "error", err) - return h.sendResponseAndCleanup(ctx, ar, nil, api.FatalError, err) + return h.sendResponseAndCleanup(ctx, ar, h.constructErrorResponse(ar.req, api.FatalError, err)) case err != nil: l.Errorw("unexpected aggregation error", "error", err) - return h.sendResponseAndCleanup(ctx, ar, nil, api.FatalError, err) + return h.sendResponseAndCleanup(ctx, ar, h.constructErrorResponse(ar.req, api.FatalError, err)) } - return h.sendResponseAndCleanup(ctx, ar, aggregatedResp, 0, nil) + rawResponse, err := jsonrpc.EncodeResponse(aggregatedResp) + if err != nil { + h.lggr.Errorw("failed to encode response", "requestID", ar.req.ID, "error", err) + return h.sendResponseAndCleanup(ctx, ar, h.constructErrorResponse(ar.req, api.NodeReponseEncodingError, err)) + } + return h.sendResponseAndCleanup(ctx, ar, gwhandlers.UserCallbackPayload{ + RawResponse: rawResponse, + ErrorCode: api.NoError, + }) } func (h *handler) fanOutToNodes(ctx context.Context, l logger.Logger, ar *activeRequest) error { @@ -327,78 +335,19 @@ func (h *handler) fanOutToNodes(ctx context.Context, l logger.Logger, ar *active } if len(nodeErrors) == len(h.donConfig.Members) && len(nodeErrors) > 0 { - return h.sendResponseAndCleanup(ctx, ar, nil, api.FatalError, errors.New("failed to forward user request to nodes")) + return h.sendResponseAndCleanup(ctx, ar, h.constructErrorResponse(ar.req, api.FatalError, errors.New("failed to forward user request to nodes"))) } l.Debugw("successfully forwarded request to relay nodes") return nil } -// sendResponseAndCleanup handles both success and error responses. For -// success, pass the aggregated resp; for errors, pass nil resp with an -// errorCode and err. The request is always removed from activeRequests +// sendResponseAndCleanup sends payload. +// The request is always removed from activeRequests // regardless of whether the send succeeds, since a failed callback cannot // be retried. -func (h *handler) sendResponseAndCleanup(ctx context.Context, ar *activeRequest, resp *jsonrpc.Response[json.RawMessage], errorCode api.ErrorCode, err error) error { - var payload gwhandlers.UserCallbackPayload - - if resp != nil { - rawResponse, encErr := jsonrpc.EncodeResponse(resp) - if encErr != nil { - h.lggr.Errorw("failed to encode response", "requestID", ar.req.ID, "error", encErr) - errorCode = api.NodeReponseEncodingError - err = fmt.Errorf("failed to marshal response: %w", encErr) - resp = nil - } else { - if resp.Error != nil { - errorCode = api.FromJSONRPCErrorCode(resp.Error.Code) - } else { - errorCode = api.NoError - } - payload = gwhandlers.UserCallbackPayload{ - RawResponse: rawResponse, - ErrorCode: errorCode, - } - } - } - - if resp == nil { - req := ar.req - switch errorCode { - case api.FatalError: - case api.NodeReponseEncodingError: - h.lggr.Errorw(err.Error(), "requestID", req.ID) - err = errors.New(errorCode.String()) - case api.InvalidParamsError: - h.lggr.Errorw("invalid params", "requestID", req.ID, "params", string(*req.Params)) - err = fmt.Errorf("invalid params error: %w", err) - case api.UnsupportedMethodError: - h.lggr.Errorw("unsupported method", "requestID", req.ID, "method", req.Method, "error", err.Error()) - err = fmt.Errorf("unsupported method(%s): %w", req.Method, err) - case api.UserMessageParseError: - h.lggr.Errorw("user message parse error", "requestID", req.ID, "error", err.Error()) - err = fmt.Errorf("user message parse error: %w", err) - case api.NoError: - case api.UnsupportedDONIdError: - case api.HandlerError: - case api.RequestTimeoutError: - case api.StaleNodeResponseError: - case api.ConflictError: - case api.LimitExceededError: - } - - payload = gwhandlers.UserCallbackPayload{ - RawResponse: h.codec.EncodeNewErrorResponse( - req.ID, - api.ToJSONRPCErrorCode(errorCode), - err.Error(), - nil, - ), - ErrorCode: errorCode, - } - } - - h.recordMetrics(ctx, errorCode) +func (h *handler) sendResponseAndCleanup(ctx context.Context, ar *activeRequest, payload gwhandlers.UserCallbackPayload) error { + h.recordMetrics(ctx, payload.ErrorCode) sendErr := ar.SendResponse(payload) h.mu.Lock() @@ -410,24 +359,17 @@ func (h *handler) sendResponseAndCleanup(ctx context.Context, ar *activeRequest, return sendErr } - h.lggr.Debugw("response sent to user", "requestID", ar.req.ID, "errorCode", errorCode) + h.lggr.Debugw("response sent to user", "requestID", ar.req.ID, "errorCode", payload.ErrorCode) return nil } func (h *handler) recordMetrics(ctx context.Context, errorCode api.ErrorCode) { switch errorCode { - case api.StaleNodeResponseError: - case api.FatalError: - case api.NodeReponseEncodingError: - case api.RequestTimeoutError: case api.HandlerError: h.metrics.requestInternalError.Add(ctx, 1, metric.WithAttributes( attribute.String("don_id", h.donConfig.DonId), attribute.String("error", errorCode.String()), )) - case api.InvalidParamsError: - case api.UnsupportedMethodError: - case api.UserMessageParseError: case api.UnsupportedDONIdError: h.metrics.requestUserError.Add(ctx, 1, metric.WithAttributes( attribute.String("don_id", h.donConfig.DonId), @@ -436,7 +378,27 @@ func (h *handler) recordMetrics(ctx context.Context, errorCode api.ErrorCode) { h.metrics.requestSuccess.Add(ctx, 1, metric.WithAttributes( attribute.String("don_id", h.donConfig.DonId), )) - case api.ConflictError: - case api.LimitExceededError: + } +} + +func (h *handler) constructErrorResponse(req jsonrpc.Request[json.RawMessage], errorCode api.ErrorCode, err error) gwhandlers.UserCallbackPayload { + switch errorCode { + case api.NodeReponseEncodingError: + err = errors.New(errorCode.String()) + case api.InvalidParamsError: + err = fmt.Errorf("invalid params error: %w", err) + case api.UnsupportedMethodError: + err = fmt.Errorf("unsupported method(%s): %w", req.Method, err) + case api.UserMessageParseError: + err = fmt.Errorf("user message parse error: %w", err) + } + return gwhandlers.UserCallbackPayload{ + RawResponse: h.codec.EncodeNewErrorResponse( + req.ID, + api.ToJSONRPCErrorCode(errorCode), + err.Error(), + nil, + ), + ErrorCode: errorCode, } } From 1d331d875098239d7bca18c85ca26c17425296a6 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 31 Mar 2026 12:01:27 +0200 Subject: [PATCH 38/67] Fix exhaustive lint: restore missing switch cases in recordMetrics and constructErrorResponse --- .../handlers/confidentialrelay/handler.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/core/services/gateway/handlers/confidentialrelay/handler.go b/core/services/gateway/handlers/confidentialrelay/handler.go index f0739a3cd23..e123da52307 100644 --- a/core/services/gateway/handlers/confidentialrelay/handler.go +++ b/core/services/gateway/handlers/confidentialrelay/handler.go @@ -365,11 +365,18 @@ func (h *handler) sendResponseAndCleanup(ctx context.Context, ar *activeRequest, func (h *handler) recordMetrics(ctx context.Context, errorCode api.ErrorCode) { switch errorCode { + case api.StaleNodeResponseError: + case api.FatalError: + case api.NodeReponseEncodingError: + case api.RequestTimeoutError: case api.HandlerError: h.metrics.requestInternalError.Add(ctx, 1, metric.WithAttributes( attribute.String("don_id", h.donConfig.DonId), attribute.String("error", errorCode.String()), )) + case api.InvalidParamsError: + case api.UnsupportedMethodError: + case api.UserMessageParseError: case api.UnsupportedDONIdError: h.metrics.requestUserError.Add(ctx, 1, metric.WithAttributes( attribute.String("don_id", h.donConfig.DonId), @@ -378,6 +385,8 @@ func (h *handler) recordMetrics(ctx context.Context, errorCode api.ErrorCode) { h.metrics.requestSuccess.Add(ctx, 1, metric.WithAttributes( attribute.String("don_id", h.donConfig.DonId), )) + case api.ConflictError: + case api.LimitExceededError: } } @@ -391,6 +400,14 @@ func (h *handler) constructErrorResponse(req jsonrpc.Request[json.RawMessage], e err = fmt.Errorf("unsupported method(%s): %w", req.Method, err) case api.UserMessageParseError: err = fmt.Errorf("user message parse error: %w", err) + case api.NoError: + case api.UnsupportedDONIdError: + case api.HandlerError: + case api.FatalError: + case api.RequestTimeoutError: + case api.StaleNodeResponseError: + case api.ConflictError: + case api.LimitExceededError: } return gwhandlers.UserCallbackPayload{ RawResponse: h.codec.EncodeNewErrorResponse( From 5c6e44632719fe78de1e0ef910b5f9e144286062 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 1 Apr 2026 13:44:47 +0200 Subject: [PATCH 39/67] fan out relay requests to don nodes concurrently --- .../handlers/confidentialrelay/handler.go | 28 ++++-- .../confidentialrelay/handler_test.go | 85 +++++++++++++++++++ 2 files changed, 106 insertions(+), 7 deletions(-) diff --git a/core/services/gateway/handlers/confidentialrelay/handler.go b/core/services/gateway/handlers/confidentialrelay/handler.go index e123da52307..e5811fbadd3 100644 --- a/core/services/gateway/handlers/confidentialrelay/handler.go +++ b/core/services/gateway/handlers/confidentialrelay/handler.go @@ -12,6 +12,7 @@ import ( "github.com/jonboulle/clockwork" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + "golang.org/x/sync/errgroup" "github.com/smartcontractkit/chainlink-common/pkg/beholder" relaytypes "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/actions/confidentialrelay" @@ -325,16 +326,29 @@ func (h *handler) HandleNodeMessage(ctx context.Context, resp *jsonrpc.Response[ } func (h *handler) fanOutToNodes(ctx context.Context, l logger.Logger, ar *activeRequest) error { - var nodeErrors []error + var ( + group errgroup.Group + nodeErrors int + nodeErrorsMu sync.Mutex + ) + for _, node := range h.donConfig.Members { - err := h.don.SendToNode(ctx, node.Address, &ar.req) - if err != nil { - nodeErrors = append(nodeErrors, err) - l.Errorw("error sending request to node", "node", node.Address, "error", err) - } + node := node + group.Go(func() error { + err := h.don.SendToNode(ctx, node.Address, &ar.req) + if err != nil { + nodeErrorsMu.Lock() + nodeErrors++ + nodeErrorsMu.Unlock() + l.Errorw("error sending request to node", "node", node.Address, "error", err) + } + return nil + }) } - if len(nodeErrors) == len(h.donConfig.Members) && len(nodeErrors) > 0 { + _ = group.Wait() + + if nodeErrors == len(h.donConfig.Members) && nodeErrors > 0 { return h.sendResponseAndCleanup(ctx, ar, h.constructErrorResponse(ar.req, api.FatalError, errors.New("failed to forward user request to nodes"))) } diff --git a/core/services/gateway/handlers/confidentialrelay/handler_test.go b/core/services/gateway/handlers/confidentialrelay/handler_test.go index efc6e2c1689..7d3f3056d65 100644 --- a/core/services/gateway/handlers/confidentialrelay/handler_test.go +++ b/core/services/gateway/handlers/confidentialrelay/handler_test.go @@ -25,6 +25,38 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers/mocks" ) +type barrierDON struct { + total int + mu sync.Mutex + started int + allStarted chan struct{} + releaseOnce sync.Once +} + +func newBarrierDON(total int) *barrierDON { + return &barrierDON{ + total: total, + allStarted: make(chan struct{}), + } +} + +func (d *barrierDON) SendToNode(_ context.Context, _ string, _ *jsonrpc.Request[json.RawMessage]) error { + d.mu.Lock() + d.started++ + if d.started == d.total { + d.releaseOnce.Do(func() { close(d.allStarted) }) + } + ch := d.allStarted + d.mu.Unlock() + + <-ch + return nil +} + +func (d *barrierDON) forceRelease() { + d.releaseOnce.Do(func() { close(d.allStarted) }) +} + var nodeOne = config.NodeConfig{ Name: "node1", Address: "0x1234", @@ -495,6 +527,59 @@ func TestConfidentialRelayHandler_AllNodesFanOutFail(t *testing.T) { wg.Wait() } +func TestConfidentialRelayHandler_FanOutToNodes_IsConcurrent(t *testing.T) { + lggr := logger.Test(t) + don := newBarrierDON(2) + donConfig := &config.DONConfig{ + DonId: "test_relay_don", + F: 1, + Members: []config.NodeConfig{ + {Name: "node0", Address: "0x0000"}, + {Name: "node1", Address: "0x0001"}, + }, + } + + methodConfig, err := json.Marshal(Config{ + RequestTimeoutSec: 30, + NodeRateLimiter: ratelimit.RateLimiterConfig{ + GlobalRPS: 100, + GlobalBurst: 100, + PerSenderRPS: 10, + PerSenderBurst: 10, + }, + }) + require.NoError(t, err) + + h, err := NewHandler(methodConfig, donConfig, don, lggr, clockwork.NewFakeClock()) + require.NoError(t, err) + + cb := common.NewCallback() + params := json.RawMessage(`{"workflow_id":"wf1"}`) + req := jsonrpc.Request[json.RawMessage]{ + ID: "req-concurrent-fanout", + Method: MethodCapabilityExec, + Params: ¶ms, + } + + done := make(chan error, 1) + go func() { + done <- h.HandleJSONRPCUserMessage(t.Context(), req, cb) + }() + + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(100 * time.Millisecond): + don.forceRelease() + t.Fatal("HandleJSONRPCUserMessage did not fan out to nodes concurrently") + } + + don.mu.Lock() + started := don.started + don.mu.Unlock() + assert.Equal(t, 2, started) +} + func TestConfidentialRelayHandler_CapabilityExecMethod(t *testing.T) { h, cb, don, _ := setupHandler(t, 4) h.aggregator = &respondingMockAggregator{} From a9cd43baa1ee4669825d3d564095690e303673fa Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 2 Apr 2026 14:55:02 +0200 Subject: [PATCH 40/67] use cre settings for relay gateway rate limits (cherry picked from commit 7b4af73a73e00482fd16a6088fc1999361c7bf62) (cherry picked from commit 9a721064b9ca7d070231a0031b33d1db82b16eef) --- core/services/gateway/handler_factory.go | 2 +- .../handlers/confidentialrelay/handler.go | 72 +++++++++++++------ .../confidentialrelay/handler_test.go | 33 +++------ deployment/cre/jobs/pkg/gateway_job.go | 9 +-- deployment/go.mod | 2 +- deployment/go.sum | 4 +- 6 files changed, 65 insertions(+), 57 deletions(-) diff --git a/core/services/gateway/handler_factory.go b/core/services/gateway/handler_factory.go index 80f3a08fecc..af62f1d0e87 100644 --- a/core/services/gateway/handler_factory.go +++ b/core/services/gateway/handler_factory.go @@ -89,7 +89,7 @@ func (hf *handlerFactory) NewHandler( requestAuthorizer := vaultcap.NewRequestAuthorizer(hf.lggr, hf.workflowRegistrySyncer) return vault.NewHandler(handlerConfig, donConfig, don, hf.capabilitiesRegistry, requestAuthorizer, hf.lggr, clockwork.NewRealClock(), hf.lf) case ConfidentialRelayHandlerType: - return confidentialrelay.NewHandler(handlerConfig, donConfig, don, hf.lggr, clockwork.NewRealClock()) + return confidentialrelay.NewHandler(handlerConfig, donConfig, don, hf.lggr, clockwork.NewRealClock(), hf.lf) default: return nil, fmt.Errorf("unsupported handler type %s", handlerType) } diff --git a/core/services/gateway/handlers/confidentialrelay/handler.go b/core/services/gateway/handlers/confidentialrelay/handler.go index e5811fbadd3..294a960d77e 100644 --- a/core/services/gateway/handlers/confidentialrelay/handler.go +++ b/core/services/gateway/handlers/confidentialrelay/handler.go @@ -18,8 +18,9 @@ import ( relaytypes "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/actions/confidentialrelay" jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/ratelimit" "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/api" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" gwhandlers "github.com/smartcontractkit/chainlink/v2/core/services/gateway/handlers" @@ -112,8 +113,7 @@ type relayAggregator interface { } type Config struct { - NodeRateLimiter ratelimit.RateLimiterConfig `json:"nodeRateLimiter"` - RequestTimeoutSec int `json:"requestTimeoutSec"` + RequestTimeoutSec int `json:"requestTimeoutSec"` } type handler struct { @@ -125,8 +125,9 @@ type handler struct { mu sync.RWMutex stopCh services.StopChan - nodeRateLimiter *ratelimit.RateLimiter - requestTimeout time.Duration + globalNodeRateLimiter limits.RateLimiter + perNodeRateLimiters map[string]limits.RateLimiter + requestTimeout time.Duration activeRequests map[string]*activeRequest metrics *metrics @@ -144,7 +145,7 @@ func (h *handler) Name() string { return h.lggr.Name() } -func NewHandler(methodConfig json.RawMessage, donConfig *config.DONConfig, don gwhandlers.DON, lggr logger.Logger, clock clockwork.Clock) (*handler, error) { +func NewHandler(methodConfig json.RawMessage, donConfig *config.DONConfig, don gwhandlers.DON, lggr logger.Logger, clock clockwork.Clock, limitsFactory limits.Factory) (*handler, error) { var cfg Config if err := json.Unmarshal(methodConfig, &cfg); err != nil { return nil, fmt.Errorf("failed to unmarshal method config: %w", err) @@ -154,9 +155,18 @@ func NewHandler(methodConfig json.RawMessage, donConfig *config.DONConfig, don g cfg.RequestTimeoutSec = 30 } - nodeRateLimiter, err := ratelimit.NewRateLimiter(cfg.NodeRateLimiter) + globalNodeRateLimiter, err := limitsFactory.MakeRateLimiter(cresettings.Default.GatewayConfidentialRelayGlobalRate) if err != nil { - return nil, fmt.Errorf("failed to create node rate limiter: %w", err) + return nil, fmt.Errorf("failed to create global node rate limiter: %w", err) + } + + perNodeRateLimiters := make(map[string]limits.RateLimiter, len(donConfig.Members)) + for _, member := range donConfig.Members { + rl, makeErr := limitsFactory.MakeRateLimiter(cresettings.Default.GatewayConfidentialRelayPerNodeRate) + if makeErr != nil { + return nil, fmt.Errorf("failed to create per-node rate limiter for %s: %w", member.Address, makeErr) + } + perNodeRateLimiters[member.Address] = rl } metrics, err := newMetrics() @@ -165,17 +175,18 @@ func NewHandler(methodConfig json.RawMessage, donConfig *config.DONConfig, don g } return &handler{ - donConfig: donConfig, - don: don, - lggr: logger.Named(lggr, "ConfidentialRelayHandler:"+donConfig.DonId), - requestTimeout: time.Duration(cfg.RequestTimeoutSec) * time.Second, - nodeRateLimiter: nodeRateLimiter, - activeRequests: make(map[string]*activeRequest), - mu: sync.RWMutex{}, - stopCh: make(services.StopChan), - metrics: metrics, - aggregator: &aggregator{}, - clock: clock, + donConfig: donConfig, + don: don, + lggr: logger.Named(lggr, "ConfidentialRelayHandler:"+donConfig.DonId), + requestTimeout: time.Duration(cfg.RequestTimeoutSec) * time.Second, + globalNodeRateLimiter: globalNodeRateLimiter, + perNodeRateLimiters: perNodeRateLimiters, + activeRequests: make(map[string]*activeRequest), + mu: sync.RWMutex{}, + stopCh: make(services.StopChan), + metrics: metrics, + aggregator: &aggregator{}, + clock: clock, }, nil } @@ -204,7 +215,14 @@ func (h *handler) Close() error { return h.StopOnce("ConfidentialRelayHandler", func() error { h.lggr.Info("closing confidential relay handler") close(h.stopCh) - return nil + var err error + if h.globalNodeRateLimiter != nil { + err = errors.Join(err, h.globalNodeRateLimiter.Close()) + } + for _, rl := range h.perNodeRateLimiters { + err = errors.Join(err, rl.Close()) + } + return err }) } @@ -283,10 +301,18 @@ func (h *handler) HandleNodeMessage(ctx context.Context, resp *jsonrpc.Response[ l := logger.With(h.lggr, "method", resp.Method, "requestID", resp.ID, "nodeAddr", nodeAddr) l.Debugw("handling node response") - if !h.nodeRateLimiter.Allow(nodeAddr) { + nodeRateLimiter, ok := h.perNodeRateLimiters[nodeAddr] + if !ok { + return fmt.Errorf("received message from unexpected node %s", nodeAddr) + } + if !nodeRateLimiter.Allow(ctx) { l.Debugw("node is rate limited", "nodeAddr", nodeAddr) return nil } + if !h.globalNodeRateLimiter.Allow(ctx) { + l.Debug("global relay rate limit exceeded") + return nil + } ar := h.getActiveRequest(resp.ID) if ar == nil { @@ -294,8 +320,8 @@ func (h *handler) HandleNodeMessage(ctx context.Context, resp *jsonrpc.Response[ return nil } - ok := ar.addResponseForNode(nodeAddr, resp) - if !ok { + added := ar.addResponseForNode(nodeAddr, resp) + if !added { l.Errorw("duplicate response from node, ignoring", "nodeAddr", nodeAddr) return nil } diff --git a/core/services/gateway/handlers/confidentialrelay/handler_test.go b/core/services/gateway/handlers/confidentialrelay/handler_test.go index 7d3f3056d65..c3c4ba20690 100644 --- a/core/services/gateway/handlers/confidentialrelay/handler_test.go +++ b/core/services/gateway/handlers/confidentialrelay/handler_test.go @@ -14,10 +14,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "golang.org/x/time/rate" jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/ratelimit" + "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/api" "github.com/smartcontractkit/chainlink/v2/core/services/gateway/config" @@ -82,18 +84,13 @@ func setupHandler(t *testing.T, numNodes int) (*handler, *common.Callback, *mock } handlerConfig := Config{ RequestTimeoutSec: 30, - NodeRateLimiter: ratelimit.RateLimiterConfig{ - GlobalRPS: 100, - GlobalBurst: 100, - PerSenderRPS: 10, - PerSenderBurst: 10, - }, } methodConfig, err := json.Marshal(handlerConfig) require.NoError(t, err) clock := clockwork.NewFakeClock() - h, err := NewHandler(methodConfig, donConfig, don, lggr, clock) + limitsFactory := limits.Factory{Settings: cresettings.DefaultGetter, Logger: lggr} + h, err := NewHandler(methodConfig, donConfig, don, lggr, clock, limitsFactory) require.NoError(t, err) h.aggregator = &mockAggregator{} cb := common.NewCallback() @@ -394,12 +391,6 @@ func TestConfidentialRelayHandler_DuplicateRequestID(t *testing.T) { func TestConfidentialRelayHandler_RateLimitedNode(t *testing.T) { handlerConfig := Config{ RequestTimeoutSec: 30, - NodeRateLimiter: ratelimit.RateLimiterConfig{ - GlobalRPS: 100, - GlobalBurst: 100, - PerSenderRPS: 0.001, // Effectively zero - PerSenderBurst: 1, - }, } methodConfig, err := json.Marshal(handlerConfig) require.NoError(t, err) @@ -412,9 +403,12 @@ func TestConfidentialRelayHandler_RateLimitedNode(t *testing.T) { Members: []config.NodeConfig{nodeOne}, } clock := clockwork.NewFakeClock() - h, err := NewHandler(methodConfig, donConfig, don, lggr, clock) + limitsFactory := limits.Factory{Settings: cresettings.DefaultGetter, Logger: lggr} + h, err := NewHandler(methodConfig, donConfig, don, lggr, clock, limitsFactory) require.NoError(t, err) h.aggregator = &respondingMockAggregator{} + h.globalNodeRateLimiter = limits.GlobalRateLimiter(rate.Limit(100), 100) + h.perNodeRateLimiters[nodeOne.Address] = limits.GlobalRateLimiter(rate.Limit(0.001), 1) don.On("SendToNode", mock.Anything, mock.Anything, mock.Anything).Return(nil) @@ -541,16 +535,11 @@ func TestConfidentialRelayHandler_FanOutToNodes_IsConcurrent(t *testing.T) { methodConfig, err := json.Marshal(Config{ RequestTimeoutSec: 30, - NodeRateLimiter: ratelimit.RateLimiterConfig{ - GlobalRPS: 100, - GlobalBurst: 100, - PerSenderRPS: 10, - PerSenderBurst: 10, - }, }) require.NoError(t, err) - h, err := NewHandler(methodConfig, donConfig, don, lggr, clockwork.NewFakeClock()) + limitsFactory := limits.Factory{Settings: cresettings.DefaultGetter, Logger: lggr} + h, err := NewHandler(methodConfig, donConfig, don, lggr, clockwork.NewFakeClock(), limitsFactory) require.NoError(t, err) cb := common.NewCallback() diff --git a/deployment/cre/jobs/pkg/gateway_job.go b/deployment/cre/jobs/pkg/gateway_job.go index cca265a0bb6..3bc6828cac1 100644 --- a/deployment/cre/jobs/pkg/gateway_job.go +++ b/deployment/cre/jobs/pkg/gateway_job.go @@ -457,8 +457,7 @@ func newDefaultHTTPCapabilitiesHandler() handler { } type confidentialRelayHandlerConfig struct { - RequestTimeoutSec int `toml:"requestTimeoutSec"` - NodeRateLimiter nodeRateLimiterConfig `toml:"NodeRateLimiter"` + RequestTimeoutSec int `toml:"requestTimeoutSec"` } func newDefaultConfidentialRelayHandler(requestTimeoutSec int) handler { @@ -467,12 +466,6 @@ func newDefaultConfidentialRelayHandler(requestTimeoutSec int) handler { ServiceName: ServiceNameConfidential, Config: confidentialRelayHandlerConfig{ RequestTimeoutSec: requestTimeoutSec, - NodeRateLimiter: nodeRateLimiterConfig{ - GlobalBurst: 10, - GlobalRPS: 50, - PerSenderBurst: 10, - PerSenderRPS: 10, - }, }, } } diff --git a/deployment/go.mod b/deployment/go.mod index aec6190e944..eb118a9b770 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -44,7 +44,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260224214816-cb23ec38649f github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260310183131-8d0f0e383288 github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260317175207-e9ff89561326 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260406055916-9aa6b6c0ae81 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260402120824-48154c0c65a6 github.com/smartcontractkit/chainlink-common/keystore v1.0.2 github.com/smartcontractkit/chainlink-deployments-framework v0.86.3 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260330133421-5151ea0c3b05 diff --git a/deployment/go.sum b/deployment/go.sum index c151e70fe9b..cfb711c5b3f 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1387,8 +1387,8 @@ github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260317175207-e9ff github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260317175207-e9ff89561326/go.mod h1:P0/tjeeIIxfsBupk5MneRjq5uI9mj+ZQpMpYnFla6WM= github.com/smartcontractkit/chainlink-ccv v0.0.0-20260324000441-d4cfddc9f7d2 h1:5HdH/A6yn8INZAltYDLb7UkUi5IKemhJzJkDW4Bgxyg= github.com/smartcontractkit/chainlink-ccv v0.0.0-20260324000441-d4cfddc9f7d2/go.mod h1:wDHq2E0KwUWG0lQ9f5frW1a7CKVW17MJLPuvKmtSRDg= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260406055916-9aa6b6c0ae81 h1:qBQxh/dndRMJX41xWEihr8FkvPL21luWwTFn/0Nl3RU= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260406055916-9aa6b6c0ae81/go.mod h1:Ob7ZRLEvPkDwGUjKdDIiHy0Mxu4+UG6oMBkR7Jv/U6o= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260402120824-48154c0c65a6 h1:oaXslIvcy5HD3zkWhx3nu8vRGdWGedYJ+XCsBD8mYkA= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260402120824-48154c0c65a6/go.mod h1:Ea94/OgfFPRTByGO2Qo+uZ7/4sWhwE2HKu7dDwdojME= github.com/smartcontractkit/chainlink-common/keystore v1.0.2 h1:AWisx4JT3QV8tcgh6J5NCrex+wAgTYpWyHsyNPSXzsQ= github.com/smartcontractkit/chainlink-common/keystore v1.0.2/go.mod h1:rSkIHdomyak3YnUtXLenl6poIq8q0V3UZPiiyYqPdGA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10 h1:FJAFgXS9oqASnkS03RE1HQwYQQxrO4l46O5JSzxqLgg= From 8023f554ee8b11e29e3856da7435b49a54e5a392 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 8 Apr 2026 15:12:26 +0200 Subject: [PATCH 41/67] Rebase cw-5 on updated cw-4 and fix private plugin build --- core/services/gateway/handler_factory.go | 3 +-- go.mod | 2 +- go.sum | 4 ++-- plugins/plugins.private.yaml | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/core/services/gateway/handler_factory.go b/core/services/gateway/handler_factory.go index af62f1d0e87..38a1ca0e299 100644 --- a/core/services/gateway/handler_factory.go +++ b/core/services/gateway/handler_factory.go @@ -86,8 +86,7 @@ func (hf *handlerFactory) NewHandler( case HTTPCapabilityType: return v2.NewGatewayHandler(handlerConfig, donConfig, don, hf.httpClient, hf.lggr, hf.lf) case VaultHandlerType: - requestAuthorizer := vaultcap.NewRequestAuthorizer(hf.lggr, hf.workflowRegistrySyncer) - return vault.NewHandler(handlerConfig, donConfig, don, hf.capabilitiesRegistry, requestAuthorizer, hf.lggr, clockwork.NewRealClock(), hf.lf) + return vault.NewHandler(handlerConfig, donConfig, don, hf.capabilitiesRegistry, hf.workflowRegistrySyncer, hf.lggr, clockwork.NewRealClock(), hf.lf) case ConfidentialRelayHandlerType: return confidentialrelay.NewHandler(handlerConfig, donConfig, don, hf.lggr, clockwork.NewRealClock(), hf.lf) default: diff --git a/go.mod b/go.mod index a37e892c17d..fe53621739e 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260224214816-cb23ec38649f github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20250912190424-fd2e35d7deb5 github.com/smartcontractkit/chainlink-ccv v0.0.0-20260324000441-d4cfddc9f7d2 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260402120824-48154c0c65a6 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260406055916-9aa6b6c0ae81 github.com/smartcontractkit/chainlink-common/keystore v1.0.2 github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10 github.com/smartcontractkit/chainlink-data-streams v0.1.13 diff --git a/go.sum b/go.sum index 71c35208109..4c02975ac41 100644 --- a/go.sum +++ b/go.sum @@ -1235,8 +1235,8 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20250 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20250912190424-fd2e35d7deb5/go.mod h1:xtZNi6pOKdC3sLvokDvXOhgHzT+cyBqH/gWwvxTxqrg= github.com/smartcontractkit/chainlink-ccv v0.0.0-20260324000441-d4cfddc9f7d2 h1:5HdH/A6yn8INZAltYDLb7UkUi5IKemhJzJkDW4Bgxyg= github.com/smartcontractkit/chainlink-ccv v0.0.0-20260324000441-d4cfddc9f7d2/go.mod h1:wDHq2E0KwUWG0lQ9f5frW1a7CKVW17MJLPuvKmtSRDg= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260402120824-48154c0c65a6 h1:oaXslIvcy5HD3zkWhx3nu8vRGdWGedYJ+XCsBD8mYkA= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260402120824-48154c0c65a6/go.mod h1:Ea94/OgfFPRTByGO2Qo+uZ7/4sWhwE2HKu7dDwdojME= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260406055916-9aa6b6c0ae81 h1:qBQxh/dndRMJX41xWEihr8FkvPL21luWwTFn/0Nl3RU= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260406055916-9aa6b6c0ae81/go.mod h1:Ob7ZRLEvPkDwGUjKdDIiHy0Mxu4+UG6oMBkR7Jv/U6o= github.com/smartcontractkit/chainlink-common/keystore v1.0.2 h1:AWisx4JT3QV8tcgh6J5NCrex+wAgTYpWyHsyNPSXzsQ= github.com/smartcontractkit/chainlink-common/keystore v1.0.2/go.mod h1:rSkIHdomyak3YnUtXLenl6poIq8q0V3UZPiiyYqPdGA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10 h1:FJAFgXS9oqASnkS03RE1HQwYQQxrO4l46O5JSzxqLgg= diff --git a/plugins/plugins.private.yaml b/plugins/plugins.private.yaml index 7276869a68f..a9c5d0f65a3 100644 --- a/plugins/plugins.private.yaml +++ b/plugins/plugins.private.yaml @@ -56,9 +56,9 @@ plugins: installPath: "." confidential-http: - moduleURI: "github.com/smartcontractkit/confidential-compute/enclave/apps/confidential-http/capability" - gitRef: "526a251" + gitRef: "fc4c85318b9039627aaf8ecbf2b89966140b4af3" installPath: "./cmd/confidential-http" confidential-workflows: - moduleURI: "github.com/smartcontractkit/confidential-compute/enclave/apps/confidential-workflows/capability" - gitRef: "526a251" + gitRef: "fc4c85318b9039627aaf8ecbf2b89966140b4af3" installPath: "./cmd/confidential-workflows" From 82dd8379a84807e721d54a1b7ccb6441db3fff7a Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 12:40:11 +0100 Subject: [PATCH 42/67] resolve go.mod conflicts --- core/services/job/models.go | 1 + .../conversions/conversions.go | 4 + core/services/workflows/artifacts/v2/orm.go | 9 +- core/services/workflows/syncer/fetcher.go | 18 +- core/services/workflows/syncer/v2/fetcher.go | 18 +- core/services/workflows/syncer/v2/handler.go | 119 +++++- .../workflows/syncer/v2/handler_test.go | 258 +++++++++++++ .../workflows/v2/confidential_module.go | 188 ++++++++++ .../workflows/v2/confidential_module_test.go | 343 ++++++++++++++++++ .../0295_add_workflow_attributes_column.sql | 5 + 10 files changed, 938 insertions(+), 25 deletions(-) create mode 100644 core/services/workflows/v2/confidential_module.go create mode 100644 core/services/workflows/v2/confidential_module_test.go create mode 100644 core/store/migrate/migrations/0295_add_workflow_attributes_column.sql diff --git a/core/services/job/models.go b/core/services/job/models.go index 20e7da2c643..2d257a33ca5 100644 --- a/core/services/job/models.go +++ b/core/services/job/models.go @@ -930,6 +930,7 @@ type WorkflowSpec struct { CreatedAt time.Time `toml:"-"` UpdatedAt time.Time `toml:"-"` SpecType WorkflowSpecType `toml:"spec_type" db:"spec_type"` + Attributes []byte `db:"attributes"` sdkWorkflow *sdk.WorkflowSpec rawSpec []byte config []byte diff --git a/core/services/standardcapabilities/conversions/conversions.go b/core/services/standardcapabilities/conversions/conversions.go index b9a233243af..d623fa9a409 100644 --- a/core/services/standardcapabilities/conversions/conversions.go +++ b/core/services/standardcapabilities/conversions/conversions.go @@ -50,6 +50,8 @@ func GetCapabilityIDFromCommand(command string, config string) string { return "http-trigger@1.0.0-alpha" case "http_action": return "http-actions@1.0.0-alpha" // plural "actions" + case "mock": + return "mock@1.0.0" default: return "" } @@ -71,6 +73,8 @@ func GetCommandFromCapabilityID(capabilityID string) string { return "http_trigger" case strings.HasPrefix(capabilityID, "http-actions"): return "http_action" + case strings.HasPrefix(capabilityID, "mock"): + return "mock" default: return "" } diff --git a/core/services/workflows/artifacts/v2/orm.go b/core/services/workflows/artifacts/v2/orm.go index 7b37afda16b..6d8887dbe06 100644 --- a/core/services/workflows/artifacts/v2/orm.go +++ b/core/services/workflows/artifacts/v2/orm.go @@ -63,7 +63,8 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) config_url, created_at, updated_at, - spec_type + spec_type, + attributes ) VALUES ( :workflow, :config, @@ -76,7 +77,8 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) :config_url, :created_at, :updated_at, - :spec_type + :spec_type, + :attributes ) ON CONFLICT (workflow_id) DO UPDATE SET workflow = EXCLUDED.workflow, @@ -89,7 +91,8 @@ func (orm *orm) UpsertWorkflowSpec(ctx context.Context, spec *job.WorkflowSpec) config_url = EXCLUDED.config_url, created_at = EXCLUDED.created_at, updated_at = EXCLUDED.updated_at, - spec_type = EXCLUDED.spec_type + spec_type = EXCLUDED.spec_type, + attributes = EXCLUDED.attributes RETURNING id ` diff --git a/core/services/workflows/syncer/fetcher.go b/core/services/workflows/syncer/fetcher.go index c959b6b4da7..33ba4b10a2e 100644 --- a/core/services/workflows/syncer/fetcher.go +++ b/core/services/workflows/syncer/fetcher.go @@ -177,12 +177,20 @@ func newFileFetcher(basePath string, lggr logger.Logger) types.FetcherFunc { if err != nil { return nil, fmt.Errorf("invalid URL: %w", err) } - fullPath := filepath.Clean(u.Path) - // ensure that the incoming request URL is either relative or absolute but within the basePath - if !filepath.IsAbs(fullPath) { - // If it's not absolute, we assume it's relative to the basePath - fullPath = filepath.Join(basePath, fullPath) + var fullPath string + if u.Scheme == "http" || u.Scheme == "https" { + // For HTTP(S) URLs, extract just the filename and resolve against basePath. + // This supports confidential workflows where the on-chain URL must be HTTP + // (so the enclave can fetch the binary), but the syncer reads from the local filesystem. + fullPath = filepath.Join(basePath, filepath.Base(u.Path)) + } else { + fullPath = filepath.Clean(u.Path) + // ensure that the incoming request URL is either relative or absolute but within the basePath + if !filepath.IsAbs(fullPath) { + // If it's not absolute, we assume it's relative to the basePath + fullPath = filepath.Join(basePath, fullPath) + } } if !strings.HasPrefix(fullPath, basePath) { return nil, fmt.Errorf("request URL %s is not within the basePath %s", fullPath, basePath) diff --git a/core/services/workflows/syncer/v2/fetcher.go b/core/services/workflows/syncer/v2/fetcher.go index e86f998f6bb..4c79fffbe24 100644 --- a/core/services/workflows/syncer/v2/fetcher.go +++ b/core/services/workflows/syncer/v2/fetcher.go @@ -211,12 +211,20 @@ func newFileFetcher(basePath string, lggr logger.Logger) types.FetcherFunc { if err != nil { return nil, fmt.Errorf("invalid URL: %w", err) } - fullPath := filepath.Clean(u.Path) - // ensure that the incoming request URL is either relative or absolute but within the basePath - if !filepath.IsAbs(fullPath) { - // If it's not absolute, we assume it's relative to the basePath - fullPath = filepath.Join(basePath, fullPath) + var fullPath string + if u.Scheme == "http" || u.Scheme == "https" { + // For HTTP(S) URLs, extract just the filename and resolve against basePath. + // This supports confidential workflows where the on-chain URL must be HTTP + // (so the enclave can fetch the binary), but the syncer reads from the local filesystem. + fullPath = filepath.Join(basePath, filepath.Base(u.Path)) + } else { + fullPath = filepath.Clean(u.Path) + // ensure that the incoming request URL is either relative or absolute but within the basePath + if !filepath.IsAbs(fullPath) { + // If it's not absolute, we assume it's relative to the basePath + fullPath = filepath.Join(basePath, fullPath) + } } if !strings.HasPrefix(fullPath, basePath) { return nil, fmt.Errorf("request URL %s is not within the basePath %s", fullPath, basePath) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 6dd3dbf9f06..b298db91722 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -558,6 +558,7 @@ func (h *eventHandler) createWorkflowSpec(ctx context.Context, payload WorkflowR SpecType: job.WASMFile, BinaryURL: payload.BinaryURL, ConfigURL: payload.ConfigURL, + Attributes: payload.Attributes, } if _, err = h.workflowArtifactsStore.UpsertWorkflowSpec(ctx, entry); err != nil { @@ -819,6 +820,15 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp return fmt.Errorf("invalid workflow name: %w", err) } + confidential, err := v2.IsConfidential(spec.Attributes) + if err != nil { + return fmt.Errorf("failed to parse workflow attributes: %w", err) + } + if confidential { + h.lggr.Infow("routing workflow to confidential execution", "workflowID", spec.WorkflowID) + return h.tryConfidentialEngineCreate(ctx, spec, wid, workflowName, decodedBinary, source) + } + // Create a channel to receive the initialization result. // This allows us to wait for the engine to complete initialization (including trigger subscriptions) // before emitting the workflowActivated event, ensuring the event accurately reflects deployment status. @@ -841,7 +851,21 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp return fmt.Errorf("failed to create workflow engine: %w", err) } - if err = engine.Start(ctx); err != nil { + return h.startAndRegisterEngine(ctx, engine, initDone, wid, spec.WorkflowID, source) +} + +// startAndRegisterEngine starts a workflow engine, waits for initialization to +// complete, and registers it in the engine registry. Used by both the normal +// and confidential engine creation paths. +func (h *eventHandler) startAndRegisterEngine( + ctx context.Context, + engine services.Service, + initDone <-chan error, + wid types.WorkflowID, + workflowID string, + source string, +) error { + if err := engine.Start(ctx); err != nil { return fmt.Errorf("failed to start workflow engine: %w", err) } @@ -849,32 +873,25 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp // This ensures we don't emit workflowActivated events before the engine initializes successfully. select { case <-ctx.Done(): - // Context cancelled while waiting for initialization if closeErr := engine.Close(); closeErr != nil { - h.lggr.Errorw("failed to close engine after context cancellation", "error", closeErr, "workflowID", spec.WorkflowID) + h.lggr.Errorw("failed to close engine after context cancellation", "error", closeErr, "workflowID", workflowID) } return fmt.Errorf("context cancelled while waiting for engine initialization: %w", ctx.Err()) case initErr := <-initDone: if initErr != nil { - // Engine initialization failed (e.g., trigger subscription failed) // TODO (cre-1482) add logic to mark a deployment as failed to avoid churn. - // Currently, failed deployments will be retried on each poll cycle (with exponential backoff). - // If the failure is due to user error (e.g., invalid trigger config), this causes unnecessary retries. - // Consider marking the workflow spec as "failed" in the database and requiring workflow redeployment. if closeErr := engine.Close(); closeErr != nil { - h.lggr.Errorw("failed to close engine after initialization failure", "error", closeErr, "workflowID", spec.WorkflowID) + h.lggr.Errorw("failed to close engine after initialization failure", "error", closeErr, "workflowID", workflowID) } return fmt.Errorf("engine initialization failed: %w", initErr) } } - // Engine is fully initialized, add to registry with source tracking if err := h.engineRegistry.Add(wid, source, engine); err != nil { if closeErr := engine.Close(); closeErr != nil { return fmt.Errorf("failed to close workflow engine: %w during invariant violation: %w", closeErr, err) } - // Check for WorkflowID collision across sources if errors.Is(err, ErrAlreadyExists) { existingEntry, found := h.engineRegistry.Get(wid) if found { @@ -886,13 +903,91 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp } } - // This shouldn't happen because we call the handler serially and - // check for running engines above, see the call to engineRegistry.Contains. return fmt.Errorf("invariant violation: %w", err) } return nil } +// tryConfidentialEngineCreate creates a V2 engine backed by a ConfidentialModule +// instead of a local WASM module. The ConfidentialModule delegates execution to +// the confidential-workflows capability which runs the WASM inside a TEE. +func (h *eventHandler) tryConfidentialEngineCreate( + ctx context.Context, + spec *job.WorkflowSpec, + wid types.WorkflowID, + workflowName types.WorkflowName, + decodedBinary []byte, + source string, +) error { + attrs, err := v2.ParseWorkflowAttributes(spec.Attributes) + if err != nil { + return fmt.Errorf("failed to parse workflow attributes: %w", err) + } + + binaryHash := v2.ComputeBinaryHash(decodedBinary) + + lggr := logger.Named(h.lggr, "WorkflowEngine.ConfidentialModule") + lggr = logger.With(lggr, "workflowID", spec.WorkflowID, "workflowName", spec.WorkflowName, "workflowOwner", spec.WorkflowOwner) + + module := v2.NewConfidentialModule( + h.capRegistry, + spec.BinaryURL, + binaryHash, + spec.WorkflowID, + spec.WorkflowOwner, + workflowName.String(), + spec.WorkflowTag, + attrs.VaultDonSecrets, + lggr, + ) + + initDone := make(chan error, 1) + + cfg := &v2.EngineConfig{ + Lggr: h.lggr, + Module: module, + WorkflowConfig: []byte(spec.Config), + CapRegistry: h.capRegistry, + DonSubscriber: h.workflowDonSubscriber, + UseLocalTimeProvider: h.useLocalTimeProvider, + DonTimeStore: h.donTimeStore, + ExecutionsStore: h.workflowStore, + WorkflowID: spec.WorkflowID, + WorkflowOwner: spec.WorkflowOwner, + WorkflowName: workflowName, + WorkflowTag: spec.WorkflowTag, + WorkflowEncryptionKey: h.workflowEncryptionKey, + + LocalLimits: v2.EngineLimits{}, + LocalLimiters: h.engineLimiters, + GlobalExecutionConcurrencyLimiter: h.workflowLimits, + + BeholderEmitter: h.emitter, + BillingClient: h.billingClient, + + WorkflowRegistryAddress: h.workflowRegistryAddress, + WorkflowRegistryChainSelector: h.workflowRegistryChainSelector, + OrgResolver: h.orgResolver, + SecretsFetcher: h.secretsFetcher, + FeatureFlags: h.featureFlags, + } + + existingHook := cfg.Hooks.OnInitialized + cfg.Hooks.OnInitialized = func(err error) { + initDone <- err + if existingHook != nil { + existingHook(err) + } + } + + engine, err := v2.NewEngine(cfg) + if err != nil { + return fmt.Errorf("failed to create confidential workflow engine: %w", err) + } + + return h.startAndRegisterEngine(ctx, engine, initDone, wid, spec.WorkflowID, source) +} + // logCustMsg emits a custom message to the external sink and logs an error if that fails. func logCustMsg(ctx context.Context, cma custmsg.MessageEmitter, msg string, log logger.Logger) { err := cma.Emit(ctx, msg) diff --git a/core/services/workflows/syncer/v2/handler_test.go b/core/services/workflows/syncer/v2/handler_test.go index ad7b80d06b9..775f059613e 100644 --- a/core/services/workflows/syncer/v2/handler_test.go +++ b/core/services/workflows/syncer/v2/handler_test.go @@ -618,6 +618,264 @@ func Test_workflowRegisteredHandler(t *testing.T) { } } +func Test_workflowRegisteredHandler_confidentialRouting(t *testing.T) { + t.Run("confidential workflow bypasses engine factory and routes to confidential path", func(t *testing.T) { + var ( + ctx = testutils.Context(t) + lggr = logger.TestLogger(t) + lf = limits.Factory{Logger: lggr} + db = pgtest.NewSqlxDB(t) + orm = artifacts.NewWorkflowRegistryDS(db, lggr) + emitter = custmsg.NewLabeler() + + binary = wasmtest.CreateTestBinary(binaryCmd, true, t) + encodedBinary = []byte(base64.StdEncoding.EncodeToString(binary)) + config = []byte("") + wfOwner = []byte("0xOwner") + workflowEncryptionKey = workflowkey.MustNewXXXTestingOnly(big.NewInt(1)) + ) + + giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, "workflow-name", binary, config, "") + require.NoError(t, err) + wfIDString := hex.EncodeToString(giveWFID[:]) + + binaryURL := "http://example.com/" + wfIDString + "/binary" + configURL := "http://example.com/" + wfIDString + "/config" + signedURLParameter := "?auth=abc123" + signedBinaryURL := binaryURL + signedURLParameter + signedConfigURL := configURL + signedURLParameter + + fetcher := newMockFetcher(map[string]mockFetchResp{ + wfIDString + "-ARTIFACT_TYPE_BINARY": {Body: []byte(signedBinaryURL), Err: nil}, + wfIDString + "-ARTIFACT_TYPE_CONFIG": {Body: []byte(signedConfigURL), Err: nil}, + signedBinaryURL: {Body: encodedBinary, Err: nil}, + signedConfigURL: {Body: config, Err: nil}, + }) + artifactStore, err := artifacts.NewStore(lggr, orm, fetcher.FetcherFunc(), fetcher.RetrieverFunc(), clockwork.NewFakeClock(), workflowkey.Key{}, custmsg.NewLabeler(), lf, artifacts.WithConfig(artifacts.StoreConfig{ + ArtifactStorageHost: "example.com", + })) + require.NoError(t, err) + + er := NewEngineRegistry() + + // Track whether the engine factory is called. The confidential path + // should bypass it entirely. + factoryCalled := false + trackingFactory := func(ctx context.Context, wfid string, owner string, name types.WorkflowName, tag string, config []byte, binary []byte, initDone chan<- error) (services.Service, error) { + factoryCalled = true + if initDone != nil { + initDone <- nil + } + return &mockEngine{}, nil + } + + wfStore := store.NewInMemoryStore(lggr, clockwork.NewFakeClock()) + registry := capabilities.NewRegistry(lggr) + registry.SetLocalRegistry(&capabilities.TestMetadataRegistry{}) + limiters, err := v2.NewLimiters(lf, nil) + require.NoError(t, err) + rl, err := ratelimiter.NewRateLimiter(rlConfig) + require.NoError(t, err) + workflowLimits, err := syncerlimiter.NewWorkflowLimits(lggr, syncerlimiter.Config{Global: 200, PerOwner: 200}, lf) + require.NoError(t, err) + + h, err := NewEventHandler(lggr, wfStore, nil, true, registry, er, emitter, limiters, nil, rl, workflowLimits, artifactStore, workflowEncryptionKey, &testDonNotifier{}, + WithEngineRegistry(er), + WithEngineFactoryFn(trackingFactory), + ) + require.NoError(t, err) + servicetest.Run(t, h) + + event := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: giveWFID, + WorkflowOwner: wfOwner, + WorkflowName: "workflow-name", + WorkflowTag: "workflow-tag", + BinaryURL: binaryURL, + ConfigURL: configURL, + Attributes: []byte(`{"confidential":true,"vault_don_secrets":[{"key":"API_KEY"}]}`), + } + + ctx = contexts.WithCRE(ctx, contexts.CRE{Owner: hex.EncodeToString(wfOwner), Workflow: wfIDString}) + err = h.workflowRegisteredEvent(ctx, event) + + // The confidential path creates a real v2.Engine. With test data + // (non-hex owner), engine creation fails. The error comes from the + // confidential path, proving routing worked correctly. + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to create confidential workflow engine") + + // The engine factory must NOT have been called; the confidential path + // bypasses it. + assert.False(t, factoryCalled, "engine factory should not be called for confidential workflows") + + // The engine should NOT be in the registry since init failed. + _, ok := er.Get(giveWFID) + assert.False(t, ok, "engine should not be registered after failed init") + }) + + t.Run("non-confidential workflow uses engine factory", func(t *testing.T) { + var ( + ctx = testutils.Context(t) + lggr = logger.TestLogger(t) + lf = limits.Factory{Logger: lggr} + db = pgtest.NewSqlxDB(t) + orm = artifacts.NewWorkflowRegistryDS(db, lggr) + emitter = custmsg.NewLabeler() + + binary = wasmtest.CreateTestBinary(binaryCmd, true, t) + encodedBinary = []byte(base64.StdEncoding.EncodeToString(binary)) + config = []byte("") + wfOwner = []byte("0xOwner") + workflowEncryptionKey = workflowkey.MustNewXXXTestingOnly(big.NewInt(1)) + ) + + giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, "workflow-name", binary, config, "") + require.NoError(t, err) + wfIDString := hex.EncodeToString(giveWFID[:]) + + binaryURL := "http://example.com/" + wfIDString + "/binary" + configURL := "http://example.com/" + wfIDString + "/config" + signedURLParameter := "?auth=abc123" + signedBinaryURL := binaryURL + signedURLParameter + signedConfigURL := configURL + signedURLParameter + + fetcher := newMockFetcher(map[string]mockFetchResp{ + wfIDString + "-ARTIFACT_TYPE_BINARY": {Body: []byte(signedBinaryURL), Err: nil}, + wfIDString + "-ARTIFACT_TYPE_CONFIG": {Body: []byte(signedConfigURL), Err: nil}, + signedBinaryURL: {Body: encodedBinary, Err: nil}, + signedConfigURL: {Body: config, Err: nil}, + }) + artifactStore, err := artifacts.NewStore(lggr, orm, fetcher.FetcherFunc(), fetcher.RetrieverFunc(), clockwork.NewFakeClock(), workflowkey.Key{}, custmsg.NewLabeler(), lf, artifacts.WithConfig(artifacts.StoreConfig{ + ArtifactStorageHost: "example.com", + })) + require.NoError(t, err) + + er := NewEngineRegistry() + + factoryCalled := false + trackingFactory := func(ctx context.Context, wfid string, owner string, name types.WorkflowName, tag string, config []byte, binary []byte, initDone chan<- error) (services.Service, error) { + factoryCalled = true + if initDone != nil { + initDone <- nil + } + return &mockEngine{}, nil + } + + wfStore := store.NewInMemoryStore(lggr, clockwork.NewFakeClock()) + registry := capabilities.NewRegistry(lggr) + registry.SetLocalRegistry(&capabilities.TestMetadataRegistry{}) + limiters, err := v2.NewLimiters(lf, nil) + require.NoError(t, err) + rl, err := ratelimiter.NewRateLimiter(rlConfig) + require.NoError(t, err) + workflowLimits, err := syncerlimiter.NewWorkflowLimits(lggr, syncerlimiter.Config{Global: 200, PerOwner: 200}, lf) + require.NoError(t, err) + + h, err := NewEventHandler(lggr, wfStore, nil, true, registry, er, emitter, limiters, nil, rl, workflowLimits, artifactStore, workflowEncryptionKey, &testDonNotifier{}, + WithEngineRegistry(er), + WithEngineFactoryFn(trackingFactory), + ) + require.NoError(t, err) + servicetest.Run(t, h) + + event := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: giveWFID, + WorkflowOwner: wfOwner, + WorkflowName: "workflow-name", + WorkflowTag: "workflow-tag", + BinaryURL: binaryURL, + ConfigURL: configURL, + // No Attributes, or non-confidential attributes. + } + + ctx = contexts.WithCRE(ctx, contexts.CRE{Owner: hex.EncodeToString(wfOwner), Workflow: wfIDString}) + err = h.workflowRegisteredEvent(ctx, event) + require.NoError(t, err) + + assert.True(t, factoryCalled, "engine factory should be called for non-confidential workflows") + + engine, ok := er.Get(giveWFID) + require.True(t, ok, "engine should be registered") + require.NoError(t, engine.Ready()) + }) + + t.Run("malformed attributes returns error", func(t *testing.T) { + var ( + ctx = testutils.Context(t) + lggr = logger.TestLogger(t) + lf = limits.Factory{Logger: lggr} + db = pgtest.NewSqlxDB(t) + orm = artifacts.NewWorkflowRegistryDS(db, lggr) + emitter = custmsg.NewLabeler() + + binary = wasmtest.CreateTestBinary(binaryCmd, true, t) + encodedBinary = []byte(base64.StdEncoding.EncodeToString(binary)) + config = []byte("") + wfOwner = []byte("0xOwner") + workflowEncryptionKey = workflowkey.MustNewXXXTestingOnly(big.NewInt(1)) + ) + + giveWFID, err := pkgworkflows.GenerateWorkflowID(wfOwner, "workflow-name", binary, config, "") + require.NoError(t, err) + wfIDString := hex.EncodeToString(giveWFID[:]) + + binaryURL := "http://example.com/" + wfIDString + "/binary" + configURL := "http://example.com/" + wfIDString + "/config" + signedURLParameter := "?auth=abc123" + signedBinaryURL := binaryURL + signedURLParameter + signedConfigURL := configURL + signedURLParameter + + fetcher := newMockFetcher(map[string]mockFetchResp{ + wfIDString + "-ARTIFACT_TYPE_BINARY": {Body: []byte(signedBinaryURL), Err: nil}, + wfIDString + "-ARTIFACT_TYPE_CONFIG": {Body: []byte(signedConfigURL), Err: nil}, + signedBinaryURL: {Body: encodedBinary, Err: nil}, + signedConfigURL: {Body: config, Err: nil}, + }) + artifactStore, err := artifacts.NewStore(lggr, orm, fetcher.FetcherFunc(), fetcher.RetrieverFunc(), clockwork.NewFakeClock(), workflowkey.Key{}, custmsg.NewLabeler(), lf, artifacts.WithConfig(artifacts.StoreConfig{ + ArtifactStorageHost: "example.com", + })) + require.NoError(t, err) + + er := NewEngineRegistry() + wfStore := store.NewInMemoryStore(lggr, clockwork.NewFakeClock()) + registry := capabilities.NewRegistry(lggr) + registry.SetLocalRegistry(&capabilities.TestMetadataRegistry{}) + limiters, err := v2.NewLimiters(lf, nil) + require.NoError(t, err) + rl, err := ratelimiter.NewRateLimiter(rlConfig) + require.NoError(t, err) + workflowLimits, err := syncerlimiter.NewWorkflowLimits(lggr, syncerlimiter.Config{Global: 200, PerOwner: 200}, lf) + require.NoError(t, err) + + h, err := NewEventHandler(lggr, wfStore, nil, true, registry, er, emitter, limiters, nil, rl, workflowLimits, artifactStore, workflowEncryptionKey, &testDonNotifier{}, + WithEngineRegistry(er), + WithEngineFactoryFn(mockEngineFactory), + ) + require.NoError(t, err) + servicetest.Run(t, h) + + event := WorkflowRegisteredEvent{ + Status: WorkflowStatusActive, + WorkflowID: giveWFID, + WorkflowOwner: wfOwner, + WorkflowName: "workflow-name", + WorkflowTag: "workflow-tag", + BinaryURL: binaryURL, + ConfigURL: configURL, + Attributes: []byte(`{not valid json`), + } + + ctx = contexts.WithCRE(ctx, contexts.CRE{Owner: hex.EncodeToString(wfOwner), Workflow: wfIDString}) + err = h.workflowRegisteredEvent(ctx, event) + + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse workflow attributes") + }) +} + type testCase struct { Name string BinaryURLFactory func(string) string diff --git a/core/services/workflows/v2/confidential_module.go b/core/services/workflows/v2/confidential_module.go new file mode 100644 index 00000000000..4cc3761e18a --- /dev/null +++ b/core/services/workflows/v2/confidential_module.go @@ -0,0 +1,188 @@ +package v2 + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/types/core" + "github.com/smartcontractkit/chainlink-common/pkg/workflows/wasm/host" + + confworkflowtypes "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/actions/confidentialworkflow" + sdkpb "github.com/smartcontractkit/chainlink-protos/cre/go/sdk" +) + +const confidentialWorkflowsCapabilityID = "confidential-workflows@1.0.0-alpha" + +// WorkflowAttributes is the JSON structure stored in WorkflowSpec.Attributes. +type WorkflowAttributes struct { + Confidential bool `json:"confidential"` + VaultDonSecrets []SecretIdentifier `json:"vault_don_secrets"` +} + +// SecretIdentifier identifies a secret in VaultDON. +type SecretIdentifier struct { + Key string `json:"key"` + Namespace string `json:"namespace,omitempty"` +} + +// ParseWorkflowAttributes parses the Attributes JSON from a WorkflowSpec. +// Returns a zero-value struct if data is nil or empty. +func ParseWorkflowAttributes(data []byte) (WorkflowAttributes, error) { + var attrs WorkflowAttributes + if len(data) == 0 { + return attrs, nil + } + if err := json.Unmarshal(data, &attrs); err != nil { + return attrs, fmt.Errorf("failed to parse workflow attributes: %w", err) + } + return attrs, nil +} + +// IsConfidential returns true if the Attributes JSON has "confidential": true. +// Returns an error if the attributes contain malformed JSON, so callers can +// fail loudly rather than silently falling through to non-confidential execution. +func IsConfidential(data []byte) (bool, error) { + attrs, err := ParseWorkflowAttributes(data) + if err != nil { + return false, err + } + return attrs.Confidential, nil +} + +// ConfidentialModule implements host.ModuleV2 for confidential workflows. +// Instead of running WASM locally, it delegates execution to the +// confidential-workflows capability via the CapabilitiesRegistry. +type ConfidentialModule struct { + capRegistry core.CapabilitiesRegistry + binaryURL string + binaryHash []byte + workflowID string + workflowOwner string + workflowName string + workflowTag string + vaultDonSecrets []SecretIdentifier + lggr logger.Logger +} + +var _ host.ModuleV2 = (*ConfidentialModule)(nil) + +func NewConfidentialModule( + capRegistry core.CapabilitiesRegistry, + binaryURL string, + binaryHash []byte, + workflowID string, + workflowOwner string, + workflowName string, + workflowTag string, + vaultDonSecrets []SecretIdentifier, + lggr logger.Logger, +) *ConfidentialModule { + return &ConfidentialModule{ + capRegistry: capRegistry, + binaryURL: binaryURL, + binaryHash: binaryHash, + workflowID: workflowID, + workflowOwner: workflowOwner, + workflowName: workflowName, + workflowTag: workflowTag, + vaultDonSecrets: vaultDonSecrets, + lggr: lggr, + } +} + +func (m *ConfidentialModule) Start() {} +func (m *ConfidentialModule) Close() {} +func (m *ConfidentialModule) IsLegacyDAG() bool { return false } + +func (m *ConfidentialModule) Execute( + ctx context.Context, + request *sdkpb.ExecuteRequest, + helper host.ExecutionHelper, +) (*sdkpb.ExecutionResult, error) { + execReqBytes, err := proto.Marshal(request) + if err != nil { + return nil, fmt.Errorf("failed to marshal ExecuteRequest: %w", err) + } + + protoSecrets := make([]*confworkflowtypes.SecretIdentifier, len(m.vaultDonSecrets)) + for i, s := range m.vaultDonSecrets { + // VaultDON treats "main" as the default namespace for secrets. + ns := s.Namespace + if ns == "" { + ns = "main" + } + protoSecrets[i] = &confworkflowtypes.SecretIdentifier{ + Key: s.Key, + Namespace: &ns, + } + } + + capInput := &confworkflowtypes.ConfidentialWorkflowRequest{ + VaultDonSecrets: protoSecrets, + Execution: &confworkflowtypes.WorkflowExecution{ + WorkflowId: m.workflowID, + BinaryUrl: m.binaryURL, + BinaryHash: m.binaryHash, + ExecuteRequest: execReqBytes, + Owner: m.workflowOwner, + ExecutionId: helper.GetWorkflowExecutionID(), + }, + } + + payload, err := anypb.New(capInput) + if err != nil { + return nil, fmt.Errorf("failed to marshal capability payload: %w", err) + } + + executable, err := m.capRegistry.GetExecutable(ctx, confidentialWorkflowsCapabilityID) + if err != nil { + return nil, fmt.Errorf("failed to get confidential-workflows capability: %w", err) + } + + capReq := capabilities.CapabilityRequest{ + Payload: payload, + Method: "Execute", + CapabilityId: confidentialWorkflowsCapabilityID, + Metadata: capabilities.RequestMetadata{ + WorkflowID: m.workflowID, + WorkflowOwner: m.workflowOwner, + WorkflowName: m.workflowName, + WorkflowTag: m.workflowTag, + WorkflowExecutionID: helper.GetWorkflowExecutionID(), + }, + } + + capResp, err := executable.Execute(ctx, capReq) + if err != nil { + return nil, fmt.Errorf("confidential-workflows capability execution failed: %w", err) + } + + if capResp.Payload == nil { + return nil, fmt.Errorf("confidential-workflows capability returned nil payload") + } + + var confResp confworkflowtypes.ConfidentialWorkflowResponse + if err := capResp.Payload.UnmarshalTo(&confResp); err != nil { + return nil, fmt.Errorf("failed to unmarshal capability response: %w", err) + } + + var result sdkpb.ExecutionResult + if err := proto.Unmarshal(confResp.ExecutionResult, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal ExecutionResult: %w", err) + } + + return &result, nil +} + +// ComputeBinaryHash returns the SHA-256 hash of the given binary. +func ComputeBinaryHash(binary []byte) []byte { + h := sha256.Sum256(binary) + return h[:] +} diff --git a/core/services/workflows/v2/confidential_module_test.go b/core/services/workflows/v2/confidential_module_test.go new file mode 100644 index 00000000000..87dab426022 --- /dev/null +++ b/core/services/workflows/v2/confidential_module_test.go @@ -0,0 +1,343 @@ +package v2 + +import ( + "context" + "crypto/sha256" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + regmocks "github.com/smartcontractkit/chainlink-common/pkg/types/core/mocks" + "github.com/smartcontractkit/chainlink-common/pkg/workflows/wasm/host" + + confworkflowtypes "github.com/smartcontractkit/chainlink-common/pkg/capabilities/v2/actions/confidentialworkflow" + capmocks "github.com/smartcontractkit/chainlink/v2/core/capabilities/mocks" + "github.com/smartcontractkit/chainlink/v2/core/utils/matches" + + sdkpb "github.com/smartcontractkit/chainlink-protos/cre/go/sdk" + valuespb "github.com/smartcontractkit/chainlink-protos/cre/go/values/pb" +) + +// stubExecutionHelper implements host.ExecutionHelper for testing. +type stubExecutionHelper struct { + executionID string +} + +var _ host.ExecutionHelper = (*stubExecutionHelper)(nil) + +func (s *stubExecutionHelper) CallCapability(context.Context, *sdkpb.CapabilityRequest) (*sdkpb.CapabilityResponse, error) { + return nil, nil +} +func (s *stubExecutionHelper) GetSecrets(context.Context, *sdkpb.GetSecretsRequest) ([]*sdkpb.SecretResponse, error) { + return nil, nil +} +func (s *stubExecutionHelper) GetWorkflowExecutionID() string { return s.executionID } +func (s *stubExecutionHelper) GetNodeTime() time.Time { return time.Time{} } +func (s *stubExecutionHelper) GetDONTime() (time.Time, error) { return time.Time{}, nil } +func (s *stubExecutionHelper) EmitUserLog(string) error { return nil } + +func TestParseWorkflowAttributes(t *testing.T) { + t.Run("valid JSON with all fields", func(t *testing.T) { + data := []byte(`{"confidential":true,"vault_don_secrets":[{"key":"API_KEY"},{"key":"SIGNING_KEY","namespace":"custom-ns"}]}`) + attrs, err := ParseWorkflowAttributes(data) + require.NoError(t, err) + assert.True(t, attrs.Confidential) + require.Len(t, attrs.VaultDonSecrets, 2) + assert.Equal(t, "API_KEY", attrs.VaultDonSecrets[0].Key) + assert.Equal(t, "", attrs.VaultDonSecrets[0].Namespace) + assert.Equal(t, "SIGNING_KEY", attrs.VaultDonSecrets[1].Key) + assert.Equal(t, "custom-ns", attrs.VaultDonSecrets[1].Namespace) + }) + + t.Run("empty data returns zero value", func(t *testing.T) { + attrs, err := ParseWorkflowAttributes(nil) + require.NoError(t, err) + assert.False(t, attrs.Confidential) + assert.Nil(t, attrs.VaultDonSecrets) + + attrs, err = ParseWorkflowAttributes([]byte{}) + require.NoError(t, err) + assert.False(t, attrs.Confidential) + }) + + t.Run("non-confidential workflow", func(t *testing.T) { + data := []byte(`{"confidential":false}`) + attrs, err := ParseWorkflowAttributes(data) + require.NoError(t, err) + assert.False(t, attrs.Confidential) + }) + + t.Run("malformed JSON returns error", func(t *testing.T) { + _, err := ParseWorkflowAttributes([]byte(`{not json}`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse workflow attributes") + }) +} + +func TestIsConfidential(t *testing.T) { + t.Run("returns true for confidential", func(t *testing.T) { + ok, err := IsConfidential([]byte(`{"confidential":true}`)) + require.NoError(t, err) + assert.True(t, ok) + }) + + t.Run("returns false for non-confidential", func(t *testing.T) { + ok, err := IsConfidential([]byte(`{"confidential":false}`)) + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("returns false for empty data", func(t *testing.T) { + ok, err := IsConfidential(nil) + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("returns error for malformed JSON", func(t *testing.T) { + _, err := IsConfidential([]byte(`broken`)) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to parse workflow attributes") + }) +} + +func TestComputeBinaryHash(t *testing.T) { + binary := []byte("hello world") + hash := ComputeBinaryHash(binary) + expected := sha256.Sum256(binary) + assert.Equal(t, expected[:], hash) + + // Deterministic: same input produces same hash. + assert.Equal(t, hash, ComputeBinaryHash(binary)) +} + +func TestConfidentialModule_Execute(t *testing.T) { + ctx := context.Background() + lggr := logger.Nop() + + // Build an ExecuteRequest to send through the module. + execReq := &sdkpb.ExecuteRequest{ + Config: []byte("test-config"), + } + + // Build the expected ExecutionResult that the enclave returns. + expectedResult := &sdkpb.ExecutionResult{ + Result: &sdkpb.ExecutionResult_Value{ + Value: valuespb.NewStringValue("enclave-output"), + }, + } + + // Serialize the result into a ConfidentialWorkflowResponse, as the capability would. + resultBytes, err := proto.Marshal(expectedResult) + require.NoError(t, err) + + confResp := &confworkflowtypes.ConfidentialWorkflowResponse{ + ExecutionResult: resultBytes, + } + respPayload, err := anypb.New(confResp) + require.NoError(t, err) + + t.Run("success", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + execCap := capmocks.NewExecutableCapability(t) + + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(execCap, nil).Once() + + execCap.EXPECT().Execute(matches.AnyContext, mock.MatchedBy(func(req capabilities.CapabilityRequest) bool { + return req.Method == "Execute" && + req.CapabilityId == confidentialWorkflowsCapabilityID && + req.Metadata.WorkflowID == "wf-123" && + req.Metadata.WorkflowOwner == "owner-abc" && + req.Metadata.WorkflowExecutionID == "exec-456" && + req.Payload != nil + })).Return(capabilities.CapabilityResponse{Payload: respPayload}, nil).Once() + + mod := NewConfidentialModule( + capReg, + "https://example.com/binary.wasm", + []byte("fakehash"), + "wf-123", + "owner-abc", + "my-workflow", + "v1", + []SecretIdentifier{ + {Key: "API_KEY"}, + {Key: "SIGNING_KEY", Namespace: "custom-ns"}, + }, + lggr, + ) + + result, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-456"}) + require.NoError(t, err) + require.NotNil(t, result) + + val := result.GetValue() + require.NotNil(t, val) + assert.Equal(t, "enclave-output", val.GetStringValue()) + }) + + t.Run("default namespace is main", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + execCap := capmocks.NewExecutableCapability(t) + + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(execCap, nil).Once() + + // Capture the request to inspect proto secrets. + var capturedReq capabilities.CapabilityRequest + execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). + Run(func(_ context.Context, req capabilities.CapabilityRequest) { + capturedReq = req + }). + Return(capabilities.CapabilityResponse{Payload: respPayload}, nil).Once() + + mod := NewConfidentialModule( + capReg, + "https://example.com/binary.wasm", + []byte("hash"), + "wf-1", "owner", "name", "tag", + []SecretIdentifier{{Key: "SECRET_A"}}, // no namespace + lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-1"}) + require.NoError(t, err) + + // Unmarshal the captured request payload and verify namespace defaulted to "main". + var confReq confworkflowtypes.ConfidentialWorkflowRequest + require.NoError(t, capturedReq.Payload.UnmarshalTo(&confReq)) + require.Len(t, confReq.VaultDonSecrets, 1) + assert.Equal(t, "SECRET_A", confReq.VaultDonSecrets[0].Key) + assert.Equal(t, "main", confReq.VaultDonSecrets[0].GetNamespace()) + }) + + t.Run("GetExecutable error", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(nil, fmt.Errorf("capability not found")).Once() + + mod := NewConfidentialModule( + capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to get confidential-workflows capability") + }) + + t.Run("capability Execute error", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + execCap := capmocks.NewExecutableCapability(t) + + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(execCap, nil).Once() + execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). + Return(capabilities.CapabilityResponse{}, fmt.Errorf("enclave unavailable")).Once() + + mod := NewConfidentialModule( + capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "confidential-workflows capability execution failed") + }) + + t.Run("nil payload in response", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + execCap := capmocks.NewExecutableCapability(t) + + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(execCap, nil).Once() + execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). + Return(capabilities.CapabilityResponse{Payload: nil}, nil).Once() + + mod := NewConfidentialModule( + capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "returned nil payload") + }) + + t.Run("request fields are forwarded correctly", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + execCap := capmocks.NewExecutableCapability(t) + + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(execCap, nil).Once() + + var capturedReq capabilities.CapabilityRequest + execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). + Run(func(_ context.Context, req capabilities.CapabilityRequest) { + capturedReq = req + }). + Return(capabilities.CapabilityResponse{Payload: respPayload}, nil).Once() + + binaryHash := ComputeBinaryHash([]byte("some-binary")) + mod := NewConfidentialModule( + capReg, + "https://example.com/wasm", + binaryHash, + "wf-abc", + "0xowner", + "my-workflow", + "v2", + []SecretIdentifier{ + {Key: "K1", Namespace: "ns1"}, + {Key: "K2"}, + }, + lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-xyz"}) + require.NoError(t, err) + + // Verify metadata. + assert.Equal(t, "Execute", capturedReq.Method) + assert.Equal(t, confidentialWorkflowsCapabilityID, capturedReq.CapabilityId) + assert.Equal(t, "wf-abc", capturedReq.Metadata.WorkflowID) + assert.Equal(t, "0xowner", capturedReq.Metadata.WorkflowOwner) + assert.Equal(t, "my-workflow", capturedReq.Metadata.WorkflowName) + assert.Equal(t, "v2", capturedReq.Metadata.WorkflowTag) + assert.Equal(t, "exec-xyz", capturedReq.Metadata.WorkflowExecutionID) + + // Verify payload contents. + var confReq confworkflowtypes.ConfidentialWorkflowRequest + require.NoError(t, capturedReq.Payload.UnmarshalTo(&confReq)) + + assert.Equal(t, "wf-abc", confReq.Execution.WorkflowId) + assert.Equal(t, "https://example.com/wasm", confReq.Execution.BinaryUrl) + assert.Equal(t, binaryHash, confReq.Execution.BinaryHash) + + // Verify the serialized ExecuteRequest round-trips. + var roundTripped sdkpb.ExecuteRequest + require.NoError(t, proto.Unmarshal(confReq.Execution.ExecuteRequest, &roundTripped)) + assert.Equal(t, execReq.GetConfig(), roundTripped.GetConfig()) + + // Verify secrets. + require.Len(t, confReq.VaultDonSecrets, 2) + assert.Equal(t, "K1", confReq.VaultDonSecrets[0].Key) + assert.Equal(t, "ns1", confReq.VaultDonSecrets[0].GetNamespace()) + assert.Equal(t, "K2", confReq.VaultDonSecrets[1].Key) + assert.Equal(t, "main", confReq.VaultDonSecrets[1].GetNamespace()) + }) +} + +func TestConfidentialModule_InterfaceMethods(t *testing.T) { + mod := &ConfidentialModule{} + + // These are no-ops but should not panic. + mod.Start() + mod.Close() + assert.False(t, mod.IsLegacyDAG()) +} diff --git a/core/store/migrate/migrations/0295_add_workflow_attributes_column.sql b/core/store/migrate/migrations/0295_add_workflow_attributes_column.sql new file mode 100644 index 00000000000..30c85db7289 --- /dev/null +++ b/core/store/migrate/migrations/0295_add_workflow_attributes_column.sql @@ -0,0 +1,5 @@ +-- +goose Up +ALTER TABLE workflow_specs_v2 ADD COLUMN attributes bytea DEFAULT ''; + +-- +goose Down +ALTER TABLE workflow_specs_v2 DROP COLUMN attributes; From 30b095189f8c81ecbe04de59a48bd8b19a03e8ec Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 13:22:16 +0100 Subject: [PATCH 43/67] Revert file fetcher HTTP URL handling The filepath.Base() code for HTTP URLs in newFileFetcher was dead code. No test or production path sends an HTTP URL to a file-based fetcher. The enclave fetches binaries via its own BinaryFetcher, independent of the node syncer's fetcher. --- core/services/workflows/syncer/fetcher.go | 18 +++++------------- core/services/workflows/syncer/v2/fetcher.go | 18 +++++------------- 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/core/services/workflows/syncer/fetcher.go b/core/services/workflows/syncer/fetcher.go index 33ba4b10a2e..c959b6b4da7 100644 --- a/core/services/workflows/syncer/fetcher.go +++ b/core/services/workflows/syncer/fetcher.go @@ -177,20 +177,12 @@ func newFileFetcher(basePath string, lggr logger.Logger) types.FetcherFunc { if err != nil { return nil, fmt.Errorf("invalid URL: %w", err) } + fullPath := filepath.Clean(u.Path) - var fullPath string - if u.Scheme == "http" || u.Scheme == "https" { - // For HTTP(S) URLs, extract just the filename and resolve against basePath. - // This supports confidential workflows where the on-chain URL must be HTTP - // (so the enclave can fetch the binary), but the syncer reads from the local filesystem. - fullPath = filepath.Join(basePath, filepath.Base(u.Path)) - } else { - fullPath = filepath.Clean(u.Path) - // ensure that the incoming request URL is either relative or absolute but within the basePath - if !filepath.IsAbs(fullPath) { - // If it's not absolute, we assume it's relative to the basePath - fullPath = filepath.Join(basePath, fullPath) - } + // ensure that the incoming request URL is either relative or absolute but within the basePath + if !filepath.IsAbs(fullPath) { + // If it's not absolute, we assume it's relative to the basePath + fullPath = filepath.Join(basePath, fullPath) } if !strings.HasPrefix(fullPath, basePath) { return nil, fmt.Errorf("request URL %s is not within the basePath %s", fullPath, basePath) diff --git a/core/services/workflows/syncer/v2/fetcher.go b/core/services/workflows/syncer/v2/fetcher.go index 4c79fffbe24..e86f998f6bb 100644 --- a/core/services/workflows/syncer/v2/fetcher.go +++ b/core/services/workflows/syncer/v2/fetcher.go @@ -211,20 +211,12 @@ func newFileFetcher(basePath string, lggr logger.Logger) types.FetcherFunc { if err != nil { return nil, fmt.Errorf("invalid URL: %w", err) } + fullPath := filepath.Clean(u.Path) - var fullPath string - if u.Scheme == "http" || u.Scheme == "https" { - // For HTTP(S) URLs, extract just the filename and resolve against basePath. - // This supports confidential workflows where the on-chain URL must be HTTP - // (so the enclave can fetch the binary), but the syncer reads from the local filesystem. - fullPath = filepath.Join(basePath, filepath.Base(u.Path)) - } else { - fullPath = filepath.Clean(u.Path) - // ensure that the incoming request URL is either relative or absolute but within the basePath - if !filepath.IsAbs(fullPath) { - // If it's not absolute, we assume it's relative to the basePath - fullPath = filepath.Join(basePath, fullPath) - } + // ensure that the incoming request URL is either relative or absolute but within the basePath + if !filepath.IsAbs(fullPath) { + // If it's not absolute, we assume it's relative to the basePath + fullPath = filepath.Join(basePath, fullPath) } if !strings.HasPrefix(fullPath, basePath) { return nil, fmt.Errorf("request URL %s is not within the basePath %s", fullPath, basePath) From 1d3412bf21193e5c265a2b6f87ab98a73a363c4d Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 13:37:43 +0100 Subject: [PATCH 44/67] Restore comments in startAndRegisterEngine, extract newV2EngineConfig Restore comments that were dropped when extracting startAndRegisterEngine from tryEngineCreate. Extract common EngineConfig construction into newV2EngineConfig and initDone hook wiring into wireInitDoneHook, reducing duplication between the normal and confidential engine paths. --- core/services/workflows/syncer/v2/handler.go | 166 +++++++++---------- 1 file changed, 76 insertions(+), 90 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index b298db91722..5ad4343b784 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -654,60 +654,11 @@ func (h *eventHandler) engineFactoryFn(ctx context.Context, workflowID string, o } // V2 aka "NoDAG" - cfg := &v2.EngineConfig{ - Lggr: h.lggr, - Module: module, - WorkflowConfig: config, - CapRegistry: h.capRegistry, - DonSubscriber: h.workflowDonSubscriber, - UseLocalTimeProvider: h.useLocalTimeProvider, - DonTimeStore: h.donTimeStore, - ExecutionsStore: h.workflowStore, - WorkflowID: workflowID, - WorkflowOwner: owner, - WorkflowName: name, - WorkflowTag: tag, - WorkflowEncryptionKey: h.workflowEncryptionKey, - - LocalLimits: v2.EngineLimits{}, // all defaults - LocalLimiters: h.engineLimiters, - FeatureFlags: h.featureFlags, - GlobalExecutionConcurrencyLimiter: h.workflowLimits, - - BeholderEmitter: func() custmsg.MessageEmitter { - h.emitterMu.RLock() - defer h.emitterMu.RUnlock() - return h.emitter - }(), - BillingClient: h.billingClient, - - WorkflowRegistryAddress: h.workflowRegistryAddress, - WorkflowRegistryChainSelector: h.workflowRegistryChainSelector, - OrgResolver: h.orgResolver, - DebugMode: h.debugMode, - SecretsFetcher: h.secretsFetcher, - SdkName: sdkName, - - ShardOrchestratorClient: h.shardOrchestratorClient, - ShardingEnabled: h.shardingEnabled, - MyShardID: h.myShardID, - ShardRoutingSteady: h.shardRoutingSteady, - } + cfg := h.newV2EngineConfig(module, workflowID, owner, name, tag, config) + cfg.DebugMode = h.debugMode + cfg.SdkName = sdkName - // Wire the initDone channel to the OnInitialized lifecycle hook. - // This will be called when the engine completes initialization (including trigger subscriptions). - // We compose with any existing hook to avoid overwriting test hooks or other user-provided hooks. - if initDone != nil { - existingHook := cfg.Hooks.OnInitialized - cfg.Hooks.OnInitialized = func(err error) { - // Signal completion to the handler first - initDone <- err - // Then call any existing hook (e.g., from tests) - if existingHook != nil { - existingHook(err) - } - } - } + h.wireInitDoneHook(cfg, initDone) return v2.NewEngine(cfg) } @@ -873,13 +824,18 @@ func (h *eventHandler) startAndRegisterEngine( // This ensures we don't emit workflowActivated events before the engine initializes successfully. select { case <-ctx.Done(): + // Context cancelled while waiting for initialization if closeErr := engine.Close(); closeErr != nil { h.lggr.Errorw("failed to close engine after context cancellation", "error", closeErr, "workflowID", workflowID) } return fmt.Errorf("context cancelled while waiting for engine initialization: %w", ctx.Err()) case initErr := <-initDone: if initErr != nil { + // Engine initialization failed (e.g., trigger subscription failed) // TODO (cre-1482) add logic to mark a deployment as failed to avoid churn. + // Currently, failed deployments will be retried on each poll cycle (with exponential backoff). + // If the failure is due to user error (e.g., invalid trigger config), this causes unnecessary retries. + // Consider marking the workflow spec as "failed" in the database and requiring workflow redeployment. if closeErr := engine.Close(); closeErr != nil { h.lggr.Errorw("failed to close engine after initialization failure", "error", closeErr, "workflowID", workflowID) } @@ -887,11 +843,13 @@ func (h *eventHandler) startAndRegisterEngine( } } + // Engine is fully initialized, add to registry with source tracking if err := h.engineRegistry.Add(wid, source, engine); err != nil { if closeErr := engine.Close(); closeErr != nil { return fmt.Errorf("failed to close workflow engine: %w during invariant violation: %w", closeErr, err) } + // Check for WorkflowID collision across sources if errors.Is(err, ErrAlreadyExists) { existingEntry, found := h.engineRegistry.Get(wid) if found { @@ -903,11 +861,73 @@ func (h *eventHandler) startAndRegisterEngine( } } + // This shouldn't happen because we call the handler serially and + // check for running engines above, see the call to engineRegistry.Contains. return fmt.Errorf("invariant violation: %w", err) } return nil } +// newV2EngineConfig builds the common EngineConfig shared by both the normal +// WASM engine and the confidential engine paths. Caller supplies the module. +func (h *eventHandler) newV2EngineConfig( + module host.ModuleV2, + workflowID string, owner string, name types.WorkflowName, tag string, config []byte, +) *v2.EngineConfig { + h.emitterMu.RLock() + emitter := h.emitter + h.emitterMu.RUnlock() + + return &v2.EngineConfig{ + Lggr: h.lggr, + Module: module, + WorkflowConfig: config, + CapRegistry: h.capRegistry, + DonSubscriber: h.workflowDonSubscriber, + UseLocalTimeProvider: h.useLocalTimeProvider, + DonTimeStore: h.donTimeStore, + ExecutionsStore: h.workflowStore, + WorkflowID: workflowID, + WorkflowOwner: owner, + WorkflowName: name, + WorkflowTag: tag, + WorkflowEncryptionKey: h.workflowEncryptionKey, + + LocalLimits: v2.EngineLimits{}, + LocalLimiters: h.engineLimiters, + FeatureFlags: h.featureFlags, + GlobalExecutionConcurrencyLimiter: h.workflowLimits, + + BeholderEmitter: emitter, + BillingClient: h.billingClient, + + WorkflowRegistryAddress: h.workflowRegistryAddress, + WorkflowRegistryChainSelector: h.workflowRegistryChainSelector, + OrgResolver: h.orgResolver, + SecretsFetcher: h.secretsFetcher, + + ShardOrchestratorClient: h.shardOrchestratorClient, + ShardingEnabled: h.shardingEnabled, + MyShardID: h.myShardID, + ShardRoutingSteady: h.shardRoutingSteady, + } +} + +// wireInitDoneHook wires the initDone channel to the OnInitialized lifecycle hook. +// It composes with any existing hook to avoid overwriting test hooks. +func (h *eventHandler) wireInitDoneHook(cfg *v2.EngineConfig, initDone chan<- error) { + if initDone == nil { + return + } + existingHook := cfg.Hooks.OnInitialized + cfg.Hooks.OnInitialized = func(err error) { + initDone <- err + if existingHook != nil { + existingHook(err) + } + } +} + // tryConfidentialEngineCreate creates a V2 engine backed by a ConfidentialModule // instead of a local WASM module. The ConfidentialModule delegates execution to // the confidential-workflows capability which runs the WASM inside a TEE. @@ -941,44 +961,10 @@ func (h *eventHandler) tryConfidentialEngineCreate( lggr, ) - initDone := make(chan error, 1) - - cfg := &v2.EngineConfig{ - Lggr: h.lggr, - Module: module, - WorkflowConfig: []byte(spec.Config), - CapRegistry: h.capRegistry, - DonSubscriber: h.workflowDonSubscriber, - UseLocalTimeProvider: h.useLocalTimeProvider, - DonTimeStore: h.donTimeStore, - ExecutionsStore: h.workflowStore, - WorkflowID: spec.WorkflowID, - WorkflowOwner: spec.WorkflowOwner, - WorkflowName: workflowName, - WorkflowTag: spec.WorkflowTag, - WorkflowEncryptionKey: h.workflowEncryptionKey, - - LocalLimits: v2.EngineLimits{}, - LocalLimiters: h.engineLimiters, - GlobalExecutionConcurrencyLimiter: h.workflowLimits, + cfg := h.newV2EngineConfig(module, spec.WorkflowID, spec.WorkflowOwner, workflowName, spec.WorkflowTag, []byte(spec.Config)) - BeholderEmitter: h.emitter, - BillingClient: h.billingClient, - - WorkflowRegistryAddress: h.workflowRegistryAddress, - WorkflowRegistryChainSelector: h.workflowRegistryChainSelector, - OrgResolver: h.orgResolver, - SecretsFetcher: h.secretsFetcher, - FeatureFlags: h.featureFlags, - } - - existingHook := cfg.Hooks.OnInitialized - cfg.Hooks.OnInitialized = func(err error) { - initDone <- err - if existingHook != nil { - existingHook(err) - } - } + initDone := make(chan error, 1) + h.wireInitDoneHook(cfg, initDone) engine, err := v2.NewEngine(cfg) if err != nil { From b8bf1815b9c95f7d9e8dd3320d3df7539141c933 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 1 Apr 2026 14:21:35 +0200 Subject: [PATCH 45/67] Unify engine creation flow for confidential and normal paths Replace the early-return pattern with a symmetric if/else that picks the factory, then converges on a single startAndRegisterEngine call. Rename tryConfidentialEngineCreate to confidentialEngineFactory and change its signature to return (services.Service, error). # Conflicts: # core/services/workflows/syncer/v2/handler.go --- core/services/workflows/syncer/v2/handler.go | 47 +++++-------------- .../workflows/syncer/v2/handler_test.go | 2 +- 2 files changed, 14 insertions(+), 35 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 5ad4343b784..62a1f5f8428 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -775,29 +775,16 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp if err != nil { return fmt.Errorf("failed to parse workflow attributes: %w", err) } - if confidential { - h.lggr.Infow("routing workflow to confidential execution", "workflowID", spec.WorkflowID) - return h.tryConfidentialEngineCreate(ctx, spec, wid, workflowName, decodedBinary, source) - } - // Create a channel to receive the initialization result. - // This allows us to wait for the engine to complete initialization (including trigger subscriptions) - // before emitting the workflowActivated event, ensuring the event accurately reflects deployment status. initDone := make(chan error, 1) + var engine services.Service - // Scope the engineFactory call so that decodedBinary goes out of scope immediately after the factory returns - engine, err := func() (services.Service, error) { - return h.engineFactory( - ctx, - spec.WorkflowID, - spec.WorkflowOwner, - workflowName, - spec.WorkflowTag, - configBytes, - decodedBinary, - initDone, - ) - }() + if confidential { + h.lggr.Infow("routing workflow to confidential execution", "workflowID", spec.WorkflowID) + engine, err = h.confidentialEngineFactory(spec, wid, workflowName, decodedBinary, initDone) + } else { + engine, err = h.engineFactory(ctx, spec.WorkflowID, spec.WorkflowOwner, workflowName, spec.WorkflowTag, configBytes, decodedBinary, initDone) + } if err != nil { return fmt.Errorf("failed to create workflow engine: %w", err) } @@ -928,20 +915,19 @@ func (h *eventHandler) wireInitDoneHook(cfg *v2.EngineConfig, initDone chan<- er } } -// tryConfidentialEngineCreate creates a V2 engine backed by a ConfidentialModule +// confidentialEngineFactory creates a V2 engine backed by a ConfidentialModule // instead of a local WASM module. The ConfidentialModule delegates execution to // the confidential-workflows capability which runs the WASM inside a TEE. -func (h *eventHandler) tryConfidentialEngineCreate( - ctx context.Context, +func (h *eventHandler) confidentialEngineFactory( spec *job.WorkflowSpec, wid types.WorkflowID, workflowName types.WorkflowName, decodedBinary []byte, - source string, -) error { + initDone chan<- error, +) (services.Service, error) { attrs, err := v2.ParseWorkflowAttributes(spec.Attributes) if err != nil { - return fmt.Errorf("failed to parse workflow attributes: %w", err) + return nil, fmt.Errorf("failed to parse workflow attributes: %w", err) } binaryHash := v2.ComputeBinaryHash(decodedBinary) @@ -962,16 +948,9 @@ func (h *eventHandler) tryConfidentialEngineCreate( ) cfg := h.newV2EngineConfig(module, spec.WorkflowID, spec.WorkflowOwner, workflowName, spec.WorkflowTag, []byte(spec.Config)) - - initDone := make(chan error, 1) h.wireInitDoneHook(cfg, initDone) - engine, err := v2.NewEngine(cfg) - if err != nil { - return fmt.Errorf("failed to create confidential workflow engine: %w", err) - } - - return h.startAndRegisterEngine(ctx, engine, initDone, wid, spec.WorkflowID, source) + return v2.NewEngine(cfg) } // logCustMsg emits a custom message to the external sink and logs an error if that fails. diff --git a/core/services/workflows/syncer/v2/handler_test.go b/core/services/workflows/syncer/v2/handler_test.go index 775f059613e..aa0a66c6362 100644 --- a/core/services/workflows/syncer/v2/handler_test.go +++ b/core/services/workflows/syncer/v2/handler_test.go @@ -704,7 +704,7 @@ func Test_workflowRegisteredHandler_confidentialRouting(t *testing.T) { // (non-hex owner), engine creation fails. The error comes from the // confidential path, proving routing worked correctly. require.Error(t, err) - assert.Contains(t, err.Error(), "failed to create confidential workflow engine") + assert.Contains(t, err.Error(), "failed to create workflow engine") // The engine factory must NOT have been called; the confidential path // bypasses it. From 061f55350258c4b5cbe8e190d69f103653d50783 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 13:50:56 +0100 Subject: [PATCH 46/67] Restore initDone comment in tryEngineCreate --- core/services/workflows/syncer/v2/handler.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 62a1f5f8428..803dba075b3 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -776,6 +776,9 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp return fmt.Errorf("failed to parse workflow attributes: %w", err) } + // Create a channel to receive the initialization result. + // This allows us to wait for the engine to complete initialization (including trigger subscriptions) + // before emitting the workflowActivated event, ensuring the event accurately reflects deployment status. initDone := make(chan error, 1) var engine services.Service From 0e00660bd153e7f198cc06afae6ab561eb8c1ab3 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 13:53:28 +0100 Subject: [PATCH 47/67] Inline startAndRegisterEngine back into tryEngineCreate No longer needed as a separate method now that both engine paths converge in tryEngineCreate. --- core/services/workflows/syncer/v2/handler.go | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 803dba075b3..8afd0673786 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -792,21 +792,7 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp return fmt.Errorf("failed to create workflow engine: %w", err) } - return h.startAndRegisterEngine(ctx, engine, initDone, wid, spec.WorkflowID, source) -} - -// startAndRegisterEngine starts a workflow engine, waits for initialization to -// complete, and registers it in the engine registry. Used by both the normal -// and confidential engine creation paths. -func (h *eventHandler) startAndRegisterEngine( - ctx context.Context, - engine services.Service, - initDone <-chan error, - wid types.WorkflowID, - workflowID string, - source string, -) error { - if err := engine.Start(ctx); err != nil { + if err = engine.Start(ctx); err != nil { return fmt.Errorf("failed to start workflow engine: %w", err) } @@ -816,7 +802,7 @@ func (h *eventHandler) startAndRegisterEngine( case <-ctx.Done(): // Context cancelled while waiting for initialization if closeErr := engine.Close(); closeErr != nil { - h.lggr.Errorw("failed to close engine after context cancellation", "error", closeErr, "workflowID", workflowID) + h.lggr.Errorw("failed to close engine after context cancellation", "error", closeErr, "workflowID", spec.WorkflowID) } return fmt.Errorf("context cancelled while waiting for engine initialization: %w", ctx.Err()) case initErr := <-initDone: @@ -827,7 +813,7 @@ func (h *eventHandler) startAndRegisterEngine( // If the failure is due to user error (e.g., invalid trigger config), this causes unnecessary retries. // Consider marking the workflow spec as "failed" in the database and requiring workflow redeployment. if closeErr := engine.Close(); closeErr != nil { - h.lggr.Errorw("failed to close engine after initialization failure", "error", closeErr, "workflowID", workflowID) + h.lggr.Errorw("failed to close engine after initialization failure", "error", closeErr, "workflowID", spec.WorkflowID) } return fmt.Errorf("engine initialization failed: %w", initErr) } From edf75667c32f57eacd029b03ecca9439a64ddc8f Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 13:55:49 +0100 Subject: [PATCH 48/67] Restore inline comments in wireInitDoneHook --- core/services/workflows/syncer/v2/handler.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 8afd0673786..96fd2abe378 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -890,14 +890,17 @@ func (h *eventHandler) newV2EngineConfig( } // wireInitDoneHook wires the initDone channel to the OnInitialized lifecycle hook. -// It composes with any existing hook to avoid overwriting test hooks. +// This will be called when the engine completes initialization (including trigger subscriptions). +// We compose with any existing hook to avoid overwriting test hooks or other user-provided hooks. func (h *eventHandler) wireInitDoneHook(cfg *v2.EngineConfig, initDone chan<- error) { if initDone == nil { return } existingHook := cfg.Hooks.OnInitialized cfg.Hooks.OnInitialized = func(err error) { + // Signal completion to the handler first initDone <- err + // Then call any existing hook (e.g., from tests) if existingHook != nil { existingHook(err) } From 6f3e39e5d4c4a3526888bc17758a7c86da608f40 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 24 Mar 2026 13:59:20 +0100 Subject: [PATCH 49/67] Restore original BeholderEmitter closure pattern in newV2EngineConfig --- core/services/workflows/syncer/v2/handler.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 96fd2abe378..86d6b57c636 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -850,10 +850,6 @@ func (h *eventHandler) newV2EngineConfig( module host.ModuleV2, workflowID string, owner string, name types.WorkflowName, tag string, config []byte, ) *v2.EngineConfig { - h.emitterMu.RLock() - emitter := h.emitter - h.emitterMu.RUnlock() - return &v2.EngineConfig{ Lggr: h.lggr, Module: module, @@ -869,13 +865,17 @@ func (h *eventHandler) newV2EngineConfig( WorkflowTag: tag, WorkflowEncryptionKey: h.workflowEncryptionKey, - LocalLimits: v2.EngineLimits{}, + LocalLimits: v2.EngineLimits{}, // all defaults LocalLimiters: h.engineLimiters, FeatureFlags: h.featureFlags, GlobalExecutionConcurrencyLimiter: h.workflowLimits, - BeholderEmitter: emitter, - BillingClient: h.billingClient, + BeholderEmitter: func() custmsg.MessageEmitter { + h.emitterMu.RLock() + defer h.emitterMu.RUnlock() + return h.emitter + }(), + BillingClient: h.billingClient, WorkflowRegistryAddress: h.workflowRegistryAddress, WorkflowRegistryChainSelector: h.workflowRegistryChainSelector, From d6d20dda41fb2b2a207cdf2b7058e3c539fca2d0 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 1 Apr 2026 14:24:49 +0200 Subject: [PATCH 50/67] Clean up factory signatures and newV2EngineConfig param ordering Group string params together in newV2EngineConfig, move SdkName and DebugMode into the constructor, drop unused wid param from confidentialEngineFactory. --- core/services/workflows/syncer/v2/handler.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 86d6b57c636..49706219ca5 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -654,9 +654,7 @@ func (h *eventHandler) engineFactoryFn(ctx context.Context, workflowID string, o } // V2 aka "NoDAG" - cfg := h.newV2EngineConfig(module, workflowID, owner, name, tag, config) - cfg.DebugMode = h.debugMode - cfg.SdkName = sdkName + cfg := h.newV2EngineConfig(module, workflowID, owner, tag, sdkName, name, config) h.wireInitDoneHook(cfg, initDone) @@ -784,7 +782,7 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp if confidential { h.lggr.Infow("routing workflow to confidential execution", "workflowID", spec.WorkflowID) - engine, err = h.confidentialEngineFactory(spec, wid, workflowName, decodedBinary, initDone) + engine, err = h.confidentialEngineFactory(spec, workflowName, decodedBinary, initDone) } else { engine, err = h.engineFactory(ctx, spec.WorkflowID, spec.WorkflowOwner, workflowName, spec.WorkflowTag, configBytes, decodedBinary, initDone) } @@ -848,7 +846,9 @@ func (h *eventHandler) tryEngineCreate(ctx context.Context, spec *job.WorkflowSp // WASM engine and the confidential engine paths. Caller supplies the module. func (h *eventHandler) newV2EngineConfig( module host.ModuleV2, - workflowID string, owner string, name types.WorkflowName, tag string, config []byte, + workflowID, owner, tag, sdkName string, + name types.WorkflowName, + config []byte, ) *v2.EngineConfig { return &v2.EngineConfig{ Lggr: h.lggr, @@ -881,6 +881,8 @@ func (h *eventHandler) newV2EngineConfig( WorkflowRegistryChainSelector: h.workflowRegistryChainSelector, OrgResolver: h.orgResolver, SecretsFetcher: h.secretsFetcher, + DebugMode: h.debugMode, + SdkName: sdkName, ShardOrchestratorClient: h.shardOrchestratorClient, ShardingEnabled: h.shardingEnabled, @@ -912,7 +914,6 @@ func (h *eventHandler) wireInitDoneHook(cfg *v2.EngineConfig, initDone chan<- er // the confidential-workflows capability which runs the WASM inside a TEE. func (h *eventHandler) confidentialEngineFactory( spec *job.WorkflowSpec, - wid types.WorkflowID, workflowName types.WorkflowName, decodedBinary []byte, initDone chan<- error, @@ -939,7 +940,7 @@ func (h *eventHandler) confidentialEngineFactory( lggr, ) - cfg := h.newV2EngineConfig(module, spec.WorkflowID, spec.WorkflowOwner, workflowName, spec.WorkflowTag, []byte(spec.Config)) + cfg := h.newV2EngineConfig(module, spec.WorkflowID, spec.WorkflowOwner, spec.WorkflowTag, "", workflowName, []byte(spec.Config)) h.wireInitDoneHook(cfg, initDone) return v2.NewEngine(cfg) From 71ae90f465b059197bc9f38a7ce88887a92ecbcc Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 25 Mar 2026 13:47:48 +0100 Subject: [PATCH 51/67] Fix lint: errors.New, assert.Empty --- core/services/workflows/v2/confidential_module.go | 3 ++- core/services/workflows/v2/confidential_module_test.go | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/core/services/workflows/v2/confidential_module.go b/core/services/workflows/v2/confidential_module.go index 4cc3761e18a..16bbb83052c 100644 --- a/core/services/workflows/v2/confidential_module.go +++ b/core/services/workflows/v2/confidential_module.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "google.golang.org/protobuf/proto" @@ -165,7 +166,7 @@ func (m *ConfidentialModule) Execute( } if capResp.Payload == nil { - return nil, fmt.Errorf("confidential-workflows capability returned nil payload") + return nil, errors.New("confidential-workflows capability returned nil payload") } var confResp confworkflowtypes.ConfidentialWorkflowResponse diff --git a/core/services/workflows/v2/confidential_module_test.go b/core/services/workflows/v2/confidential_module_test.go index 87dab426022..598f82c0119 100644 --- a/core/services/workflows/v2/confidential_module_test.go +++ b/core/services/workflows/v2/confidential_module_test.go @@ -3,7 +3,7 @@ package v2 import ( "context" "crypto/sha256" - "fmt" + "errors" "testing" "time" @@ -52,7 +52,7 @@ func TestParseWorkflowAttributes(t *testing.T) { assert.True(t, attrs.Confidential) require.Len(t, attrs.VaultDonSecrets, 2) assert.Equal(t, "API_KEY", attrs.VaultDonSecrets[0].Key) - assert.Equal(t, "", attrs.VaultDonSecrets[0].Namespace) + assert.Empty(t, attrs.VaultDonSecrets[0].Namespace) assert.Equal(t, "SIGNING_KEY", attrs.VaultDonSecrets[1].Key) assert.Equal(t, "custom-ns", attrs.VaultDonSecrets[1].Namespace) }) @@ -222,7 +222,7 @@ func TestConfidentialModule_Execute(t *testing.T) { t.Run("GetExecutable error", func(t *testing.T) { capReg := regmocks.NewCapabilitiesRegistry(t) capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). - Return(nil, fmt.Errorf("capability not found")).Once() + Return(nil, errors.New("capability not found")).Once() mod := NewConfidentialModule( capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, @@ -240,7 +240,7 @@ func TestConfidentialModule_Execute(t *testing.T) { capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). Return(execCap, nil).Once() execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). - Return(capabilities.CapabilityResponse{}, fmt.Errorf("enclave unavailable")).Once() + Return(capabilities.CapabilityResponse{}, errors.New("enclave unavailable")).Once() mod := NewConfidentialModule( capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, From 90c0f05d6adc95613661eb1ca499b8a641dea3e4 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 26 Mar 2026 17:36:46 +0100 Subject: [PATCH 52/67] Add BinaryURLResolver to ConfidentialModule for presigned URL support The enclave needs an authenticated URL to download WASM binaries from the CRE storage service. BinaryURLResolver resolves the raw on-chain URL into a presigned/ephemeral URL per execution. Nil-safe: falls back to the raw URL when no resolver is set. PR 5/5 (#21642) wires this to the storage service retriever. --- core/services/workflows/syncer/v2/handler.go | 9 +- .../workflows/v2/confidential_module.go | 61 +++++++---- .../workflows/v2/confidential_module_test.go | 103 +++++++++++++++++- 3 files changed, 143 insertions(+), 30 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 49706219ca5..0f357ba996e 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -928,15 +928,16 @@ func (h *eventHandler) confidentialEngineFactory( lggr := logger.Named(h.lggr, "WorkflowEngine.ConfidentialModule") lggr = logger.With(lggr, "workflowID", spec.WorkflowID, "workflowName", spec.WorkflowName, "workflowOwner", spec.WorkflowOwner) + // nil resolver: raw binaryURL is passed to the enclave as-is. + // PR 5/5 (#21642) wires this to the storage service retriever + // so the enclave receives a presigned URL. module := v2.NewConfidentialModule( h.capRegistry, spec.BinaryURL, binaryHash, - spec.WorkflowID, - spec.WorkflowOwner, - workflowName.String(), - spec.WorkflowTag, + spec.WorkflowID, spec.WorkflowOwner, workflowName.String(), spec.WorkflowTag, attrs.VaultDonSecrets, + nil, lggr, ) diff --git a/core/services/workflows/v2/confidential_module.go b/core/services/workflows/v2/confidential_module.go index 16bbb83052c..7a1983d2a27 100644 --- a/core/services/workflows/v2/confidential_module.go +++ b/core/services/workflows/v2/confidential_module.go @@ -57,19 +57,26 @@ func IsConfidential(data []byte) (bool, error) { return attrs.Confidential, nil } +// BinaryURLResolver resolves a raw binary URL into an ephemeral/presigned +// URL that the enclave can fetch without authentication. In production this +// calls the CRE storage service; nil means the raw URL is used as-is. +// PR 5/5 (#21642) wires this to the storage service retriever. +type BinaryURLResolver func(ctx context.Context, workflowID string) (string, error) + // ConfidentialModule implements host.ModuleV2 for confidential workflows. // Instead of running WASM locally, it delegates execution to the // confidential-workflows capability via the CapabilitiesRegistry. type ConfidentialModule struct { - capRegistry core.CapabilitiesRegistry - binaryURL string - binaryHash []byte - workflowID string - workflowOwner string - workflowName string - workflowTag string - vaultDonSecrets []SecretIdentifier - lggr logger.Logger + capRegistry core.CapabilitiesRegistry + binaryURL string + binaryHash []byte + workflowID string + workflowOwner string + workflowName string + workflowTag string + vaultDonSecrets []SecretIdentifier + binaryURLResolver BinaryURLResolver + lggr logger.Logger } var _ host.ModuleV2 = (*ConfidentialModule)(nil) @@ -78,23 +85,22 @@ func NewConfidentialModule( capRegistry core.CapabilitiesRegistry, binaryURL string, binaryHash []byte, - workflowID string, - workflowOwner string, - workflowName string, - workflowTag string, + workflowID, workflowOwner, workflowName, workflowTag string, vaultDonSecrets []SecretIdentifier, + binaryURLResolver BinaryURLResolver, lggr logger.Logger, ) *ConfidentialModule { return &ConfidentialModule{ - capRegistry: capRegistry, - binaryURL: binaryURL, - binaryHash: binaryHash, - workflowID: workflowID, - workflowOwner: workflowOwner, - workflowName: workflowName, - workflowTag: workflowTag, - vaultDonSecrets: vaultDonSecrets, - lggr: lggr, + capRegistry: capRegistry, + binaryURL: binaryURL, + binaryHash: binaryHash, + workflowID: workflowID, + workflowOwner: workflowOwner, + workflowName: workflowName, + workflowTag: workflowTag, + vaultDonSecrets: vaultDonSecrets, + binaryURLResolver: binaryURLResolver, + lggr: lggr, } } @@ -125,11 +131,20 @@ func (m *ConfidentialModule) Execute( } } + binaryURL := m.binaryURL + if m.binaryURLResolver != nil { + resolved, resolveErr := m.binaryURLResolver(ctx, m.workflowID) + if resolveErr != nil { + return nil, fmt.Errorf("failed to resolve binary URL: %w", resolveErr) + } + binaryURL = resolved + } + capInput := &confworkflowtypes.ConfidentialWorkflowRequest{ VaultDonSecrets: protoSecrets, Execution: &confworkflowtypes.WorkflowExecution{ WorkflowId: m.workflowID, - BinaryUrl: m.binaryURL, + BinaryUrl: binaryURL, BinaryHash: m.binaryHash, ExecuteRequest: execReqBytes, Owner: m.workflowOwner, diff --git a/core/services/workflows/v2/confidential_module_test.go b/core/services/workflows/v2/confidential_module_test.go index 598f82c0119..e72d7164b29 100644 --- a/core/services/workflows/v2/confidential_module_test.go +++ b/core/services/workflows/v2/confidential_module_test.go @@ -172,6 +172,7 @@ func TestConfidentialModule_Execute(t *testing.T) { {Key: "API_KEY"}, {Key: "SIGNING_KEY", Namespace: "custom-ns"}, }, + nil, lggr, ) @@ -205,6 +206,7 @@ func TestConfidentialModule_Execute(t *testing.T) { []byte("hash"), "wf-1", "owner", "name", "tag", []SecretIdentifier{{Key: "SECRET_A"}}, // no namespace + nil, lggr, ) @@ -225,7 +227,7 @@ func TestConfidentialModule_Execute(t *testing.T) { Return(nil, errors.New("capability not found")).Once() mod := NewConfidentialModule( - capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, + capReg, "", nil, "wf", "owner", "name", "tag", nil, nil, lggr, ) _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) @@ -243,7 +245,7 @@ func TestConfidentialModule_Execute(t *testing.T) { Return(capabilities.CapabilityResponse{}, errors.New("enclave unavailable")).Once() mod := NewConfidentialModule( - capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, + capReg, "", nil, "wf", "owner", "name", "tag", nil, nil, lggr, ) _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) @@ -261,7 +263,7 @@ func TestConfidentialModule_Execute(t *testing.T) { Return(capabilities.CapabilityResponse{Payload: nil}, nil).Once() mod := NewConfidentialModule( - capReg, "", nil, "wf", "owner", "name", "tag", nil, lggr, + capReg, "", nil, "wf", "owner", "name", "tag", nil, nil, lggr, ) _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{}) @@ -296,6 +298,7 @@ func TestConfidentialModule_Execute(t *testing.T) { {Key: "K1", Namespace: "ns1"}, {Key: "K2"}, }, + nil, lggr, ) @@ -333,6 +336,100 @@ func TestConfidentialModule_Execute(t *testing.T) { }) } +func TestConfidentialModule_BinaryURLResolver(t *testing.T) { + ctx := context.Background() + lggr := logger.Nop() + + execReq := &sdkpb.ExecuteRequest{Config: []byte("cfg")} + + expectedResult := &sdkpb.ExecutionResult{ + Result: &sdkpb.ExecutionResult_Value{ + Value: valuespb.NewStringValue("ok"), + }, + } + resultBytes, err := proto.Marshal(expectedResult) + require.NoError(t, err) + confResp := &confworkflowtypes.ConfidentialWorkflowResponse{ExecutionResult: resultBytes} + respPayload, err := anypb.New(confResp) + require.NoError(t, err) + + t.Run("resolver replaces binary URL", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + execCap := capmocks.NewExecutableCapability(t) + + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(execCap, nil).Once() + + var capturedReq capabilities.CapabilityRequest + execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). + Run(func(_ context.Context, req capabilities.CapabilityRequest) { + capturedReq = req + }). + Return(capabilities.CapabilityResponse{Payload: respPayload}, nil).Once() + + resolver := func(_ context.Context, wfID string) (string, error) { + return "https://presigned.example.com/" + wfID + "?token=abc", nil + } + + mod := NewConfidentialModule( + capReg, "https://storage.example.com/raw", []byte("hash"), + "wf-1", "owner", "name", "tag", nil, resolver, lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-1"}) + require.NoError(t, err) + + var confReq confworkflowtypes.ConfidentialWorkflowRequest + require.NoError(t, capturedReq.Payload.UnmarshalTo(&confReq)) + assert.Equal(t, "https://presigned.example.com/wf-1?token=abc", confReq.Execution.BinaryUrl) + }) + + t.Run("nil resolver uses raw URL", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + execCap := capmocks.NewExecutableCapability(t) + + capReg.EXPECT().GetExecutable(matches.AnyContext, confidentialWorkflowsCapabilityID). + Return(execCap, nil).Once() + + var capturedReq capabilities.CapabilityRequest + execCap.EXPECT().Execute(matches.AnyContext, mock.Anything). + Run(func(_ context.Context, req capabilities.CapabilityRequest) { + capturedReq = req + }). + Return(capabilities.CapabilityResponse{Payload: respPayload}, nil).Once() + + mod := NewConfidentialModule( + capReg, "https://storage.example.com/raw", []byte("hash"), + "wf-1", "owner", "name", "tag", nil, nil, lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-1"}) + require.NoError(t, err) + + var confReq confworkflowtypes.ConfidentialWorkflowRequest + require.NoError(t, capturedReq.Payload.UnmarshalTo(&confReq)) + assert.Equal(t, "https://storage.example.com/raw", confReq.Execution.BinaryUrl) + }) + + t.Run("resolver error propagates", func(t *testing.T) { + capReg := regmocks.NewCapabilitiesRegistry(t) + + resolver := func(_ context.Context, _ string) (string, error) { + return "", errors.New("storage service unavailable") + } + + mod := NewConfidentialModule( + capReg, "https://storage.example.com/raw", []byte("hash"), + "wf-1", "owner", "name", "tag", nil, resolver, lggr, + ) + + _, err := mod.Execute(ctx, execReq, &stubExecutionHelper{executionID: "exec-1"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to resolve binary URL") + assert.Contains(t, err.Error(), "storage service unavailable") + }) +} + func TestConfidentialModule_InterfaceMethods(t *testing.T) { mod := &ConfidentialModule{} From 830c81f1c8605f7c335c4b81ff3ec623e07b934f Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 8 Apr 2026 13:46:08 +0200 Subject: [PATCH 53/67] add EmitUserMetric to stubExecutionHelper host.ExecutionHelper gained EmitUserMetric in chainlink-common #1924. --- core/services/workflows/v2/confidential_module_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/services/workflows/v2/confidential_module_test.go b/core/services/workflows/v2/confidential_module_test.go index e72d7164b29..96a58cafa13 100644 --- a/core/services/workflows/v2/confidential_module_test.go +++ b/core/services/workflows/v2/confidential_module_test.go @@ -24,6 +24,7 @@ import ( sdkpb "github.com/smartcontractkit/chainlink-protos/cre/go/sdk" valuespb "github.com/smartcontractkit/chainlink-protos/cre/go/values/pb" + wfpb "github.com/smartcontractkit/chainlink-protos/workflows/go/v2" ) // stubExecutionHelper implements host.ExecutionHelper for testing. @@ -43,6 +44,9 @@ func (s *stubExecutionHelper) GetWorkflowExecutionID() string { return s.executi func (s *stubExecutionHelper) GetNodeTime() time.Time { return time.Time{} } func (s *stubExecutionHelper) GetDONTime() (time.Time, error) { return time.Time{}, nil } func (s *stubExecutionHelper) EmitUserLog(string) error { return nil } +func (s *stubExecutionHelper) EmitUserMetric(context.Context, *wfpb.WorkflowUserMetric) error { + return nil +} func TestParseWorkflowAttributes(t *testing.T) { t.Run("valid JSON with all fields", func(t *testing.T) { From 5a538a0c003dbfcdc84cc5f73116d01e63994ad5 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 8 Apr 2026 14:23:57 +0200 Subject: [PATCH 54/67] fix struct field alignment in ConfidentialModule --- .../workflows/v2/confidential_module.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/core/services/workflows/v2/confidential_module.go b/core/services/workflows/v2/confidential_module.go index 7a1983d2a27..246886bcebf 100644 --- a/core/services/workflows/v2/confidential_module.go +++ b/core/services/workflows/v2/confidential_module.go @@ -67,16 +67,16 @@ type BinaryURLResolver func(ctx context.Context, workflowID string) (string, err // Instead of running WASM locally, it delegates execution to the // confidential-workflows capability via the CapabilitiesRegistry. type ConfidentialModule struct { - capRegistry core.CapabilitiesRegistry - binaryURL string - binaryHash []byte - workflowID string - workflowOwner string - workflowName string - workflowTag string - vaultDonSecrets []SecretIdentifier - binaryURLResolver BinaryURLResolver - lggr logger.Logger + capRegistry core.CapabilitiesRegistry + binaryURL string + binaryHash []byte + workflowID string + workflowOwner string + workflowName string + workflowTag string + vaultDonSecrets []SecretIdentifier + binaryURLResolver BinaryURLResolver + lggr logger.Logger } var _ host.ModuleV2 = (*ConfidentialModule)(nil) From 433ecdabbb50473047ba0e7b243f9c0274227288 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Fri, 10 Apr 2026 18:18:58 +0200 Subject: [PATCH 55/67] set org_id on WorkflowExecution from CRE context Bump chainlink-common to pick up OrgId field on WorkflowExecution proto. Read org from the CRE execution context and include it in the proto sent to the enclave, matching the pattern used by the framework executor for VaultDON requests. --- .../workflows/v2/confidential_module.go | 2 + go.mod | 50 ++++----- go.sum | 103 +++++++++--------- 3 files changed, 79 insertions(+), 76 deletions(-) diff --git a/core/services/workflows/v2/confidential_module.go b/core/services/workflows/v2/confidential_module.go index 246886bcebf..6c27ea6c7f1 100644 --- a/core/services/workflows/v2/confidential_module.go +++ b/core/services/workflows/v2/confidential_module.go @@ -11,6 +11,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/contexts" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/types/core" "github.com/smartcontractkit/chainlink-common/pkg/workflows/wasm/host" @@ -149,6 +150,7 @@ func (m *ConfidentialModule) Execute( ExecuteRequest: execReqBytes, Owner: m.workflowOwner, ExecutionId: helper.GetWorkflowExecutionID(), + OrgId: contexts.CREValue(ctx).Org, }, } diff --git a/go.mod b/go.mod index fe53621739e..18ce64e3cc8 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260224214816-cb23ec38649f github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20250912190424-fd2e35d7deb5 github.com/smartcontractkit/chainlink-ccv v0.0.0-20260324000441-d4cfddc9f7d2 - github.com/smartcontractkit/chainlink-common v0.11.2-0.20260406055916-9aa6b6c0ae81 + github.com/smartcontractkit/chainlink-common v0.11.2-0.20260410155936-7371d7bd86ed github.com/smartcontractkit/chainlink-common/keystore v1.0.2 github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10 github.com/smartcontractkit/chainlink-data-streams v0.1.13 @@ -97,7 +97,7 @@ require ( github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260326122810-b657beadfb57 github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260326180413-c69f27e37a13 github.com/smartcontractkit/chainlink-protos/billing/go v0.0.0-20251024234028-0988426d98f4 - github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260326111235-8c09d1a4491f + github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260409211238-5b99921cbc7c github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260319180422-b5808c964785 github.com/smartcontractkit/chainlink-protos/orchestrator v0.10.0 @@ -129,22 +129,22 @@ require ( github.com/wk8/go-ordered-map/v2 v2.1.8 go.dedis.ch/kyber/v3 v3.1.0 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.49.0 - go.opentelemetry.io/otel v1.42.0 - go.opentelemetry.io/otel/log v0.15.0 - go.opentelemetry.io/otel/metric v1.42.0 - go.opentelemetry.io/otel/sdk/metric v1.41.0 - go.opentelemetry.io/otel/trace v1.42.0 + go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel/log v0.19.0 + go.opentelemetry.io/otel/metric v1.43.0 + go.opentelemetry.io/otel/sdk/metric v1.43.0 + go.opentelemetry.io/otel/trace v1.43.0 go.uber.org/atomic v1.11.0 go.uber.org/zap v1.27.1 - golang.org/x/crypto v0.48.0 + golang.org/x/crypto v0.49.0 golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa golang.org/x/mod v0.33.0 - golang.org/x/oauth2 v0.32.0 + golang.org/x/oauth2 v0.35.0 golang.org/x/sync v0.20.0 - golang.org/x/term v0.40.0 + golang.org/x/term v0.41.0 golang.org/x/time v0.14.0 gonum.org/v1/gonum v0.17.0 - google.golang.org/grpc v1.78.0 + google.golang.org/grpc v1.79.3 google.golang.org/protobuf v1.36.11 gopkg.in/guregu/null.v4 v4.0.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -193,7 +193,7 @@ require ( github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect github.com/buger/goterm v1.0.4 // indirect - github.com/buger/jsonparser v1.1.1 // indirect + github.com/buger/jsonparser v1.1.2 // indirect github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 // indirect github.com/bytedance/sonic v1.12.3 // indirect github.com/bytedance/sonic/loader v0.2.0 // indirect @@ -278,7 +278,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 // indirect github.com/hashicorp/go-bexpr v0.1.10 // indirect @@ -403,31 +403,31 @@ require ( go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.12.2 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.12.2 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.40.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.13.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.36.0 // indirect - go.opentelemetry.io/otel/sdk v1.41.0 // indirect - go.opentelemetry.io/otel/sdk/log v0.15.0 // indirect - go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/sdk/log v0.16.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/ratelimit v0.3.1 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/arch v0.11.0 // indirect - golang.org/x/net v0.50.0 // indirect - golang.org/x/sys v0.41.0 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sys v0.42.0 // indirect golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect - golang.org/x/text v0.34.0 // indirect + golang.org/x/text v0.35.0 // indirect golang.org/x/tools v0.42.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260114163908-3f89685c29c3 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect gopkg.in/guregu/null.v2 v2.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gotest.tools/v3 v3.5.2 // indirect diff --git a/go.sum b/go.sum index 4c02975ac41..03d1af4414c 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,9 @@ github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129/go.mod h1:u9UyCz2eTrSGy6fbupqJ54eY5c4IC8gREQ1053dK12U= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= +github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/buraksezer/consistent v0.10.0 h1:hqBgz1PvNLC5rkWcEBVAL9dFMBWz6I0VgUCW25rrZlU= github.com/buraksezer/consistent v0.10.0/go.mod h1:6BrVajWq7wbKZlTOUPs/XVfR8c0maujuPowduSpZqmw= github.com/bytecodealliance/wasmtime-go/v28 v28.0.0 h1:aBU8cexP2rPZ0Qz488kvn2NXvWZHL2aG1/+n7Iv+xGc= @@ -684,8 +685,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 h1:BpJ2o0OR5FV7vrkDYfXYVJQeMNWa8RhklZOpW2ITAIQ= @@ -1235,8 +1236,8 @@ github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20250 github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20250912190424-fd2e35d7deb5/go.mod h1:xtZNi6pOKdC3sLvokDvXOhgHzT+cyBqH/gWwvxTxqrg= github.com/smartcontractkit/chainlink-ccv v0.0.0-20260324000441-d4cfddc9f7d2 h1:5HdH/A6yn8INZAltYDLb7UkUi5IKemhJzJkDW4Bgxyg= github.com/smartcontractkit/chainlink-ccv v0.0.0-20260324000441-d4cfddc9f7d2/go.mod h1:wDHq2E0KwUWG0lQ9f5frW1a7CKVW17MJLPuvKmtSRDg= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260406055916-9aa6b6c0ae81 h1:qBQxh/dndRMJX41xWEihr8FkvPL21luWwTFn/0Nl3RU= -github.com/smartcontractkit/chainlink-common v0.11.2-0.20260406055916-9aa6b6c0ae81/go.mod h1:Ob7ZRLEvPkDwGUjKdDIiHy0Mxu4+UG6oMBkR7Jv/U6o= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260410155936-7371d7bd86ed h1:icSvIEyu+A6FnhkFcKf+4tEoBcB43hUYv9G6SHySvxU= +github.com/smartcontractkit/chainlink-common v0.11.2-0.20260410155936-7371d7bd86ed/go.mod h1:2jgRjnUvuGOq8HTZFGdhUxBEBsi+7FHacIUrTQsrk1Q= github.com/smartcontractkit/chainlink-common/keystore v1.0.2 h1:AWisx4JT3QV8tcgh6J5NCrex+wAgTYpWyHsyNPSXzsQ= github.com/smartcontractkit/chainlink-common/keystore v1.0.2/go.mod h1:rSkIHdomyak3YnUtXLenl6poIq8q0V3UZPiiyYqPdGA= github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.10 h1:FJAFgXS9oqASnkS03RE1HQwYQQxrO4l46O5JSzxqLgg= @@ -1271,8 +1272,8 @@ github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-discovery v0. github.com/smartcontractkit/chainlink-protos/chainlink-ccv/message-discovery v0.0.0-20251211142334-5c3421fe2c8d/go.mod h1:ATjAPIVJibHRcIfiG47rEQkUIOoYa6KDvWj3zwCAw6g= github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d h1:AJy55QJ/pBhXkZjc7N+ATnWfxrcjq9BI9DmdtdjwDUQ= github.com/smartcontractkit/chainlink-protos/chainlink-ccv/verifier v0.0.0-20251211142334-5c3421fe2c8d/go.mod h1:5JdppgngCOUS76p61zCinSCgOhPeYQ+OcDUuome5THQ= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260326111235-8c09d1a4491f h1:8p3vE987AHM3Of1JvnNJXNE/AtWtfNvJhk3TeeAG3Qw= -github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260326111235-8c09d1a4491f/go.mod h1:Jqt53s27Tr0jDl8mdBXg1xhu6F8Fci8JOuq43tgHOM8= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260409211238-5b99921cbc7c h1:7V+V3R//Z/g1kiz/to9MGT4KmcSTcbv2YSsRDD0RN5A= +github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260409211238-5b99921cbc7c/go.mod h1:Jqt53s27Tr0jDl8mdBXg1xhu6F8Fci8JOuq43tgHOM8= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b h1:QuI6SmQFK/zyUlVWEf0GMkiUYBPY4lssn26nKSd/bOM= github.com/smartcontractkit/chainlink-protos/linking-service/go v0.0.0-20251002192024-d2ad9222409b/go.mod h1:qSTSwX3cBP3FKQwQacdjArqv0g6QnukjV4XuzO6UyoY= github.com/smartcontractkit/chainlink-protos/node-platform v0.0.0-20260319180422-b5808c964785 h1:oli+2uLU6jcrJGCuYFqk3475hiwL17SWlITWLv+tx/w= @@ -1517,48 +1518,48 @@ go.opentelemetry.io/contrib/propagators/b3 v1.24.0 h1:n4xwCdTx3pZqZs2CjS/CUZAs03 go.opentelemetry.io/contrib/propagators/b3 v1.24.0/go.mod h1:k5wRxKRU2uXx2F8uNJ4TaonuEO/V7/5xoz7kdsDACT8= go.opentelemetry.io/otel v1.6.3/go.mod h1:7BgNga5fNlF/iZjG06hM3yofffp0ofKCDwSXx1GC4dI= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= -go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= -go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.12.2 h1:06ZeJRe5BnYXceSM9Vya83XXVaNGe3H1QqsvqRANQq8= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.12.2/go.mod h1:DvPtKE63knkDVP88qpatBj81JxN+w1bqfVbsbCbj1WY= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.12.2 h1:tPLwQlXbJ8NSOfZc4OkgU5h2A38M4c9kfHSVc4PFQGs= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.12.2/go.mod h1:QTnxBwT/1rBIgAG1goq6xMydfYOBKU6KTiYF4fp5zL8= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0 h1:djrxvDxAe44mJUrKataUbOhCKhR3F8QCyWucO16hTQs= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.16.0/go.mod h1:dt3nxpQEiSoKvfTVxp3TUg5fHPLhKtbcnN3Z1I1ePD0= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0 h1:gAU726w9J8fwr4qRDqu1GYMNNs4gXrU+Pv20/N1UpB4= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0/go.mod h1:RboSDkp7N292rgu+T0MgVt2qgFGu6qa1RpZDOtpL76w= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.40.0 h1:9y5sHvAxWzft1WQ4BwqcvA+IFVUJ1Ya75mSAUnFEVwE= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.40.0/go.mod h1:eQqT90eR3X5Dbs1g9YSM30RavwLF725Ris5/XSXWvqE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0/go.mod h1:QjUEoiGCPkvFZ/MjK6ZZfNOS6mfVEVKYE99dFhuN2LI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 h1:Ckwye2FpXkYgiHX7fyVrN1uA/UYd9ounqqTuSNAv0k4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0/go.mod h1:teIFJh5pW2y+AN7riv6IBPX2DuesS3HgP39mwOspKwU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 h1:wVZXIWjQSeSmMoxF74LzAnpVQOAFDo3pPji9Y4SOFKc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0/go.mod h1:khvBS2IggMFNwZK/6lEeHg/W57h/IX6J4URh57fuI40= go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.13.0 h1:yEX3aC9KDgvYPhuKECHbOlr5GLwH6KTjLJ1sBSkkxkc= go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.13.0/go.mod h1:/GXR0tBmmkxDaCUGahvksvp66mx4yh5+cFXgSlhg0vQ= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.36.0 h1:G8Xec/SgZQricwWBJF/mHZc7A02YHedfFDENwJEdRA0= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.36.0/go.mod h1:PD57idA/AiFD5aqoxGxCvT/ILJPeHy3MjqU/NS7KogY= -go.opentelemetry.io/otel/log v0.15.0 h1:0VqVnc3MgyYd7QqNVIldC3dsLFKgazR6P3P3+ypkyDY= -go.opentelemetry.io/otel/log v0.15.0/go.mod h1:9c/G1zbyZfgu1HmQD7Qj84QMmwTp2QCQsZH1aeoWDE4= +go.opentelemetry.io/otel/log v0.19.0 h1:KUZs/GOsw79TBBMfDWsXS+KZ4g2Ckzksd1ymzsIEbo4= +go.opentelemetry.io/otel/log v0.19.0/go.mod h1:5DQYeGmxVIr4n0/BcJvF4upsraHjg6vudJJpnkL6Ipk= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= -go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= -go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8= -go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90= -go.opentelemetry.io/otel/sdk/log v0.15.0 h1:WgMEHOUt5gjJE93yqfqJOkRflApNif84kxoHWS9VVHE= -go.opentelemetry.io/otel/sdk/log v0.15.0/go.mod h1:qDC/FlKQCXfH5hokGsNg9aUBGMJQsrUyeOiW5u+dKBQ= -go.opentelemetry.io/otel/sdk/log/logtest v0.13.0 h1:9yio6AFZ3QD9j9oqshV1Ibm9gPLlHNxurno5BreMtIA= -go.opentelemetry.io/otel/sdk/log/logtest v0.13.0/go.mod h1:QOGiAJHl+fob8Nu85ifXfuQYmJTFAvcrxL6w5/tu168= -go.opentelemetry.io/otel/sdk/metric v1.41.0 h1:siZQIYBAUd1rlIWQT2uCxWJxcCO7q3TriaMlf08rXw8= -go.opentelemetry.io/otel/sdk/metric v1.41.0/go.mod h1:HNBuSvT7ROaGtGI50ArdRLUnvRTRGniSUZbxiWxSO8Y= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/log v0.16.0 h1:e/b4bdlQwC5fnGtG3dlXUrNOnP7c8YLVSpSfEBIkTnI= +go.opentelemetry.io/otel/sdk/log v0.16.0/go.mod h1:JKfP3T6ycy7QEuv3Hj8oKDy7KItrEkus8XJE6EoSzw4= +go.opentelemetry.io/otel/sdk/log/logtest v0.16.0 h1:/XVkpZ41rVRTP4DfMgYv1nEtNmf65XPPyAdqV90TMy4= +go.opentelemetry.io/otel/sdk/log/logtest v0.16.0/go.mod h1:iOOPgQr5MY9oac/F5W86mXdeyWZGleIx3uXO98X2R6Y= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= go.opentelemetry.io/otel/trace v1.6.3/go.mod h1:GNJQusJlUgZl9/TQBPKU/Y/ty+0iVB5fjhKeJGZPGFs= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= -go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= -go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= -go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= -go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1620,8 +1621,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1726,8 +1727,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= -golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1740,8 +1741,8 @@ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= -golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= +golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1848,8 +1849,8 @@ golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 h1:bTLqdHv7xrGlFbvf5/TXNxy/iUwwdkjhqQTJDjW7aj0= golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4/go.mod h1:g5NllXBEermZrmR51cJDQxmJUHUOfRAaNyWBM+R+548= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1864,8 +1865,8 @@ golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1881,8 +1882,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2047,10 +2048,10 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 h1:1tXaIXCracvtsRxSBsYDiSBN0cuJvM7QYW+MrpIRY78= google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:49MsLSx0oWMOZqcpB3uL8ZOkAh1+TndpJ8ONoCBWiZk= -google.golang.org/genproto/googleapis/api v0.0.0-20260114163908-3f89685c29c3 h1:X9z6obt+cWRX8XjDVOn+SZWhWe5kZHm46TThU9j+jss= -google.golang.org/genproto/googleapis/api v0.0.0-20260114163908-3f89685c29c3/go.mod h1:dd646eSK+Dk9kxVBl1nChEOhJPtMXriCcVb4x3o6J+E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -2073,8 +2074,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= -google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 626dbcb15d4421f461225a45b0f4da1bbc2b5267 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 14 Apr 2026 15:23:54 +0200 Subject: [PATCH 56/67] plugins: pin confidential plugins to latest cc-test commit --- plugins/plugins.private.yaml | 4 ++-- system-tests/tests/test-helpers/t_helpers.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/plugins.private.yaml b/plugins/plugins.private.yaml index a9c5d0f65a3..b52c983efd2 100644 --- a/plugins/plugins.private.yaml +++ b/plugins/plugins.private.yaml @@ -56,9 +56,9 @@ plugins: installPath: "." confidential-http: - moduleURI: "github.com/smartcontractkit/confidential-compute/enclave/apps/confidential-http/capability" - gitRef: "fc4c85318b9039627aaf8ecbf2b89966140b4af3" + gitRef: "6903e98bae3e004d414e75d2bc5bcf10f601bf91" installPath: "./cmd/confidential-http" confidential-workflows: - moduleURI: "github.com/smartcontractkit/confidential-compute/enclave/apps/confidential-workflows/capability" - gitRef: "fc4c85318b9039627aaf8ecbf2b89966140b4af3" + gitRef: "6903e98bae3e004d414e75d2bc5bcf10f601bf91" installPath: "./cmd/confidential-workflows" diff --git a/system-tests/tests/test-helpers/t_helpers.go b/system-tests/tests/test-helpers/t_helpers.go index 19fe5eba175..e32757847a2 100644 --- a/system-tests/tests/test-helpers/t_helpers.go +++ b/system-tests/tests/test-helpers/t_helpers.go @@ -402,19 +402,19 @@ func workflowConfigFactory[T WorkflowConfig](t *testing.T, testLogger zerolog.Lo testLogger.Info().Msg("PoR v2 workflow config file created.") case *AptosReadWorkflowConfig: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create aptos read workflow config file") testLogger.Info().Msg("Aptos read workflow config file created.") case *aptoswrite_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create aptos write workflow config file") testLogger.Info().Msg("Aptos write workflow config file created.") case *aptoswriteroundtrip_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create aptos write roundtrip workflow config file") testLogger.Info().Msg("Aptos write roundtrip workflow config file created.") @@ -491,7 +491,7 @@ func workflowConfigFactory[T WorkflowConfig](t *testing.T, testLogger zerolog.Lo testLogger.Info().Msg("Solana write workflow config file created.") case *vaultsecret_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create vaultsecret workflow config file") testLogger.Info().Msg("Vault secret workflow config file created.") From ab3c0c5abfbf896696f194427bcc1229a2f4aeda Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 14 Apr 2026 15:28:04 +0200 Subject: [PATCH 57/67] revert: keep t_helpers unchanged on cw-5 branch --- system-tests/tests/test-helpers/t_helpers.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/system-tests/tests/test-helpers/t_helpers.go b/system-tests/tests/test-helpers/t_helpers.go index e32757847a2..19fe5eba175 100644 --- a/system-tests/tests/test-helpers/t_helpers.go +++ b/system-tests/tests/test-helpers/t_helpers.go @@ -402,19 +402,19 @@ func workflowConfigFactory[T WorkflowConfig](t *testing.T, testLogger zerolog.Lo testLogger.Info().Msg("PoR v2 workflow config file created.") case *AptosReadWorkflowConfig: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create aptos read workflow config file") testLogger.Info().Msg("Aptos read workflow config file created.") case *aptoswrite_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create aptos write workflow config file") testLogger.Info().Msg("Aptos write workflow config file created.") case *aptoswriteroundtrip_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create aptos write roundtrip workflow config file") testLogger.Info().Msg("Aptos write roundtrip workflow config file created.") @@ -491,7 +491,7 @@ func workflowConfigFactory[T WorkflowConfig](t *testing.T, testLogger zerolog.Lo testLogger.Info().Msg("Solana write workflow config file created.") case *vaultsecret_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create vaultsecret workflow config file") testLogger.Info().Msg("Vault secret workflow config file created.") From 87a0d3b6bfc09d2356c714af8da4666ff08536a4 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 14 Apr 2026 15:31:27 +0200 Subject: [PATCH 58/67] plugins: bump confidential plugin refs to db40fb4 --- plugins/plugins.private.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/plugins.private.yaml b/plugins/plugins.private.yaml index b52c983efd2..72393f720bf 100644 --- a/plugins/plugins.private.yaml +++ b/plugins/plugins.private.yaml @@ -56,9 +56,9 @@ plugins: installPath: "." confidential-http: - moduleURI: "github.com/smartcontractkit/confidential-compute/enclave/apps/confidential-http/capability" - gitRef: "6903e98bae3e004d414e75d2bc5bcf10f601bf91" + gitRef: "db40fb4578a4c5020fc97954ea65ad761dc74310" installPath: "./cmd/confidential-http" confidential-workflows: - moduleURI: "github.com/smartcontractkit/confidential-compute/enclave/apps/confidential-workflows/capability" - gitRef: "6903e98bae3e004d414e75d2bc5bcf10f601bf91" + gitRef: "db40fb4578a4c5020fc97954ea65ad761dc74310" installPath: "./cmd/confidential-workflows" From e95b0e9c6266f5ac22e2a2763ef1bbb932b6742a Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 14 Apr 2026 15:34:45 +0200 Subject: [PATCH 59/67] Revert "revert: keep t_helpers unchanged on cw-5 branch" This reverts commit ab3c0c5abfbf896696f194427bcc1229a2f4aeda. --- system-tests/tests/test-helpers/t_helpers.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/system-tests/tests/test-helpers/t_helpers.go b/system-tests/tests/test-helpers/t_helpers.go index 19fe5eba175..e32757847a2 100644 --- a/system-tests/tests/test-helpers/t_helpers.go +++ b/system-tests/tests/test-helpers/t_helpers.go @@ -402,19 +402,19 @@ func workflowConfigFactory[T WorkflowConfig](t *testing.T, testLogger zerolog.Lo testLogger.Info().Msg("PoR v2 workflow config file created.") case *AptosReadWorkflowConfig: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create aptos read workflow config file") testLogger.Info().Msg("Aptos read workflow config file created.") case *aptoswrite_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create aptos write workflow config file") testLogger.Info().Msg("Aptos write workflow config file created.") case *aptoswriteroundtrip_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create aptos write roundtrip workflow config file") testLogger.Info().Msg("Aptos write roundtrip workflow config file created.") @@ -491,7 +491,7 @@ func workflowConfigFactory[T WorkflowConfig](t *testing.T, testLogger zerolog.Lo testLogger.Info().Msg("Solana write workflow config file created.") case *vaultsecret_config.Config: - workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg, outputDir) + workflowCfgFilePath, configErr := CreateWorkflowYamlConfigFile(workflowName, cfg) workflowConfigFilePath = workflowCfgFilePath require.NoError(t, configErr, "failed to create vaultsecret workflow config file") testLogger.Info().Msg("Vault secret workflow config file created.") From e072e2bb5cd4e06332040d559a067f5e5446ec07 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 14 Apr 2026 16:27:02 +0200 Subject: [PATCH 60/67] Carry PR 21641 test delta missing from PR 21642 --- .../executable/request/client_request_test.go | 76 ++++++++++--------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/core/capabilities/remote/executable/request/client_request_test.go b/core/capabilities/remote/executable/request/client_request_test.go index a871c2b6335..d35f8c11082 100644 --- a/core/capabilities/remote/executable/request/client_request_test.go +++ b/core/capabilities/remote/executable/request/client_request_test.go @@ -32,8 +32,6 @@ const ( workflowID1 = "15c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0" workflowExecutionID1 = "95ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0abbadeed" stepRef1 = "stepRef1" - - testDispatcherChanCap = 100 ) func Test_ClientRequest_MessageValidation(t *testing.T) { @@ -84,7 +82,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() capabilityPeers, capDonInfo, capInfo := capabilityDon(t, 2, 1) - dispatcher := newClientRequestTestDispatcher() + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") defer req.Cancel(errors.New("test end")) @@ -135,7 +133,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() capabilityPeers, capDonInfo, capInfo := capabilityDon(t, 2, 1) - dispatcher := newClientRequestTestDispatcher() + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") require.NoError(t, err) @@ -169,7 +167,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() capabilityPeers, capDonInfo, capInfo := capabilityDon(t, 2, 1) - dispatcher := newClientRequestTestDispatcher() + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") require.NoError(t, err) @@ -200,13 +198,15 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() capabilityPeers, capDonInfo, capInfo := capabilityDon(t, 4, 1) - dispatcher := newClientRequestTestDispatcher() + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") require.NoError(t, err) defer req.Cancel(errors.New("test end")) - drainInitialPeerSends(t, dispatcher, len(capabilityPeers)) + <-dispatcher.msgs + <-dispatcher.msgs + assert.Empty(t, dispatcher.msgs) msgWithError := &types.MessageBody{ CapabilityId: capInfo.ID, @@ -242,13 +242,15 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() capabilityPeers, capDonInfo, capInfo := capabilityDon(t, 4, 1) - dispatcher := newClientRequestTestDispatcher() + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") require.NoError(t, err) defer req.Cancel(errors.New("test end")) - drainInitialPeerSends(t, dispatcher, len(capabilityPeers)) + <-dispatcher.msgs + <-dispatcher.msgs + assert.Empty(t, dispatcher.msgs) serialized := caperrors.NewPublicUserError(errors.New("rpc error: EVM error invalid argument"), caperrors.FailedPrecondition).SerializeToRemoteString() msgWithError := &types.MessageBody{ @@ -286,13 +288,15 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() capabilityPeers, capDonInfo, capInfo := capabilityDon(t, 4, 1) - dispatcher := newClientRequestTestDispatcher() + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") require.NoError(t, err) defer req.Cancel(errors.New("test end")) - drainInitialPeerSends(t, dispatcher, len(capabilityPeers)) + <-dispatcher.msgs + <-dispatcher.msgs + assert.Empty(t, dispatcher.msgs) msgWithError := &types.MessageBody{ CapabilityId: capInfo.ID, @@ -345,13 +349,15 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() capabilityPeers, capDonInfo, capInfo := capabilityDon(t, 4, 1) - dispatcher := newClientRequestTestDispatcher() + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") require.NoError(t, err) defer req.Cancel(errors.New("test end")) - drainInitialPeerSends(t, dispatcher, len(capabilityPeers)) + <-dispatcher.msgs + <-dispatcher.msgs + assert.Empty(t, dispatcher.msgs) msg := &types.MessageBody{ CapabilityId: capInfo.ID, @@ -391,7 +397,9 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { // that the schedule is still executed entirely. cancelFn() - dispatcher := newClientRequestTestDispatcher() + // Buffered channel so the goroutines block + // when executing the schedule + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody)} req, err := request.NewClientExecuteRequest( ctxWithCancel, lggr, @@ -408,7 +416,10 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { // Despite the context being cancelled, // we still send the full schedule. - drainInitialPeerSends(t, dispatcher, len(capPeers)) + <-dispatcher.msgs + <-dispatcher.msgs + <-dispatcher.msgs + assert.Empty(t, dispatcher.msgs) msg := &types.MessageBody{ CapabilityId: capInfo.ID, @@ -505,7 +516,9 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx, cancelFn := context.WithTimeout(ctx, 15*time.Second) defer cancelFn() - dispatcher := newClientRequestTestDispatcher() + // Buffered channel so the goroutines block + // when executing the schedule + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody)} req, err := request.NewClientExecuteRequest( ctx, lggr, @@ -522,7 +535,10 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { // Despite the context being cancelled, // we still send the full schedule. - drainInitialPeerSends(t, dispatcher, len(capPeers)) + <-dispatcher.msgs + <-dispatcher.msgs + <-dispatcher.msgs + assert.Empty(t, dispatcher.msgs) msg := &types.MessageBody{ CapabilityId: capInfo.ID, @@ -603,7 +619,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() - dispatcher := newClientRequestTestDispatcher() + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") require.NoError(t, err) @@ -656,7 +672,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { beholderTester := beholdertest.NewObserver(t) lggr, obs := logger.TestObserved(t, zapcore.DebugLevel) capPeers, capDonInfo, capInfo := capabilityDon(t, 3, 1) - dispatcher := newClientRequestTestDispatcher() + dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody)} req, err := request.NewClientExecuteRequest( t.Context(), lggr, @@ -674,7 +690,11 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { require.NoError(t, err) defer req.Cancel(errors.New("test end")) - drainInitialPeerSends(t, dispatcher, len(capPeers)) + // Expect all 3 capability nodes to receive the request. + <-dispatcher.msgs + <-dispatcher.msgs + <-dispatcher.msgs + assert.Empty(t, dispatcher.msgs) msg := &types.MessageBody{ CapabilityId: capInfo.ID, @@ -734,22 +754,6 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { }) } -func newClientRequestTestDispatcher() *clientRequestTestDispatcher { - return &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, testDispatcherChanCap)} -} - -func drainInitialPeerSends(t *testing.T, d *clientRequestTestDispatcher, numCapabilityPeers int) { - t.Helper() - require.Eventually(t, func() bool { - return len(d.msgs) == numCapabilityPeers - }, 2*time.Second, time.Millisecond, "timed out waiting for %d buffered outbound messages", numCapabilityPeers) - require.Len(t, d.msgs, numCapabilityPeers, "dispatcher outbound buffer before draining initial peer sends") - for range numCapabilityPeers { - <-d.msgs - } - require.Empty(t, d.msgs) -} - func capabilityDon(t *testing.T, numCapabilityPeers int, f uint8) ([]p2ptypes.PeerID, commoncap.DON, commoncap.CapabilityInfo) { capabilityPeers := make([]p2ptypes.PeerID, numCapabilityPeers) for i := range numCapabilityPeers { From 352c8612d72ec87f162d7f2e5c3b7d5e8c635c81 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 15 Apr 2026 14:16:56 +0200 Subject: [PATCH 61/67] block user workflows from calling system-only capabilities Adds a deny-list check in ExecutionHelper.CallCapability() that prevents user workflow steps from invoking internal capabilities like confidential-workflows. The ConfidentialModule bypasses this gate because it calls the registry directly. --- .../workflows/v2/capability_executor.go | 17 +++++++++++++++++ .../workflows/v2/capability_executor_test.go | 16 ++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/core/services/workflows/v2/capability_executor.go b/core/services/workflows/v2/capability_executor.go index 13917d978a7..ab887b4577a 100644 --- a/core/services/workflows/v2/capability_executor.go +++ b/core/services/workflows/v2/capability_executor.go @@ -70,6 +70,13 @@ type capCall struct { // CallCapability handles requests generated by the wasm guest func (c *ExecutionHelper) CallCapability(ctx context.Context, request *sdkpb.CapabilityRequest) (*sdkpb.CapabilityResponse, error) { + // Block user workflows from calling system-only capabilities directly. + // These are internal plumbing (e.g. CRE -> enclave dispatch) and must not + // be invocable from workflow steps. + if isSystemCapability(request.Id) { + return nil, fmt.Errorf("capability %q is system-only and cannot be called from a workflow", request.Id) + } + capName, labels, _ := capabilities.ParseID(request.Id) if chainSelector, err := capabilities.ChainSelectorLabel(labels); err != nil { return nil, err @@ -287,3 +294,13 @@ func (c *ExecutionHelper) EmitUserMetric(ctx context.Context, metric *eventsv2.W loggerLabels := *c.loggerLabels.Load() return events.EmitUserMetric(ctx, loggerLabels, metric) } + +// systemCapabilities lists capability IDs that are internal plumbing and must +// not be callable from user workflow steps. +var systemCapabilities = map[string]bool{ + confidentialWorkflowsCapabilityID: true, +} + +func isSystemCapability(capID string) bool { + return systemCapabilities[capID] +} diff --git a/core/services/workflows/v2/capability_executor_test.go b/core/services/workflows/v2/capability_executor_test.go index 297da51bdbb..9a4d4335df4 100644 --- a/core/services/workflows/v2/capability_executor_test.go +++ b/core/services/workflows/v2/capability_executor_test.go @@ -14,6 +14,22 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/logger" ) +func TestExecutionHelper_SystemCapabilityBlocked(t *testing.T) { + t.Parallel() + + exec := &ExecutionHelper{} + + req := &sdk.CapabilityRequest{ + Id: confidentialWorkflowsCapabilityID, + Method: "Execute", + CallbackId: 1, + } + + _, err := exec.CallCapability(t.Context(), req) + require.Error(t, err) + assert.Contains(t, err.Error(), "system-only") +} + func TestExecutionHelper_ConfidentialHTTPPerWorkflowLimit(t *testing.T) { t.Parallel() From 463ff00dd4f36e3ddbd34646a1aca7e76db290b2 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Tue, 14 Apr 2026 16:44:22 +0200 Subject: [PATCH 62/67] Fix confidential relay service wiring to pass limits factory --- core/services/cre/cre.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index 143d6d40e29..e763a855faf 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -173,11 +173,12 @@ func (s *Services) newSubservices( s.GatewayConnectorWrapper = gatewayConnectorWrapper srvs = append(srvs, gatewayConnectorWrapper) - relayService := confidentialrelay.NewService( - gatewayConnectorWrapper, - opts.CapabilitiesRegistry, - lggr, - ) + relayService := confidentialrelay.NewService( + gatewayConnectorWrapper, + opts.CapabilitiesRegistry, + lggr, + opts.LimitsFactory, + ) srvs = append(srvs, relayService) } From 9c073ea753a28bc39d725f5f7fc14efd67948686 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Wed, 15 Apr 2026 17:24:15 +0200 Subject: [PATCH 63/67] gofmt core/services/cre/cre.go after relay limits factory fix --- core/services/cre/cre.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/services/cre/cre.go b/core/services/cre/cre.go index e763a855faf..1cce012b083 100644 --- a/core/services/cre/cre.go +++ b/core/services/cre/cre.go @@ -173,12 +173,12 @@ func (s *Services) newSubservices( s.GatewayConnectorWrapper = gatewayConnectorWrapper srvs = append(srvs, gatewayConnectorWrapper) - relayService := confidentialrelay.NewService( - gatewayConnectorWrapper, - opts.CapabilitiesRegistry, - lggr, - opts.LimitsFactory, - ) + relayService := confidentialrelay.NewService( + gatewayConnectorWrapper, + opts.CapabilitiesRegistry, + lggr, + opts.LimitsFactory, + ) srvs = append(srvs, relayService) } From ef57b772707acb5e4b170cf5f463abf8efef538e Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 16 Apr 2026 13:16:04 +0200 Subject: [PATCH 64/67] fix: restore relay handler and aggregator to develop versions Drop stale relay changes that diverged from develop. The aggregator non-determinism fix (#21907) and handler atomic cleanup are already on develop; no need to carry older versions in this PR. --- .../handlers/confidentialrelay/aggregator.go | 25 ++++- .../handlers/confidentialrelay/handler.go | 34 ++----- .../confidentialrelay/handler_test.go | 93 +++++++++++++++++-- 3 files changed, 116 insertions(+), 36 deletions(-) diff --git a/core/services/gateway/handlers/confidentialrelay/aggregator.go b/core/services/gateway/handlers/confidentialrelay/aggregator.go index 5eb00664571..657cc78c639 100644 --- a/core/services/gateway/handlers/confidentialrelay/aggregator.go +++ b/core/services/gateway/handlers/confidentialrelay/aggregator.go @@ -4,6 +4,8 @@ import ( "encoding/json" "errors" "fmt" + "maps" + "slices" jsonrpc "github.com/smartcontractkit/chainlink-common/pkg/jsonrpc2" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -41,8 +43,27 @@ func (a *aggregator) Aggregate(resps map[string]jsonrpc.Response[json.RawMessage if shaToCount[sha] > maxShaToCount { maxShaToCount = shaToCount[sha] } - if shaToCount[sha] >= requiredQuorum { - return &r, nil + } + + var qualifiedDigests []string + for sha, n := range shaToCount { + if n >= requiredQuorum { + qualifiedDigests = append(qualifiedDigests, sha) + } + } + if len(qualifiedDigests) > 0 { + slices.Sort(qualifiedDigests) + want := qualifiedDigests[0] + for _, k := range slices.Sorted(maps.Keys(resps)) { + r := resps[k] + sha, err := r.Digest() + if err != nil { + continue + } + if sha == want { + out := r + return &out, nil + } } } diff --git a/core/services/gateway/handlers/confidentialrelay/handler.go b/core/services/gateway/handlers/confidentialrelay/handler.go index 294a960d77e..3d99174da44 100644 --- a/core/services/gateway/handlers/confidentialrelay/handler.go +++ b/core/services/gateway/handlers/confidentialrelay/handler.go @@ -7,6 +7,7 @@ import ( "fmt" "strconv" "sync" + "sync/atomic" "time" "github.com/jonboulle/clockwork" @@ -353,19 +354,15 @@ func (h *handler) HandleNodeMessage(ctx context.Context, resp *jsonrpc.Response[ func (h *handler) fanOutToNodes(ctx context.Context, l logger.Logger, ar *activeRequest) error { var ( - group errgroup.Group - nodeErrors int - nodeErrorsMu sync.Mutex + group errgroup.Group + nodeErrors atomic.Uint32 ) for _, node := range h.donConfig.Members { - node := node group.Go(func() error { err := h.don.SendToNode(ctx, node.Address, &ar.req) if err != nil { - nodeErrorsMu.Lock() - nodeErrors++ - nodeErrorsMu.Unlock() + nodeErrors.Add(1) l.Errorw("error sending request to node", "node", node.Address, "error", err) } return nil @@ -374,7 +371,9 @@ func (h *handler) fanOutToNodes(ctx context.Context, l logger.Logger, ar *active _ = group.Wait() - if nodeErrors == len(h.donConfig.Members) && nodeErrors > 0 { + numNodeErrors := nodeErrors.Load() + remainingPossibleResponses := len(h.donConfig.Members) - int(numNodeErrors) + if remainingPossibleResponses < h.donConfig.F+1 && numNodeErrors > 0 { return h.sendResponseAndCleanup(ctx, ar, h.constructErrorResponse(ar.req, api.FatalError, errors.New("failed to forward user request to nodes"))) } @@ -404,19 +403,13 @@ func (h *handler) sendResponseAndCleanup(ctx context.Context, ar *activeRequest, } func (h *handler) recordMetrics(ctx context.Context, errorCode api.ErrorCode) { + //nolint:exhaustive // do not record other errors switch errorCode { - case api.StaleNodeResponseError: - case api.FatalError: - case api.NodeReponseEncodingError: - case api.RequestTimeoutError: case api.HandlerError: h.metrics.requestInternalError.Add(ctx, 1, metric.WithAttributes( attribute.String("don_id", h.donConfig.DonId), attribute.String("error", errorCode.String()), )) - case api.InvalidParamsError: - case api.UnsupportedMethodError: - case api.UserMessageParseError: case api.UnsupportedDONIdError: h.metrics.requestUserError.Add(ctx, 1, metric.WithAttributes( attribute.String("don_id", h.donConfig.DonId), @@ -425,12 +418,11 @@ func (h *handler) recordMetrics(ctx context.Context, errorCode api.ErrorCode) { h.metrics.requestSuccess.Add(ctx, 1, metric.WithAttributes( attribute.String("don_id", h.donConfig.DonId), )) - case api.ConflictError: - case api.LimitExceededError: } } func (h *handler) constructErrorResponse(req jsonrpc.Request[json.RawMessage], errorCode api.ErrorCode, err error) gwhandlers.UserCallbackPayload { + //nolint:exhaustive // do not modify other error codes switch errorCode { case api.NodeReponseEncodingError: err = errors.New(errorCode.String()) @@ -440,14 +432,6 @@ func (h *handler) constructErrorResponse(req jsonrpc.Request[json.RawMessage], e err = fmt.Errorf("unsupported method(%s): %w", req.Method, err) case api.UserMessageParseError: err = fmt.Errorf("user message parse error: %w", err) - case api.NoError: - case api.UnsupportedDONIdError: - case api.HandlerError: - case api.FatalError: - case api.RequestTimeoutError: - case api.StaleNodeResponseError: - case api.ConflictError: - case api.LimitExceededError: } return gwhandlers.UserCallbackPayload{ RawResponse: h.codec.EncodeNewErrorResponse( diff --git a/core/services/gateway/handlers/confidentialrelay/handler_test.go b/core/services/gateway/handlers/confidentialrelay/handler_test.go index c3c4ba20690..0cf3537405d 100644 --- a/core/services/gateway/handlers/confidentialrelay/handler_test.go +++ b/core/services/gateway/handlers/confidentialrelay/handler_test.go @@ -42,7 +42,7 @@ func newBarrierDON(total int) *barrierDON { } } -func (d *barrierDON) SendToNode(_ context.Context, _ string, _ *jsonrpc.Request[json.RawMessage]) error { +func (d *barrierDON) SendToNode(ctx context.Context, _ string, _ *jsonrpc.Request[json.RawMessage]) error { d.mu.Lock() d.started++ if d.started == d.total { @@ -51,12 +51,12 @@ func (d *barrierDON) SendToNode(_ context.Context, _ string, _ *jsonrpc.Request[ ch := d.allStarted d.mu.Unlock() - <-ch - return nil -} - -func (d *barrierDON) forceRelease() { - d.releaseOnce.Do(func() { close(d.allStarted) }) + select { + case <-ch: + return nil + case <-ctx.Done(): + return ctx.Err() + } } var nodeOne = config.NodeConfig{ @@ -521,6 +521,77 @@ func TestConfidentialRelayHandler_AllNodesFanOutFail(t *testing.T) { wg.Wait() } +func TestConfidentialRelayHandler_FanOutWaitsWhileQuorumStillPossible(t *testing.T) { + h, cb, don, _ := setupHandler(t, 4) + don.On("SendToNode", mock.Anything, mock.Anything, mock.Anything).Return( + func(_ context.Context, nodeAddress string, _ *jsonrpc.Request[json.RawMessage]) error { + switch nodeAddress { + case "0x0000", "0x0001": + return errors.New("connection refused") + default: + return nil + } + }, + ) + + params := json.RawMessage(`{"workflow_id":"wf1"}`) + req := jsonrpc.Request[json.RawMessage]{ + ID: "req-still-possible", + Method: MethodCapabilityExec, + Params: ¶ms, + } + + err := h.HandleJSONRPCUserMessage(t.Context(), req, cb) + require.NoError(t, err) + + require.NotNil(t, h.getActiveRequest(req.ID), "request should remain active while quorum is still possible") + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond) + defer cancel() + _, err = cb.Wait(ctx) + require.Error(t, err) +} + +func TestConfidentialRelayHandler_FanOutFailsWhenQuorumBecomesImpossible(t *testing.T) { + h, cb, don, _ := setupHandler(t, 4) + don.On("SendToNode", mock.Anything, mock.Anything, mock.Anything).Return( + func(_ context.Context, nodeAddress string, _ *jsonrpc.Request[json.RawMessage]) error { + switch nodeAddress { + case "0x0000", "0x0001", "0x0002": + return errors.New("connection refused") + default: + return nil + } + }, + ) + + params := json.RawMessage(`{"workflow_id":"wf1"}`) + req := jsonrpc.Request[json.RawMessage]{ + ID: "req-quorum-impossible", + Method: MethodCapabilityExec, + Params: ¶ms, + } + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + resp, err := cb.Wait(t.Context()) + assert.NoError(t, err) + assert.Equal(t, api.FatalError, resp.ErrorCode) + var jsonResp jsonrpc.Response[json.RawMessage] + err = json.Unmarshal(resp.RawResponse, &jsonResp) + assert.NoError(t, err) + assert.Contains(t, jsonResp.Error.Message, "failed to forward user request to nodes") + }() + + err := h.HandleJSONRPCUserMessage(t.Context(), req, cb) + require.NoError(t, err) + wg.Wait() + + require.Nil(t, h.getActiveRequest(req.ID), "request should be cleaned up once quorum is impossible") +} + func TestConfidentialRelayHandler_FanOutToNodes_IsConcurrent(t *testing.T) { lggr := logger.Test(t) don := newBarrierDON(2) @@ -550,16 +621,20 @@ func TestConfidentialRelayHandler_FanOutToNodes_IsConcurrent(t *testing.T) { Params: ¶ms, } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) go func() { - done <- h.HandleJSONRPCUserMessage(t.Context(), req, cb) + done <- h.HandleJSONRPCUserMessage(ctx, req, cb) }() select { case err := <-done: require.NoError(t, err) case <-time.After(100 * time.Millisecond): - don.forceRelease() + cancel() + <-done t.Fatal("HandleJSONRPCUserMessage did not fan out to nodes concurrently") } From 00f883ed9b42afc4f90ef13c050c6ce987b265f4 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 16 Apr 2026 13:17:50 +0200 Subject: [PATCH 65/67] chore: rename confidential_relay to confidentialrelay --- .../confidentialrelay.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename system-tests/lib/cre/features/{confidential_relay/confidential_relay.go => confidentialrelay/confidentialrelay.go} (100%) diff --git a/system-tests/lib/cre/features/confidential_relay/confidential_relay.go b/system-tests/lib/cre/features/confidentialrelay/confidentialrelay.go similarity index 100% rename from system-tests/lib/cre/features/confidential_relay/confidential_relay.go rename to system-tests/lib/cre/features/confidentialrelay/confidentialrelay.go From 87ed27e3eec90a7694537986f17520185e8a918b Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 16 Apr 2026 13:24:19 +0200 Subject: [PATCH 66/67] revert: restore client_request_test.go to develop version Drop unrelated test refactoring (helper function inlining) that was not part of the confidential relay changes. --- .../executable/request/client_request_test.go | 76 +++++++++---------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/core/capabilities/remote/executable/request/client_request_test.go b/core/capabilities/remote/executable/request/client_request_test.go index d35f8c11082..a871c2b6335 100644 --- a/core/capabilities/remote/executable/request/client_request_test.go +++ b/core/capabilities/remote/executable/request/client_request_test.go @@ -32,6 +32,8 @@ const ( workflowID1 = "15c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0" workflowExecutionID1 = "95ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0abbadeed" stepRef1 = "stepRef1" + + testDispatcherChanCap = 100 ) func Test_ClientRequest_MessageValidation(t *testing.T) { @@ -82,7 +84,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() capabilityPeers, capDonInfo, capInfo := capabilityDon(t, 2, 1) - dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} + dispatcher := newClientRequestTestDispatcher() req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") defer req.Cancel(errors.New("test end")) @@ -133,7 +135,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() capabilityPeers, capDonInfo, capInfo := capabilityDon(t, 2, 1) - dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} + dispatcher := newClientRequestTestDispatcher() req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") require.NoError(t, err) @@ -167,7 +169,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() capabilityPeers, capDonInfo, capInfo := capabilityDon(t, 2, 1) - dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} + dispatcher := newClientRequestTestDispatcher() req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") require.NoError(t, err) @@ -198,15 +200,13 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() capabilityPeers, capDonInfo, capInfo := capabilityDon(t, 4, 1) - dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} + dispatcher := newClientRequestTestDispatcher() req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") require.NoError(t, err) defer req.Cancel(errors.New("test end")) - <-dispatcher.msgs - <-dispatcher.msgs - assert.Empty(t, dispatcher.msgs) + drainInitialPeerSends(t, dispatcher, len(capabilityPeers)) msgWithError := &types.MessageBody{ CapabilityId: capInfo.ID, @@ -242,15 +242,13 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() capabilityPeers, capDonInfo, capInfo := capabilityDon(t, 4, 1) - dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} + dispatcher := newClientRequestTestDispatcher() req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") require.NoError(t, err) defer req.Cancel(errors.New("test end")) - <-dispatcher.msgs - <-dispatcher.msgs - assert.Empty(t, dispatcher.msgs) + drainInitialPeerSends(t, dispatcher, len(capabilityPeers)) serialized := caperrors.NewPublicUserError(errors.New("rpc error: EVM error invalid argument"), caperrors.FailedPrecondition).SerializeToRemoteString() msgWithError := &types.MessageBody{ @@ -288,15 +286,13 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() capabilityPeers, capDonInfo, capInfo := capabilityDon(t, 4, 1) - dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} + dispatcher := newClientRequestTestDispatcher() req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") require.NoError(t, err) defer req.Cancel(errors.New("test end")) - <-dispatcher.msgs - <-dispatcher.msgs - assert.Empty(t, dispatcher.msgs) + drainInitialPeerSends(t, dispatcher, len(capabilityPeers)) msgWithError := &types.MessageBody{ CapabilityId: capInfo.ID, @@ -349,15 +345,13 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() capabilityPeers, capDonInfo, capInfo := capabilityDon(t, 4, 1) - dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} + dispatcher := newClientRequestTestDispatcher() req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") require.NoError(t, err) defer req.Cancel(errors.New("test end")) - <-dispatcher.msgs - <-dispatcher.msgs - assert.Empty(t, dispatcher.msgs) + drainInitialPeerSends(t, dispatcher, len(capabilityPeers)) msg := &types.MessageBody{ CapabilityId: capInfo.ID, @@ -397,9 +391,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { // that the schedule is still executed entirely. cancelFn() - // Buffered channel so the goroutines block - // when executing the schedule - dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody)} + dispatcher := newClientRequestTestDispatcher() req, err := request.NewClientExecuteRequest( ctxWithCancel, lggr, @@ -416,10 +408,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { // Despite the context being cancelled, // we still send the full schedule. - <-dispatcher.msgs - <-dispatcher.msgs - <-dispatcher.msgs - assert.Empty(t, dispatcher.msgs) + drainInitialPeerSends(t, dispatcher, len(capPeers)) msg := &types.MessageBody{ CapabilityId: capInfo.ID, @@ -516,9 +505,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx, cancelFn := context.WithTimeout(ctx, 15*time.Second) defer cancelFn() - // Buffered channel so the goroutines block - // when executing the schedule - dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody)} + dispatcher := newClientRequestTestDispatcher() req, err := request.NewClientExecuteRequest( ctx, lggr, @@ -535,10 +522,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { // Despite the context being cancelled, // we still send the full schedule. - <-dispatcher.msgs - <-dispatcher.msgs - <-dispatcher.msgs - assert.Empty(t, dispatcher.msgs) + drainInitialPeerSends(t, dispatcher, len(capPeers)) msg := &types.MessageBody{ CapabilityId: capInfo.ID, @@ -619,7 +603,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { ctx := t.Context() - dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, 100)} + dispatcher := newClientRequestTestDispatcher() req, err := request.NewClientExecuteRequest(ctx, logger.Test(t), capabilityRequest, capInfo, workflowDonInfo, dispatcher, 10*time.Minute, nil, "") require.NoError(t, err) @@ -672,7 +656,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { beholderTester := beholdertest.NewObserver(t) lggr, obs := logger.TestObserved(t, zapcore.DebugLevel) capPeers, capDonInfo, capInfo := capabilityDon(t, 3, 1) - dispatcher := &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody)} + dispatcher := newClientRequestTestDispatcher() req, err := request.NewClientExecuteRequest( t.Context(), lggr, @@ -690,11 +674,7 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { require.NoError(t, err) defer req.Cancel(errors.New("test end")) - // Expect all 3 capability nodes to receive the request. - <-dispatcher.msgs - <-dispatcher.msgs - <-dispatcher.msgs - assert.Empty(t, dispatcher.msgs) + drainInitialPeerSends(t, dispatcher, len(capPeers)) msg := &types.MessageBody{ CapabilityId: capInfo.ID, @@ -754,6 +734,22 @@ func Test_ClientRequest_MessageValidation(t *testing.T) { }) } +func newClientRequestTestDispatcher() *clientRequestTestDispatcher { + return &clientRequestTestDispatcher{msgs: make(chan *types.MessageBody, testDispatcherChanCap)} +} + +func drainInitialPeerSends(t *testing.T, d *clientRequestTestDispatcher, numCapabilityPeers int) { + t.Helper() + require.Eventually(t, func() bool { + return len(d.msgs) == numCapabilityPeers + }, 2*time.Second, time.Millisecond, "timed out waiting for %d buffered outbound messages", numCapabilityPeers) + require.Len(t, d.msgs, numCapabilityPeers, "dispatcher outbound buffer before draining initial peer sends") + for range numCapabilityPeers { + <-d.msgs + } + require.Empty(t, d.msgs) +} + func capabilityDon(t *testing.T, numCapabilityPeers int, f uint8) ([]p2ptypes.PeerID, commoncap.DON, commoncap.CapabilityInfo) { capabilityPeers := make([]p2ptypes.PeerID, numCapabilityPeers) for i := range numCapabilityPeers { From 940d1ecd6d9084ad7e816f2f7be14891efff3420 Mon Sep 17 00:00:00 2001 From: Tejaswi Nadahalli Date: Thu, 16 Apr 2026 13:46:34 +0200 Subject: [PATCH 67/67] fix comments: BinaryURLResolver is nil by design (PRIV-389) Enclave fetches binaries via a credentials sidecar, not via node-side presigned URLs (which would break consensus). --- core/services/workflows/syncer/v2/handler.go | 5 +++-- core/services/workflows/v2/confidential_module.go | 7 +++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 36148e7477f..3d82fa9ee07 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -945,8 +945,9 @@ func (h *eventHandler) confidentialEngineFactory( lggr := logger.Named(h.lggr, "WorkflowEngine.ConfidentialModule") lggr = logger.With(lggr, "workflowID", spec.WorkflowID, "workflowName", spec.WorkflowName, "workflowOwner", spec.WorkflowOwner) - // nil resolver: raw binaryURL is passed to the enclave as-is. - // TODO: wire to storage service retriever so the enclave receives a presigned URL. + // nil resolver: raw binaryURL (an identifier, not directly fetchable) is passed + // to the enclave as-is. The enclave fetches the binary via a credentials sidecar + // that authenticates to the storage service directly (PRIV-389). module := v2.NewConfidentialModule( h.capRegistry, spec.BinaryURL, diff --git a/core/services/workflows/v2/confidential_module.go b/core/services/workflows/v2/confidential_module.go index 6c27ea6c7f1..db9f7522e07 100644 --- a/core/services/workflows/v2/confidential_module.go +++ b/core/services/workflows/v2/confidential_module.go @@ -58,10 +58,9 @@ func IsConfidential(data []byte) (bool, error) { return attrs.Confidential, nil } -// BinaryURLResolver resolves a raw binary URL into an ephemeral/presigned -// URL that the enclave can fetch without authentication. In production this -// calls the CRE storage service; nil means the raw URL is used as-is. -// PR 5/5 (#21642) wires this to the storage service retriever. +// BinaryURLResolver resolves a raw binary URL into a fetchable URL. +// nil means the raw binaryURL is passed to the enclave as-is; the enclave +// authenticates to the storage service via a credentials sidecar (PRIV-389). type BinaryURLResolver func(ctx context.Context, workflowID string) (string, error) // ConfidentialModule implements host.ModuleV2 for confidential workflows.