diff --git a/deployment/cre/capabilities_registry/v2/changeset/add_capabilities.go b/deployment/cre/capabilities_registry/v2/changeset/add_capabilities.go index 9e4621bdc0e..ac42c65f148 100644 --- a/deployment/cre/capabilities_registry/v2/changeset/add_capabilities.go +++ b/deployment/cre/capabilities_registry/v2/changeset/add_capabilities.go @@ -43,6 +43,11 @@ type AddCapabilitiesInput struct { // SkipDuplicateCapabilityValidation disables the precondition check that rejects assigning // the same capability to multiple DONs within a DON family. SkipDuplicateCapabilityValidation bool `json:"skipDuplicateCapabilityValidation,omitempty" yaml:"skipDuplicateCapabilityValidation,omitempty"` + + // SkipSignerFamilyJobCheck disables the precondition check that, when this change adds + // extraSignerFamilies to an OCR3 config, every node already has an APPROVED consensus job with a + // key bundle for each family. Use only for emergencies. + SkipSignerFamilyJobCheck bool `json:"skipSignerFamilyJobCheck,omitempty" yaml:"skipSignerFamilyJobCheck,omitempty"` } type AddCapabilities struct{} @@ -71,6 +76,14 @@ func (u AddCapabilities) VerifyPreconditions(e cldf.Environment, config AddCapab } } + if !config.SkipSignerFamilyJobCheck { + for donName, donCapConfigs := range config.DonCapabilityConfigs { + if err := verifyDONSignerFamilies(e, config.RegistryChainSel, config.RegistryQualifier, donName, donCapConfigs, nil); err != nil { + return fmt.Errorf("AddCapabilities precondition failed: %w", err) + } + } + } + return nil } diff --git a/deployment/cre/capabilities_registry/v2/changeset/configure_capabilities_registry.go b/deployment/cre/capabilities_registry/v2/changeset/configure_capabilities_registry.go index 2aed23729e4..26fcc676a63 100644 --- a/deployment/cre/capabilities_registry/v2/changeset/configure_capabilities_registry.go +++ b/deployment/cre/capabilities_registry/v2/changeset/configure_capabilities_registry.go @@ -38,6 +38,11 @@ type ConfigureCapabilitiesRegistryInput struct { Nodes []CapabilitiesRegistryNodeParams `json:"nodes,omitempty" yaml:"nodes,omitempty"` DONs []CapabilitiesRegistryNewDONParams `json:"dons,omitempty" yaml:"dons,omitempty"` Qualifier string `json:"qualifier,omitempty" yaml:"qualifier,omitempty"` + + // SkipSignerFamilyJobCheck disables the precondition check that, when this change adds + // extraSignerFamilies to an OCR3 config, every node already has an APPROVED consensus job with a + // key bundle for each family. Use only for emergencies. + SkipSignerFamilyJobCheck bool `json:"skipSignerFamilyJobCheck,omitempty" yaml:"skipSignerFamilyJobCheck,omitempty"` } type ConfigureCapabilitiesRegistryDeps struct { @@ -55,6 +60,22 @@ func (l ConfigureCapabilitiesRegistry) VerifyPreconditions(e cldf.Environment, c return fmt.Errorf("chain %d not found in environment", config.ChainSelector) } + if !config.SkipSignerFamilyJobCheck { + for _, don := range config.DONs { + capConfigs := make([]ocr3CapConfig, len(don.CapabilityConfigurations)) + for i, cc := range don.CapabilityConfigurations { + capConfigs[i] = ocr3CapConfig(cc) + } + families, err := requestedSignerFamilies(capConfigs) + if err != nil { + return fmt.Errorf("ConfigureCapabilitiesRegistry precondition failed: DON %q: %w", don.Name, err) + } + if err := verifySignerFamilyJobs(e, don.Nodes, families); err != nil { + return fmt.Errorf("ConfigureCapabilitiesRegistry precondition failed: DON %q: %w", don.Name, err) + } + } + } + return nil } diff --git a/deployment/cre/capabilities_registry/v2/changeset/signer_family_changesets_test.go b/deployment/cre/capabilities_registry/v2/changeset/signer_family_changesets_test.go new file mode 100644 index 00000000000..e26015aacfb --- /dev/null +++ b/deployment/cre/capabilities_registry/v2/changeset/signer_family_changesets_test.go @@ -0,0 +1,79 @@ +package changeset_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + cldf_offchain "github.com/smartcontractkit/chainlink-deployments-framework/offchain" + nodev1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/node" + + "github.com/smartcontractkit/chainlink/deployment/cre/capabilities_registry/v2/changeset" + "github.com/smartcontractkit/chainlink/deployment/cre/capabilities_registry/v2/changeset/operations/contracts" + "github.com/smartcontractkit/chainlink/deployment/cre/test" +) + +// emptyJD knows no nodes, so no node can have an accepted job. +type emptyJD struct{ cldf_offchain.Client } + +func (emptyJD) ListNodes(context.Context, *nodev1.ListNodesRequest, ...grpc.CallOption) (*nodev1.ListNodesResponse, error) { + return &nodev1.ListNodesResponse{}, nil +} + +func stellarOCR3Config() map[string]any { + return map[string]any{"ocr3Configs": map[string]any{"__default__": map[string]any{ + "offchainConfig": map[string]any{"maxFaultyOracles": 1}, + "extraSignerFamilies": []any{"stellar"}, + }}} +} + +func TestAddCapabilities_VerifyPreconditions_SignerFamilyJobCheck(t *testing.T) { + h := test.NewTestHarness(t) + + e := h.Runtime.Environment() + e.Offchain = emptyJD{} + + in := changeset.AddCapabilitiesInput{ + RegistryChainSel: h.RegistrySelector, + RegistryQualifier: test.RegistryQualifier, + DonCapabilityConfigs: map[string][]contracts.CapabilityConfig{ + test.DONName: {{Capability: contracts.Capability{CapabilityID: "consensus@1.0.0"}, Config: stellarOCR3Config()}}, + }, + SkipDuplicateCapabilityValidation: true, + } + + require.ErrorContains(t, changeset.AddCapabilities{}.VerifyPreconditions(e, in), "not in JD") + + missing := in + missing.DonCapabilityConfigs = map[string][]contracts.CapabilityConfig{"no-such-don": in.DonCapabilityConfigs[test.DONName]} + require.ErrorContains(t, changeset.AddCapabilities{}.VerifyPreconditions(e, missing), "not found on-chain") + + in.SkipSignerFamilyJobCheck = true + require.NoError(t, changeset.AddCapabilities{}.VerifyPreconditions(e, in)) +} + +func TestConfigureCapabilitiesRegistry_VerifyPreconditions_SignerFamilyJobCheck(t *testing.T) { + h := test.NewTestHarness(t) + + e := h.Runtime.Environment() + e.Offchain = emptyJD{} + + in := changeset.ConfigureCapabilitiesRegistryInput{ + ChainSelector: h.RegistrySelector, + Qualifier: test.RegistryQualifier, + DONs: []changeset.CapabilitiesRegistryNewDONParams{{ + Name: "new-don", + Nodes: h.Don.GetP2PIDs().Strings(), + CapabilityConfigurations: []changeset.CapabilitiesRegistryCapabilityConfiguration{ + {CapabilityID: "consensus@1.0.0", Config: stellarOCR3Config()}, + }, + }}, + } + + require.ErrorContains(t, changeset.ConfigureCapabilitiesRegistry{}.VerifyPreconditions(e, in), "not in JD") + + in.SkipSignerFamilyJobCheck = true + require.NoError(t, changeset.ConfigureCapabilitiesRegistry{}.VerifyPreconditions(e, in)) +} diff --git a/deployment/cre/capabilities_registry/v2/changeset/signer_family_gate.go b/deployment/cre/capabilities_registry/v2/changeset/signer_family_gate.go new file mode 100644 index 00000000000..f871a1c09b7 --- /dev/null +++ b/deployment/cre/capabilities_registry/v2/changeset/signer_family_gate.go @@ -0,0 +1,271 @@ +package changeset + +import ( + "fmt" + "slices" + "strings" + + "github.com/pelletier/go-toml/v2" + "google.golang.org/grpc" + + "github.com/smartcontractkit/chainlink-common/keystore/corekeys/p2pkey" + cldf "github.com/smartcontractkit/chainlink-deployments-framework/deployment" + capabilities_registry_v2 "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/capabilities_registry_wrapper_v2" + jobv1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/job" + nodev1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/node" + "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/shared/ptypes" + + "github.com/smartcontractkit/chainlink/deployment/cre/capabilities_registry/v2/changeset/operations/contracts" + "github.com/smartcontractkit/chainlink/deployment/cre/capabilities_registry/v2/changeset/sequences" + "github.com/smartcontractkit/chainlink/deployment/cre/pkg/offchain" +) + +const ( + jdP2PIDLabel = "p2p_id" + + skipSignerFamilyJobCheckHint = "set skipSignerFamilyJobCheck to bypass" +) + +// requestedSignerFamilies returns the sorted, de-duplicated extraSignerFamilies of every OCR3 config +// entry that this change will generate. Entries whose offchainConfig is already processed are not +// regenerated, so they request nothing. +func requestedSignerFamilies(capConfigs []ocr3CapConfig) ([]string, error) { + seen := map[string]bool{} + for _, capCfg := range capConfigs { + for key, entryCfg := range extractOCR3Configs(capCfg.Config) { + entryMap, ok := entryCfg.(map[string]any) + if !ok { + continue + } + if _, isMap := entryMap["offchainConfig"].(map[string]any); !isMap { + continue + } + families, err := parseExtraSignerFamilies(entryMap) + if err != nil { + return nil, fmt.Errorf("capability %q, ocr3Configs[%q].extraSignerFamilies: %w", capCfg.CapabilityID, key, err) + } + for _, f := range families { + seen[f] = true + } + } + } + out := make([]string, 0, len(seen)) + for f := range seen { + out = append(out, f) + } + slices.Sort(out) + + return out, nil +} + +// donNodeP2PIDs returns the "p2p_" IDs of a DON's on-chain nodes. +func donNodeP2PIDs(don *capabilities_registry_v2.CapabilitiesRegistryDONInfo) []string { + ids := make([]string, len(don.NodeP2PIds)) + for i, id := range don.NodeP2PIds { + ids[i] = p2pkey.PeerID(id).String() + } + + return ids +} + +// consensusSpec is the part of a standard-capability job spec that carries the per-family key bundles. +type consensusSpec struct { + Type string `toml:"type"` + Command string `toml:"command"` + OracleFactory struct { + OnchainSigningStrategy struct { + Config map[string]string `toml:"config"` + } `toml:"onchainSigningStrategy"` + } `toml:"oracle_factory"` +} + +// consensusSpecFamilies returns the chain families a consensus job spec has a key bundle for. Other +// job specs, and specs that do not parse, yield none. The consensus job is the standard capability +// whose command is the consensus binary. +func consensusSpecFamilies(spec string) map[string]bool { + var s consensusSpec + if err := toml.Unmarshal([]byte(spec), &s); err != nil { + return nil + } + if s.Type != "standardcapabilities" || !strings.HasSuffix(s.Command, "consensus") { + return nil + } + out := map[string]bool{} + for family, bundleID := range s.OracleFactory.OnchainSigningStrategy.Config { + if bundleID != "" { + out[family] = true + } + } + + return out +} + +// verifySignerFamilyJobs fails unless every node in p2pIDs has an APPROVED consensus job signing for +// every family in families. Adding a signer family to the consensus OCR3 config before the nodes run +// a job holding that family's key bundle halts OCR (INCIDENT-2694). Nodes JD does not know cannot +// have accepted anything, so they fail the check too. It runs at proposal time, not MCMS execution. +func verifySignerFamilyJobs(e cldf.Environment, p2pIDs, families []string) error { + if len(families) == 0 || len(p2pIDs) == 0 { + return nil + } + if e.Offchain == nil { + return fmt.Errorf("signer family job check needs a job distributor client; %s", skipSignerFamilyJobCheckHint) + } + + jdNodes, err := offchain.FetchNodesFromJD(e.GetContext(), e.Offchain, &nodev1.ListNodesRequest_Filter{ + Selectors: []*ptypes.Selector{{Key: jdP2PIDLabel, Op: ptypes.SelectorOp_EXIST}}, + }) + if err != nil { + return fmt.Errorf("signer family job check: %w", err) + } + + // A peer can have several JD records (e.g. re-registered with a new CSA key); any may hold the job. + nodeIDsByP2P := make(map[string][]string, len(jdNodes)) + nodeNameByID := make(map[string]string, len(jdNodes)) + for _, n := range jdNodes { + for _, l := range n.Labels { + if l.Key == jdP2PIDLabel && l.Value != nil { + nodeIDsByP2P[normalizeP2PID(*l.Value)] = append(nodeIDsByP2P[normalizeP2PID(*l.Value)], n.Id) + nodeNameByID[n.Id] = n.Name + } + } + } + + var allNodeIDs []string + for _, p := range p2pIDs { + allNodeIDs = append(allNodeIDs, nodeIDsByP2P[normalizeP2PID(p)]...) + } + covered, err := approvedConsensusFamilies(e, allNodeIDs) + if err != nil { + return err + } + + var problems []string + for _, p := range p2pIDs { + ids := nodeIDsByP2P[normalizeP2PID(p)] + if len(ids) == 0 { + problems = append(problems, p+" (not in JD)") + continue + } + var missing []string + for _, f := range families { + if !slices.ContainsFunc(ids, func(id string) bool { return covered[id][f] }) { + missing = append(missing, f) + } + } + if len(missing) > 0 { + problems = append(problems, fmt.Sprintf("%s (%s)", nodeNameByID[ids[0]], strings.Join(missing, ", "))) + } + } + if len(problems) == 0 { + return nil + } + slices.Sort(problems) + + return fmt.Errorf( + "consensus config requests signer families %v, but these nodes have no APPROVED consensus job with a key bundle for them: %s; "+ + "propose the consensus job and have every NOP accept it before changing the capabilities registry (see INCIDENT-2694); %s", + families, strings.Join(problems, "; "), skipSignerFamilyJobCheckHint, + ) +} + +// verifyDONSignerFamilies gates one DON: the families its capability configs request must be signed for +// by an APPROVED consensus job on each node. nodes overrides the DON's current on-chain node set. +func verifyDONSignerFamilies( + e cldf.Environment, chainSel uint64, qualifier, donName string, capConfigs []contracts.CapabilityConfig, nodes []string, +) error { + ocr3Configs := make([]ocr3CapConfig, len(capConfigs)) + for i, cc := range capConfigs { + ocr3Configs[i] = ocr3CapConfig{CapabilityID: cc.Capability.CapabilityID, Config: cc.Config} + } + families, err := requestedSignerFamilies(ocr3Configs) + if err != nil || len(families) == 0 { + return err + } + + if len(nodes) == 0 { + capReg, err := capRegForPreconditionCheck(e, chainSel, qualifier) + if err != nil { + return err + } + // By name, not the first-32 DON list, so DONs beyond that window still resolve. + don, _, err := sequences.GetDonNodes(donName, capReg) + if err != nil { + return fmt.Errorf("DON %q not found on-chain: %w", donName, err) + } + nodes = donNodeP2PIDs(don) + } + + return verifySignerFamilyJobs(e, nodes, families) +} + +// normalizeP2PID maps a bare-hex or "p2p_"-prefixed peer ID to the "p2p_" form JD labels use. +func normalizeP2PID(p string) string { + if id, err := p2pkey.MakePeerID(p); err == nil { + return id.String() + } + + return p +} + +// approvedConsensusFamilies maps each JD node ID to the families its live consensus jobs sign for, +// counting only a job's latest revision and only when that revision is APPROVED. +func approvedConsensusFamilies(e cldf.Environment, nodeIDs []string) (map[string]map[string]bool, error) { + covered := map[string]map[string]bool{} + if len(nodeIDs) == 0 { + return covered, nil + } + + const maxRecvSize = 50 * 1024 * 1024 + jobsResp, err := e.Offchain.ListJobs(e.GetContext(), &jobv1.ListJobsRequest{ + Filter: &jobv1.ListJobsRequest_Filter{NodeIds: nodeIDs}, + }, grpc.MaxCallRecvMsgSize(maxRecvSize)) + if err != nil { + return nil, fmt.Errorf("signer family job check: failed to list jobs: %w", err) + } + + nodeByJob := make(map[string]string) + jobIDs := make([]string, 0, len(jobsResp.Jobs)) + for _, j := range jobsResp.Jobs { + if j.DeletedAt != nil { + continue + } + nodeByJob[j.Id] = j.NodeId + jobIDs = append(jobIDs, j.Id) + } + if len(jobIDs) == 0 { + return covered, nil + } + + propResp, err := e.Offchain.ListProposals(e.GetContext(), &jobv1.ListProposalsRequest{ + Filter: &jobv1.ListProposalsRequest_Filter{JobIds: jobIDs}, + }, grpc.MaxCallRecvMsgSize(maxRecvSize)) + if err != nil { + return nil, fmt.Errorf("signer family job check: failed to list proposals: %w", err) + } + + latest := make(map[string]*jobv1.Proposal) + for _, p := range propResp.Proposals { + if _, live := nodeByJob[p.JobId]; !live { + continue + } + if cur, ok := latest[p.JobId]; !ok || p.Revision > cur.Revision { + latest[p.JobId] = p + } + } + + for jobID, p := range latest { + if p.Status != jobv1.ProposalStatus_PROPOSAL_STATUS_APPROVED { + continue + } + node := nodeByJob[jobID] + for f := range consensusSpecFamilies(p.Spec) { + if covered[node] == nil { + covered[node] = map[string]bool{} + } + covered[node][f] = true + } + } + + return covered, nil +} diff --git a/deployment/cre/capabilities_registry/v2/changeset/signer_family_gate_test.go b/deployment/cre/capabilities_registry/v2/changeset/signer_family_gate_test.go new file mode 100644 index 00000000000..74ddc71e0d3 --- /dev/null +++ b/deployment/cre/capabilities_registry/v2/changeset/signer_family_gate_test.go @@ -0,0 +1,311 @@ +package changeset + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/smartcontractkit/chainlink-common/keystore/corekeys/p2pkey" + cldf "github.com/smartcontractkit/chainlink-deployments-framework/deployment" + cldf_offchain "github.com/smartcontractkit/chainlink-deployments-framework/offchain" + capabilities_registry_v2 "github.com/smartcontractkit/chainlink-evm/gethwrappers/workflow/generated/capabilities_registry_wrapper_v2" + jobv1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/job" + nodev1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/node" + "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/shared/ptypes" + + "github.com/smartcontractkit/chainlink/deployment/cre/capabilities_registry/v2/changeset/operations/contracts" +) + +type fakeJD struct { + cldf_offchain.Client + nodes []*nodev1.Node + jobs []*jobv1.Job + proposals []*jobv1.Proposal + jobsErr error + propsErr error +} + +func (f *fakeJD) ListNodes(context.Context, *nodev1.ListNodesRequest, ...grpc.CallOption) (*nodev1.ListNodesResponse, error) { + return &nodev1.ListNodesResponse{Nodes: f.nodes}, nil +} + +func (f *fakeJD) ListJobs(context.Context, *jobv1.ListJobsRequest, ...grpc.CallOption) (*jobv1.ListJobsResponse, error) { + if f.jobsErr != nil { + return nil, f.jobsErr + } + + return &jobv1.ListJobsResponse{Jobs: f.jobs}, nil +} + +func (f *fakeJD) ListProposals(context.Context, *jobv1.ListProposalsRequest, ...grpc.CallOption) (*jobv1.ListProposalsResponse, error) { + if f.propsErr != nil { + return nil, f.propsErr + } + + return &jobv1.ListProposalsResponse{Proposals: f.proposals}, nil +} + +func jdNode(id, name, p2p string) *nodev1.Node { + return &nodev1.Node{Id: id, Name: name, Labels: []*ptypes.Label{{Key: jdP2PIDLabel, Value: &p2p}}} +} + +func envWith(jd cldf_offchain.Client) cldf.Environment { + return cldf.Environment{Offchain: jd, GetContext: context.Background} +} + +// consensusSpecWith renders a consensus job spec with a key bundle per family. +func consensusSpecWith(families ...string) string { + var sb strings.Builder + sb.WriteString("type = \"standardcapabilities\"\nname = \"OCR-Consensus-CRE (node)\"\ncommand = \"/usr/local/bin/consensus\"\n" + + "[oracle_factory]\nenabled = true\n[oracle_factory.onchainSigningStrategy]\nstrategyName = \"multi-chain\"\n" + + "[oracle_factory.onchainSigningStrategy.config]\nevm = \"bundle-evm\"\n") + for _, f := range families { + fmt.Fprintf(&sb, "%s = \"bundle-%s\"\n", f, f) + } + + return sb.String() +} + +func proposal(job string, rev int64, st jobv1.ProposalStatus, spec string) *jobv1.Proposal { + return &jobv1.Proposal{Id: job + "-p", JobId: job, Revision: rev, Status: st, Spec: spec} +} + +func TestVerifySignerFamilyJobs(t *testing.T) { + t.Parallel() + + approved := jobv1.ProposalStatus_PROPOSAL_STATUS_APPROVED + proposed := jobv1.ProposalStatus_PROPOSAL_STATUS_PROPOSED + stellar := []string{"stellar"} + + base := func() *fakeJD { + return &fakeJD{ + nodes: []*nodev1.Node{jdNode("n1", "node-1", "p2p_a"), jdNode("n2", "node-2", "p2p_b")}, + jobs: []*jobv1.Job{{Id: "j1", NodeId: "n1"}, {Id: "j2", NodeId: "n2"}}, + } + } + + t.Run("no families is a no-op without JD", func(t *testing.T) { + t.Parallel() + require.NoError(t, verifySignerFamilyJobs(cldf.Environment{}, []string{"p2p_a"}, nil)) + }) + + t.Run("missing JD client errors", func(t *testing.T) { + t.Parallel() + err := verifySignerFamilyJobs(cldf.Environment{}, []string{"p2p_a"}, stellar) + require.ErrorContains(t, err, "job distributor client") + }) + + t.Run("approved consensus job with the family passes", func(t *testing.T) { + t.Parallel() + jd := base() + jd.proposals = []*jobv1.Proposal{ + proposal("j1", 1, approved, consensusSpecWith("stellar")), + proposal("j2", 1, approved, consensusSpecWith("stellar")), + } + require.NoError(t, verifySignerFamilyJobs(envWith(jd), []string{"p2p_a", "p2p_b"}, stellar)) + }) + + t.Run("job not proposed at all blocks (INCIDENT-2694)", func(t *testing.T) { + t.Parallel() + jd := base() + err := verifySignerFamilyJobs(envWith(jd), []string{"p2p_a"}, stellar) + require.ErrorContains(t, err, "node-1 (stellar)") + }) + + t.Run("proposed but not accepted blocks", func(t *testing.T) { + t.Parallel() + jd := base() + jd.proposals = []*jobv1.Proposal{proposal("j1", 1, proposed, consensusSpecWith("stellar"))} + require.ErrorContains(t, verifySignerFamilyJobs(envWith(jd), []string{"p2p_a"}, stellar), "node-1") + }) + + t.Run("only the failing node is named", func(t *testing.T) { + t.Parallel() + jd := base() + jd.proposals = []*jobv1.Proposal{ + proposal("j1", 1, approved, consensusSpecWith("stellar")), + proposal("j2", 1, proposed, consensusSpecWith("stellar")), + } + err := verifySignerFamilyJobs(envWith(jd), []string{"p2p_a", "p2p_b"}, stellar) + require.ErrorContains(t, err, "node-2") + require.NotContains(t, err.Error(), "node-1") + }) + + t.Run("approved job without the family blocks", func(t *testing.T) { + t.Parallel() + jd := base() + jd.proposals = []*jobv1.Proposal{proposal("j1", 1, approved, consensusSpecWith("aptos"))} + require.ErrorContains(t, verifySignerFamilyJobs(envWith(jd), []string{"p2p_a"}, stellar), "node-1 (stellar)") + }) + + t.Run("every requested family must be covered", func(t *testing.T) { + t.Parallel() + jd := base() + jd.proposals = []*jobv1.Proposal{proposal("j1", 1, approved, consensusSpecWith("stellar"))} + err := verifySignerFamilyJobs(envWith(jd), []string{"p2p_a"}, []string{"aptos", "stellar"}) + require.ErrorContains(t, err, "node-1 (aptos)") + }) + + t.Run("non-consensus job does not count", func(t *testing.T) { + t.Parallel() + jd := base() + spec := "type = \"standardcapabilities\"\ncommand = \"/usr/local/bin/evm\"\n" + + "[oracle_factory.onchainSigningStrategy.config]\nstellar = \"bundle-stellar\"\n" + jd.proposals = []*jobv1.Proposal{proposal("j1", 1, approved, spec)} + require.ErrorContains(t, verifySignerFamilyJobs(envWith(jd), []string{"p2p_a"}, stellar), "node-1") + }) + + t.Run("only the latest revision counts", func(t *testing.T) { + t.Parallel() + jd := base() + jd.proposals = []*jobv1.Proposal{ + proposal("j1", 1, approved, consensusSpecWith("stellar")), + proposal("j1", 2, proposed, consensusSpecWith()), + } + require.ErrorContains(t, verifySignerFamilyJobs(envWith(jd), []string{"p2p_a"}, stellar), "node-1") + }) + + t.Run("deleted jobs do not count", func(t *testing.T) { + t.Parallel() + jd := base() + jd.jobs[0].DeletedAt = timestamppb.Now() + jd.proposals = []*jobv1.Proposal{proposal("j1", 1, approved, consensusSpecWith("stellar"))} + require.ErrorContains(t, verifySignerFamilyJobs(envWith(jd), []string{"p2p_a"}, stellar), "node-1") + }) + + t.Run("node unknown to JD blocks", func(t *testing.T) { + t.Parallel() + jd := base() + err := verifySignerFamilyJobs(envWith(jd), []string{"p2p_missing"}, stellar) + require.ErrorContains(t, err, "p2p_missing (not in JD)") + }) + + t.Run("any JD record of a peer may hold the job", func(t *testing.T) { + t.Parallel() + jd := &fakeJD{ + nodes: []*nodev1.Node{jdNode("n1", "old-record", "p2p_a"), jdNode("n2", "new-record", "p2p_a")}, + jobs: []*jobv1.Job{{Id: "j1", NodeId: "n1"}}, + proposals: []*jobv1.Proposal{proposal("j1", 1, approved, consensusSpecWith("stellar"))}, + } + require.NoError(t, verifySignerFamilyJobs(envWith(jd), []string{"p2p_a"}, stellar)) + }) + + t.Run("bare hex IDs match a prefixed JD label and vice versa", func(t *testing.T) { + t.Parallel() + var id p2pkey.PeerID + id[0] = 3 + jd := &fakeJD{ + nodes: []*nodev1.Node{jdNode("n1", "node-1", id.Raw())}, + jobs: []*jobv1.Job{{Id: "j1", NodeId: "n1"}}, + proposals: []*jobv1.Proposal{proposal("j1", 1, approved, consensusSpecWith("stellar"))}, + } + require.NoError(t, verifySignerFamilyJobs(envWith(jd), []string{id.String()}, stellar)) + require.NoError(t, verifySignerFamilyJobs(envWith(jd), []string{id.Raw()}, stellar)) + }) + + t.Run("JD list errors propagate", func(t *testing.T) { + t.Parallel() + jd := base() + jd.jobsErr = errors.New("boom") + require.ErrorContains(t, verifySignerFamilyJobs(envWith(jd), []string{"p2p_a"}, stellar), "boom") + + jd = base() + jd.propsErr = errors.New("bang") + require.ErrorContains(t, verifySignerFamilyJobs(envWith(jd), []string{"p2p_a"}, stellar), "bang") + }) +} + +func TestConsensusSpecFamilies(t *testing.T) { + t.Parallel() + + require.Equal(t, map[string]bool{"evm": true, "stellar": true}, consensusSpecFamilies(consensusSpecWith("stellar"))) + require.Empty(t, consensusSpecFamilies("not toml [")) + require.Empty(t, consensusSpecFamilies(`type = "offchainreporting2"`+"\n"+`command = "consensus"`)) + require.Empty(t, consensusSpecFamilies(`type = "standardcapabilities"`+"\n"+`command = "/bin/consensus"`)) + + empty := "type = \"standardcapabilities\"\ncommand = \"consensus\"\n[oracle_factory.onchainSigningStrategy.config]\nstellar = \"\"\n" + require.Empty(t, consensusSpecFamilies(empty)) +} + +func ocr3Entry(families ...any) map[string]any { + entry := map[string]any{"offchainConfig": map[string]any{"maxFaultyOracles": 1}} + if families != nil { + entry["extraSignerFamilies"] = families + } + + return map[string]any{"ocr3Configs": map[string]any{"__default__": entry}} +} + +func TestRequestedSignerFamilies(t *testing.T) { + t.Parallel() + + got, err := requestedSignerFamilies([]ocr3CapConfig{ + {CapabilityID: "consensus@1.0.0", Config: ocr3Entry("stellar", "aptos")}, + {CapabilityID: "vault@1.0.0", Config: ocr3Entry("aptos")}, + {CapabilityID: "other@1.0.0", Config: map[string]any{"k": "v"}}, + }) + require.NoError(t, err) + require.Equal(t, []string{"aptos", "stellar"}, got) + + got, err = requestedSignerFamilies([]ocr3CapConfig{{CapabilityID: "c", Config: ocr3Entry()}}) + require.NoError(t, err) + require.Empty(t, got) + + processed := map[string]any{"ocr3Configs": map[string]any{"__default__": map[string]any{ + "offchainConfig": "base64==", "extraSignerFamilies": []any{"stellar"}, + }}} + got, err = requestedSignerFamilies([]ocr3CapConfig{{CapabilityID: "c", Config: processed}}) + require.NoError(t, err) + require.Empty(t, got) + + _, err = requestedSignerFamilies([]ocr3CapConfig{{CapabilityID: "c", Config: ocr3Entry("evm")}}) + require.Error(t, err) +} + +func TestDonNodeP2PIDs(t *testing.T) { + t.Parallel() + + var id p2pkey.PeerID + id[0] = 7 + don := &capabilities_registry_v2.CapabilitiesRegistryDONInfo{NodeP2PIds: [][32]byte{id}} + require.Equal(t, []string{id.String()}, donNodeP2PIDs(don)) +} + +func TestUpdateDONPreconditions_SignerFamilyJobCheck(t *testing.T) { + t.Parallel() + + e := envWith(&fakeJD{nodes: []*nodev1.Node{jdNode("n1", "node-1", "p2p_a")}, jobs: []*jobv1.Job{{Id: "j1", NodeId: "n1"}}}) + in := UpdateDONInput{ + DONName: "d", Nodes: []string{"p2p_a"}, SkipDuplicateCapabilityValidation: true, + CapabilityConfigs: []contracts.CapabilityConfig{{ + Capability: contracts.Capability{CapabilityID: "consensus@1.0.0"}, + Config: ocr3Entry("stellar"), + }}, + } + + require.ErrorContains(t, UpdateDON{}.VerifyPreconditions(e, in), "node-1 (stellar)") + + withJob := e + withJob.Offchain = &fakeJD{ + nodes: []*nodev1.Node{jdNode("n1", "node-1", "p2p_a")}, + jobs: []*jobv1.Job{{Id: "j1", NodeId: "n1"}}, + proposals: []*jobv1.Proposal{proposal("j1", 1, jobv1.ProposalStatus_PROPOSAL_STATUS_APPROVED, consensusSpecWith("stellar"))}, + } + require.NoError(t, UpdateDON{}.VerifyPreconditions(withJob, in)) + + in.SkipSignerFamilyJobCheck = true + require.NoError(t, UpdateDON{}.VerifyPreconditions(cldf.Environment{}, in)) + + in.SkipSignerFamilyJobCheck = false + in.CapabilityConfigs = []contracts.CapabilityConfig{{ + Capability: contracts.Capability{CapabilityID: "consensus@1.0.0"}, + Config: ocr3Entry(), + }} + require.NoError(t, UpdateDON{}.VerifyPreconditions(cldf.Environment{}, in), "no families requested: JD is not needed") +} diff --git a/deployment/cre/capabilities_registry/v2/changeset/types.go b/deployment/cre/capabilities_registry/v2/changeset/types.go index d14a6b8938b..618eb091bd2 100644 --- a/deployment/cre/capabilities_registry/v2/changeset/types.go +++ b/deployment/cre/capabilities_registry/v2/changeset/types.go @@ -182,12 +182,10 @@ func (don CapabilitiesRegistryNewDONParams) applyModifiersToCapabilityConfigs(e return capabilityConfigurations, nil } -// getExistingDONsForPreconditionCheck reads the current on-chain DONs for the capabilities -// registry identified by chainSel/qualifier, so preconditions can check a proposed capability -// assignment against what other DONs already have configured. -func getExistingDONsForPreconditionCheck( +// capRegForPreconditionCheck opens the capabilities registry identified by chainSel/qualifier for reads. +func capRegForPreconditionCheck( e cldf.Environment, chainSel uint64, qualifier string, -) ([]capabilities_registry_v2.CapabilitiesRegistryDONInfo, error) { +) (*capabilities_registry_v2.CapabilitiesRegistry, error) { chain, ok := e.BlockChains.EVMChains()[chainSel] if !ok { return nil, fmt.Errorf("chain not found for selector %d", chainSel) @@ -205,6 +203,20 @@ func getExistingDONsForPreconditionCheck( return nil, fmt.Errorf("failed to create CapabilitiesRegistry: %w", err) } + return capReg, nil +} + +// getExistingDONsForPreconditionCheck reads the current on-chain DONs for the capabilities +// registry identified by chainSel/qualifier, so preconditions can check a proposed capability +// assignment against what other DONs already have configured. +func getExistingDONsForPreconditionCheck( + e cldf.Environment, chainSel uint64, qualifier string, +) ([]capabilities_registry_v2.CapabilitiesRegistryDONInfo, error) { + capReg, err := capRegForPreconditionCheck(e, chainSel, qualifier) + if err != nil { + return nil, err + } + existingDONs, err := pkg.GetDONs(&bind.CallOpts{}, capReg) if err != nil { return nil, fmt.Errorf("failed to get existing DONs: %w", err) diff --git a/deployment/cre/capabilities_registry/v2/changeset/update_don.go b/deployment/cre/capabilities_registry/v2/changeset/update_don.go index e3d910ec158..ae6460faf2c 100644 --- a/deployment/cre/capabilities_registry/v2/changeset/update_don.go +++ b/deployment/cre/capabilities_registry/v2/changeset/update_don.go @@ -66,6 +66,11 @@ type UpdateDONInput struct { // the same capability to multiple DONs within a DON family. SkipDuplicateCapabilityValidation bool `json:"skipDuplicateCapabilityValidation,omitempty" yaml:"skipDuplicateCapabilityValidation,omitempty"` + // SkipSignerFamilyJobCheck disables the precondition check that, when this change adds + // extraSignerFamilies to an OCR3 config, every node already has an APPROVED consensus job with a + // key bundle for each family. Use only for emergencies. + SkipSignerFamilyJobCheck bool `json:"skipSignerFamilyJobCheck,omitempty" yaml:"skipSignerFamilyJobCheck,omitempty"` + MCMSConfig *crecontracts.MCMSConfig `json:"mcmsConfig" yaml:"mcmsConfig"` } @@ -88,9 +93,19 @@ func (u UpdateDON) VerifyPreconditions(e cldf.Environment, config UpdateDONInput } } + if !config.SkipSignerFamilyJobCheck { + if err := verifyUpdateDONSignerFamilies(e, config); err != nil { + return fmt.Errorf("UpdateDON precondition failed: %w", err) + } + } + return nil } +func verifyUpdateDONSignerFamilies(e cldf.Environment, config UpdateDONInput) error { + return verifyDONSignerFamilies(e, config.RegistryChainSel, config.RegistryQualifier, config.DONName, config.CapabilityConfigs, config.Nodes) +} + func (u UpdateDON) Apply(e cldf.Environment, config UpdateDONInput) (cldf.ChangesetOutput, error) { var mcmsContracts *evmstate.MCMSWithTimelockState if config.MCMSConfig != nil {