diff --git a/pkg/capabilities/pb/capabilities.pb.go b/pkg/capabilities/pb/capabilities.pb.go index bf3d73a4ba..77280e14b5 100644 --- a/pkg/capabilities/pb/capabilities.pb.go +++ b/pkg/capabilities/pb/capabilities.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.12 +// protoc-gen-go v1.36.10 // protoc v5.29.3 // source: capabilities.proto diff --git a/pkg/capabilities/pb/registry.pb.go b/pkg/capabilities/pb/registry.pb.go index 3a2fd8911a..748ddb62a2 100644 --- a/pkg/capabilities/pb/registry.pb.go +++ b/pkg/capabilities/pb/registry.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.12 +// protoc-gen-go v1.36.10 // protoc v5.29.3 // source: registry.proto diff --git a/pkg/capabilities/pb/triggers.go b/pkg/capabilities/pb/triggers.go new file mode 100644 index 0000000000..26e2fc58e9 --- /dev/null +++ b/pkg/capabilities/pb/triggers.go @@ -0,0 +1,9 @@ +package pb + +import "cmp" + +func (e *ConsensusQueueEvent) CompareDeadline(o *ConsensusQueueEvent) int { + return cmp.Or(cmp.Compare(e.Deadline.Seconds, o.Deadline.Seconds), + cmp.Compare(e.Deadline.Nanos, o.Deadline.Nanos), + cmp.Compare(e.TriggerEventID, o.TriggerEventID)) +} diff --git a/pkg/capabilities/pb/triggers.pb.go b/pkg/capabilities/pb/triggers.pb.go index 5d5f88fb87..209bb8b13e 100644 --- a/pkg/capabilities/pb/triggers.pb.go +++ b/pkg/capabilities/pb/triggers.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.12 +// protoc-gen-go v1.36.10 // protoc v5.29.3 // source: triggers.proto @@ -23,16 +23,15 @@ const ( ) type CoordinatedEvent struct { - state protoimpl.MessageState `protogen:"open.v1"` - WorkflowID string `protobuf:"bytes,1,opt,name=workflowID,proto3" json:"workflowID,omitempty"` - TriggerCapID string `protobuf:"bytes,2,opt,name=triggerCapID,proto3" json:"triggerCapID,omitempty"` - TriggerIndex int64 `protobuf:"varint,3,opt,name=triggerIndex,proto3" json:"triggerIndex,omitempty"` - ObservedAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=observedAt,proto3" json:"observedAt,omitempty"` - Deadline *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=deadline,proto3" json:"deadline,omitempty"` - SequenceNumber uint64 `protobuf:"varint,6,opt,name=sequenceNumber,proto3" json:"sequenceNumber,omitempty"` - Event *TriggerResponse `protobuf:"bytes,7,opt,name=event,proto3" json:"event,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + WorkflowID string `protobuf:"bytes,1,opt,name=workflowID,proto3" json:"workflowID,omitempty"` + TriggerCapID string `protobuf:"bytes,2,opt,name=triggerCapID,proto3" json:"triggerCapID,omitempty"` + TriggerIndex int64 `protobuf:"varint,3,opt,name=triggerIndex,proto3" json:"triggerIndex,omitempty"` + ObservedAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=observedAt,proto3" json:"observedAt,omitempty"` + Deadline *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=deadline,proto3" json:"deadline,omitempty"` + Event *TriggerResponse `protobuf:"bytes,6,opt,name=event,proto3" json:"event,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CoordinatedEvent) Reset() { @@ -100,30 +99,301 @@ func (x *CoordinatedEvent) GetDeadline() *timestamppb.Timestamp { return nil } -func (x *CoordinatedEvent) GetSequenceNumber() uint64 { +func (x *CoordinatedEvent) GetEvent() *TriggerResponse { + if x != nil { + return x.Event + } + return nil +} + +type CoordinatedEventList struct { + state protoimpl.MessageState `protogen:"open.v1"` + Events []*CoordinatedEvent `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CoordinatedEventList) Reset() { + *x = CoordinatedEventList{} + mi := &file_triggers_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CoordinatedEventList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CoordinatedEventList) ProtoMessage() {} + +func (x *CoordinatedEventList) ProtoReflect() protoreflect.Message { + mi := &file_triggers_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CoordinatedEventList.ProtoReflect.Descriptor instead. +func (*CoordinatedEventList) Descriptor() ([]byte, []int) { + return file_triggers_proto_rawDescGZIP(), []int{1} +} + +func (x *CoordinatedEventList) GetEvents() []*CoordinatedEvent { + if x != nil { + return x.Events + } + return nil +} + +// ConsensusQueueEvent holds minimal event metadata for queueing and medianization within the plugin, and one or more +// blob reference pointing to the rest. +type ConsensusQueueEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + WorkflowID string `protobuf:"bytes,1,opt,name=workflowID,proto3" json:"workflowID,omitempty"` + TriggerEventID string `protobuf:"bytes,2,opt,name=triggerEventID,proto3" json:"triggerEventID,omitempty"` + ObservedAt *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=observedAt,proto3" json:"observedAt,omitempty"` + Deadline *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=deadline,proto3" json:"deadline,omitempty"` + TimelessHash []byte `protobuf:"bytes,5,opt,name=timelessHash,proto3" json:"timelessHash,omitempty"` // hash of the blob data, timestamp fields we have extracted for medianization set to nil + Blobs []*CoordinatedEventBlobHandle `protobuf:"bytes,6,rep,name=blobs,proto3" json:"blobs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConsensusQueueEvent) Reset() { + *x = ConsensusQueueEvent{} + mi := &file_triggers_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConsensusQueueEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConsensusQueueEvent) ProtoMessage() {} + +func (x *ConsensusQueueEvent) ProtoReflect() protoreflect.Message { + mi := &file_triggers_proto_msgTypes[2] if x != nil { - return x.SequenceNumber + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConsensusQueueEvent.ProtoReflect.Descriptor instead. +func (*ConsensusQueueEvent) Descriptor() ([]byte, []int) { + return file_triggers_proto_rawDescGZIP(), []int{2} +} + +func (x *ConsensusQueueEvent) GetWorkflowID() string { + if x != nil { + return x.WorkflowID + } + return "" +} + +func (x *ConsensusQueueEvent) GetTriggerEventID() string { + if x != nil { + return x.TriggerEventID + } + return "" +} + +func (x *ConsensusQueueEvent) GetObservedAt() *timestamppb.Timestamp { + if x != nil { + return x.ObservedAt + } + return nil +} + +func (x *ConsensusQueueEvent) GetDeadline() *timestamppb.Timestamp { + if x != nil { + return x.Deadline + } + return nil +} + +func (x *ConsensusQueueEvent) GetTimelessHash() []byte { + if x != nil { + return x.TimelessHash + } + return nil +} + +func (x *ConsensusQueueEvent) GetBlobs() []*CoordinatedEventBlobHandle { + if x != nil { + return x.Blobs + } + return nil +} + +// ConsensusQueueEventsBlobHandle refers to a CoordinatedEvent by blob handle and index within an CoordinatedEventList. +type CoordinatedEventBlobHandle struct { + state protoimpl.MessageState `protogen:"open.v1"` + Blob uint32 `protobuf:"varint,1,opt,name=blob,proto3" json:"blob,omitempty"` + Element uint32 `protobuf:"varint,2,opt,name=element,proto3" json:"element,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CoordinatedEventBlobHandle) Reset() { + *x = CoordinatedEventBlobHandle{} + mi := &file_triggers_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CoordinatedEventBlobHandle) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CoordinatedEventBlobHandle) ProtoMessage() {} + +func (x *CoordinatedEventBlobHandle) ProtoReflect() protoreflect.Message { + mi := &file_triggers_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CoordinatedEventBlobHandle.ProtoReflect.Descriptor instead. +func (*CoordinatedEventBlobHandle) Descriptor() ([]byte, []int) { + return file_triggers_proto_rawDescGZIP(), []int{3} +} + +func (x *CoordinatedEventBlobHandle) GetBlob() uint32 { + if x != nil { + return x.Blob } return 0 } -func (x *CoordinatedEvent) GetEvent() *TriggerResponse { +func (x *CoordinatedEventBlobHandle) GetElement() uint32 { if x != nil { - return x.Event + return x.Element + } + return 0 +} + +// ConsensusQueueEvents holds a list of events. +type ConsensusQueueEventList struct { + state protoimpl.MessageState `protogen:"open.v1"` + Events []*ConsensusQueueEvent `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConsensusQueueEventList) Reset() { + *x = ConsensusQueueEventList{} + mi := &file_triggers_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConsensusQueueEventList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConsensusQueueEventList) ProtoMessage() {} + +func (x *ConsensusQueueEventList) ProtoReflect() protoreflect.Message { + mi := &file_triggers_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConsensusQueueEventList.ProtoReflect.Descriptor instead. +func (*ConsensusQueueEventList) Descriptor() ([]byte, []int) { + return file_triggers_proto_rawDescGZIP(), []int{4} +} + +func (x *ConsensusQueueEventList) GetEvents() []*ConsensusQueueEvent { + if x != nil { + return x.Events } return nil } -type ConsensusQueueObservation struct { +// ConsensusQueueBlobHandleList holds a list of blob handles referenced from ConsensusQueueEventList events. +type ConsensusQueueBlobHandleList struct { state protoimpl.MessageState `protogen:"open.v1"` - Events []*CoordinatedEvent `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"` + BlobHandles [][]byte `protobuf:"bytes,1,rep,name=blobHandles,proto3" json:"blobHandles,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } +func (x *ConsensusQueueBlobHandleList) Reset() { + *x = ConsensusQueueBlobHandleList{} + mi := &file_triggers_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConsensusQueueBlobHandleList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConsensusQueueBlobHandleList) ProtoMessage() {} + +func (x *ConsensusQueueBlobHandleList) ProtoReflect() protoreflect.Message { + mi := &file_triggers_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConsensusQueueBlobHandleList.ProtoReflect.Descriptor instead. +func (*ConsensusQueueBlobHandleList) Descriptor() ([]byte, []int) { + return file_triggers_proto_rawDescGZIP(), []int{5} +} + +func (x *ConsensusQueueBlobHandleList) GetBlobHandles() [][]byte { + if x != nil { + return x.BlobHandles + } + return nil +} + +// ConsensusQueueObservation is a ReportingPlugin observation of events and capacity meta-data. +type ConsensusQueueObservation struct { + state protoimpl.MessageState `protogen:"open.v1"` + Events []*ConsensusQueueEvent `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"` // sorted + BlobHandles [][]byte `protobuf:"bytes,2,rep,name=blobHandles,proto3" json:"blobHandles,omitempty"` // referenced from events by index + Expiration *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=expiration,proto3" json:"expiration,omitempty"` // observation time - grace period + Capacity uint32 `protobuf:"varint,4,opt,name=capacity,proto3" json:"capacity,omitempty"` // Global remaining capacity + WorkflowCapacities map[string]uint32 `protobuf:"bytes,5,rep,name=workflowCapacities,proto3" json:"workflowCapacities,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` // Workflow-specific free executions + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + func (x *ConsensusQueueObservation) Reset() { *x = ConsensusQueueObservation{} - mi := &file_triggers_proto_msgTypes[1] + mi := &file_triggers_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -135,7 +405,7 @@ func (x *ConsensusQueueObservation) String() string { func (*ConsensusQueueObservation) ProtoMessage() {} func (x *ConsensusQueueObservation) ProtoReflect() protoreflect.Message { - mi := &file_triggers_proto_msgTypes[1] + mi := &file_triggers_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -148,38 +418,68 @@ func (x *ConsensusQueueObservation) ProtoReflect() protoreflect.Message { // Deprecated: Use ConsensusQueueObservation.ProtoReflect.Descriptor instead. func (*ConsensusQueueObservation) Descriptor() ([]byte, []int) { - return file_triggers_proto_rawDescGZIP(), []int{1} + return file_triggers_proto_rawDescGZIP(), []int{6} } -func (x *ConsensusQueueObservation) GetEvents() []*CoordinatedEvent { +func (x *ConsensusQueueObservation) GetEvents() []*ConsensusQueueEvent { if x != nil { return x.Events } return nil } -type ConsensusQueueReport struct { - state protoimpl.MessageState `protogen:"open.v1"` - Events []*CoordinatedEvent `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"` +func (x *ConsensusQueueObservation) GetBlobHandles() [][]byte { + if x != nil { + return x.BlobHandles + } + return nil +} + +func (x *ConsensusQueueObservation) GetExpiration() *timestamppb.Timestamp { + if x != nil { + return x.Expiration + } + return nil +} + +func (x *ConsensusQueueObservation) GetCapacity() uint32 { + if x != nil { + return x.Capacity + } + return 0 +} + +func (x *ConsensusQueueObservation) GetWorkflowCapacities() map[string]uint32 { + if x != nil { + return x.WorkflowCapacities + } + return nil +} + +// ConsensusQueueObservedEvents holds a working set of observed events that are awaiting consensus. +type ConsensusQueueObservedEvents struct { + state protoimpl.MessageState `protogen:"open.v1"` + Observations []*ConsensusQueueObservedEventEntry `protobuf:"bytes,1,rep,name=observations,proto3" json:"observations,omitempty"` // map entries since we have []byte keys + BlobHandles [][]byte `protobuf:"bytes,2,rep,name=blobHandles,proto3" json:"blobHandles,omitempty"` // referenced from observations events by index unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *ConsensusQueueReport) Reset() { - *x = ConsensusQueueReport{} - mi := &file_triggers_proto_msgTypes[2] +func (x *ConsensusQueueObservedEvents) Reset() { + *x = ConsensusQueueObservedEvents{} + mi := &file_triggers_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *ConsensusQueueReport) String() string { +func (x *ConsensusQueueObservedEvents) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ConsensusQueueReport) ProtoMessage() {} +func (*ConsensusQueueObservedEvents) ProtoMessage() {} -func (x *ConsensusQueueReport) ProtoReflect() protoreflect.Message { - mi := &file_triggers_proto_msgTypes[2] +func (x *ConsensusQueueObservedEvents) ProtoReflect() protoreflect.Message { + mi := &file_triggers_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -190,14 +490,158 @@ func (x *ConsensusQueueReport) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ConsensusQueueReport.ProtoReflect.Descriptor instead. -func (*ConsensusQueueReport) Descriptor() ([]byte, []int) { - return file_triggers_proto_rawDescGZIP(), []int{2} +// Deprecated: Use ConsensusQueueObservedEvents.ProtoReflect.Descriptor instead. +func (*ConsensusQueueObservedEvents) Descriptor() ([]byte, []int) { + return file_triggers_proto_rawDescGZIP(), []int{7} } -func (x *ConsensusQueueReport) GetEvents() []*CoordinatedEvent { +func (x *ConsensusQueueObservedEvents) GetObservations() []*ConsensusQueueObservedEventEntry { if x != nil { - return x.Events + return x.Observations + } + return nil +} + +func (x *ConsensusQueueObservedEvents) GetBlobHandles() [][]byte { + if x != nil { + return x.BlobHandles + } + return nil +} + +type ConsensusQueueObservedEventEntry struct { + state protoimpl.MessageState `protogen:"open.v1"` + TimelessHash []byte `protobuf:"bytes,1,opt,name=timelessHash,proto3" json:"timelessHash,omitempty"` // key + Event *ConsensusQueueObservedEvent `protobuf:"bytes,2,opt,name=event,proto3" json:"event,omitempty"` // val + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConsensusQueueObservedEventEntry) Reset() { + *x = ConsensusQueueObservedEventEntry{} + mi := &file_triggers_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConsensusQueueObservedEventEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConsensusQueueObservedEventEntry) ProtoMessage() {} + +func (x *ConsensusQueueObservedEventEntry) ProtoReflect() protoreflect.Message { + mi := &file_triggers_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConsensusQueueObservedEventEntry.ProtoReflect.Descriptor instead. +func (*ConsensusQueueObservedEventEntry) Descriptor() ([]byte, []int) { + return file_triggers_proto_rawDescGZIP(), []int{8} +} + +func (x *ConsensusQueueObservedEventEntry) GetTimelessHash() []byte { + if x != nil { + return x.TimelessHash + } + return nil +} + +func (x *ConsensusQueueObservedEventEntry) GetEvent() *ConsensusQueueObservedEvent { + if x != nil { + return x.Event + } + return nil +} + +// ConsensusQueueObservedEvent tracks observation metadata for events awaiting consensus. +type ConsensusQueueObservedEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + WorkflowID string `protobuf:"bytes,1,opt,name=workflowID,proto3" json:"workflowID,omitempty"` + TriggerEventID string `protobuf:"bytes,2,opt,name=triggerEventID,proto3" json:"triggerEventID,omitempty"` + NodeIds []byte `protobuf:"bytes,3,opt,name=nodeIds,proto3" json:"nodeIds,omitempty"` // []OracleID + ObservedAts []*timestamppb.Timestamp `protobuf:"bytes,4,rep,name=observedAts,proto3" json:"observedAts,omitempty"` + Deadlines []*timestamppb.Timestamp `protobuf:"bytes,5,rep,name=deadlines,proto3" json:"deadlines,omitempty"` + Blobs []*CoordinatedEventBlobHandle `protobuf:"bytes,6,rep,name=blobs,proto3" json:"blobs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConsensusQueueObservedEvent) Reset() { + *x = ConsensusQueueObservedEvent{} + mi := &file_triggers_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConsensusQueueObservedEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConsensusQueueObservedEvent) ProtoMessage() {} + +func (x *ConsensusQueueObservedEvent) ProtoReflect() protoreflect.Message { + mi := &file_triggers_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConsensusQueueObservedEvent.ProtoReflect.Descriptor instead. +func (*ConsensusQueueObservedEvent) Descriptor() ([]byte, []int) { + return file_triggers_proto_rawDescGZIP(), []int{9} +} + +func (x *ConsensusQueueObservedEvent) GetWorkflowID() string { + if x != nil { + return x.WorkflowID + } + return "" +} + +func (x *ConsensusQueueObservedEvent) GetTriggerEventID() string { + if x != nil { + return x.TriggerEventID + } + return "" +} + +func (x *ConsensusQueueObservedEvent) GetNodeIds() []byte { + if x != nil { + return x.NodeIds + } + return nil +} + +func (x *ConsensusQueueObservedEvent) GetObservedAts() []*timestamppb.Timestamp { + if x != nil { + return x.ObservedAts + } + return nil +} + +func (x *ConsensusQueueObservedEvent) GetDeadlines() []*timestamppb.Timestamp { + if x != nil { + return x.Deadlines + } + return nil +} + +func (x *ConsensusQueueObservedEvent) GetBlobs() []*CoordinatedEventBlobHandle { + if x != nil { + return x.Blobs } return nil } @@ -206,7 +650,7 @@ var File_triggers_proto protoreflect.FileDescriptor const file_triggers_proto_rawDesc = "" + "\n" + - "\x0etriggers.proto\x12\btriggers\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x12capabilities.proto\"\xcb\x02\n" + + "\x0etriggers.proto\x12\btriggers\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x12capabilities.proto\"\xa3\x02\n" + "\x10CoordinatedEvent\x12\x1e\n" + "\n" + "workflowID\x18\x01 \x01(\tR\n" + @@ -216,13 +660,54 @@ const file_triggers_proto_rawDesc = "" + "\n" + "observedAt\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\n" + "observedAt\x126\n" + - "\bdeadline\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\bdeadline\x12&\n" + - "\x0esequenceNumber\x18\x06 \x01(\x04R\x0esequenceNumber\x123\n" + - "\x05event\x18\a \x01(\v2\x1d.capabilities.TriggerResponseR\x05event\"O\n" + - "\x19ConsensusQueueObservation\x122\n" + - "\x06events\x18\x01 \x03(\v2\x1a.triggers.CoordinatedEventR\x06events\"J\n" + - "\x14ConsensusQueueReport\x122\n" + - "\x06events\x18\x01 \x03(\v2\x1a.triggers.CoordinatedEventR\x06eventsBBZ@github.com/smartcontractkit/chainlink-common/pkg/capabilities/pbb\x06proto3" + "\bdeadline\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\bdeadline\x123\n" + + "\x05event\x18\x06 \x01(\v2\x1d.capabilities.TriggerResponseR\x05event\"J\n" + + "\x14CoordinatedEventList\x122\n" + + "\x06events\x18\x01 \x03(\v2\x1a.triggers.CoordinatedEventR\x06events\"\xb1\x02\n" + + "\x13ConsensusQueueEvent\x12\x1e\n" + + "\n" + + "workflowID\x18\x01 \x01(\tR\n" + + "workflowID\x12&\n" + + "\x0etriggerEventID\x18\x02 \x01(\tR\x0etriggerEventID\x12:\n" + + "\n" + + "observedAt\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "observedAt\x126\n" + + "\bdeadline\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\bdeadline\x12\"\n" + + "\ftimelessHash\x18\x05 \x01(\fR\ftimelessHash\x12:\n" + + "\x05blobs\x18\x06 \x03(\v2$.triggers.CoordinatedEventBlobHandleR\x05blobs\"J\n" + + "\x1aCoordinatedEventBlobHandle\x12\x12\n" + + "\x04blob\x18\x01 \x01(\rR\x04blob\x12\x18\n" + + "\aelement\x18\x02 \x01(\rR\aelement\"P\n" + + "\x17ConsensusQueueEventList\x125\n" + + "\x06events\x18\x01 \x03(\v2\x1d.triggers.ConsensusQueueEventR\x06events\"@\n" + + "\x1cConsensusQueueBlobHandleList\x12 \n" + + "\vblobHandles\x18\x01 \x03(\fR\vblobHandles\"\x80\x03\n" + + "\x19ConsensusQueueObservation\x125\n" + + "\x06events\x18\x01 \x03(\v2\x1d.triggers.ConsensusQueueEventR\x06events\x12 \n" + + "\vblobHandles\x18\x02 \x03(\fR\vblobHandles\x12:\n" + + "\n" + + "expiration\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "expiration\x12\x1a\n" + + "\bcapacity\x18\x04 \x01(\rR\bcapacity\x12k\n" + + "\x12workflowCapacities\x18\x05 \x03(\v2;.triggers.ConsensusQueueObservation.WorkflowCapacitiesEntryR\x12workflowCapacities\x1aE\n" + + "\x17WorkflowCapacitiesEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\rR\x05value:\x028\x01\"\x90\x01\n" + + "\x1cConsensusQueueObservedEvents\x12N\n" + + "\fobservations\x18\x01 \x03(\v2*.triggers.ConsensusQueueObservedEventEntryR\fobservations\x12 \n" + + "\vblobHandles\x18\x02 \x03(\fR\vblobHandles\"\x83\x01\n" + + " ConsensusQueueObservedEventEntry\x12\"\n" + + "\ftimelessHash\x18\x01 \x01(\fR\ftimelessHash\x12;\n" + + "\x05event\x18\x02 \x01(\v2%.triggers.ConsensusQueueObservedEventR\x05event\"\xb3\x02\n" + + "\x1bConsensusQueueObservedEvent\x12\x1e\n" + + "\n" + + "workflowID\x18\x01 \x01(\tR\n" + + "workflowID\x12&\n" + + "\x0etriggerEventID\x18\x02 \x01(\tR\x0etriggerEventID\x12\x18\n" + + "\anodeIds\x18\x03 \x01(\fR\anodeIds\x12<\n" + + "\vobservedAts\x18\x04 \x03(\v2\x1a.google.protobuf.TimestampR\vobservedAts\x128\n" + + "\tdeadlines\x18\x05 \x03(\v2\x1a.google.protobuf.TimestampR\tdeadlines\x12:\n" + + "\x05blobs\x18\x06 \x03(\v2$.triggers.CoordinatedEventBlobHandleR\x05blobsBBZ@github.com/smartcontractkit/chainlink-common/pkg/capabilities/pbb\x06proto3" var ( file_triggers_proto_rawDescOnce sync.Once @@ -236,25 +721,44 @@ func file_triggers_proto_rawDescGZIP() []byte { return file_triggers_proto_rawDescData } -var file_triggers_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_triggers_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_triggers_proto_goTypes = []any{ - (*CoordinatedEvent)(nil), // 0: triggers.CoordinatedEvent - (*ConsensusQueueObservation)(nil), // 1: triggers.ConsensusQueueObservation - (*ConsensusQueueReport)(nil), // 2: triggers.ConsensusQueueReport - (*timestamppb.Timestamp)(nil), // 3: google.protobuf.Timestamp - (*TriggerResponse)(nil), // 4: capabilities.TriggerResponse + (*CoordinatedEvent)(nil), // 0: triggers.CoordinatedEvent + (*CoordinatedEventList)(nil), // 1: triggers.CoordinatedEventList + (*ConsensusQueueEvent)(nil), // 2: triggers.ConsensusQueueEvent + (*CoordinatedEventBlobHandle)(nil), // 3: triggers.CoordinatedEventBlobHandle + (*ConsensusQueueEventList)(nil), // 4: triggers.ConsensusQueueEventList + (*ConsensusQueueBlobHandleList)(nil), // 5: triggers.ConsensusQueueBlobHandleList + (*ConsensusQueueObservation)(nil), // 6: triggers.ConsensusQueueObservation + (*ConsensusQueueObservedEvents)(nil), // 7: triggers.ConsensusQueueObservedEvents + (*ConsensusQueueObservedEventEntry)(nil), // 8: triggers.ConsensusQueueObservedEventEntry + (*ConsensusQueueObservedEvent)(nil), // 9: triggers.ConsensusQueueObservedEvent + nil, // 10: triggers.ConsensusQueueObservation.WorkflowCapacitiesEntry + (*timestamppb.Timestamp)(nil), // 11: google.protobuf.Timestamp + (*TriggerResponse)(nil), // 12: capabilities.TriggerResponse } var file_triggers_proto_depIdxs = []int32{ - 3, // 0: triggers.CoordinatedEvent.observedAt:type_name -> google.protobuf.Timestamp - 3, // 1: triggers.CoordinatedEvent.deadline:type_name -> google.protobuf.Timestamp - 4, // 2: triggers.CoordinatedEvent.event:type_name -> capabilities.TriggerResponse - 0, // 3: triggers.ConsensusQueueObservation.events:type_name -> triggers.CoordinatedEvent - 0, // 4: triggers.ConsensusQueueReport.events:type_name -> triggers.CoordinatedEvent - 5, // [5:5] is the sub-list for method output_type - 5, // [5:5] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 11, // 0: triggers.CoordinatedEvent.observedAt:type_name -> google.protobuf.Timestamp + 11, // 1: triggers.CoordinatedEvent.deadline:type_name -> google.protobuf.Timestamp + 12, // 2: triggers.CoordinatedEvent.event:type_name -> capabilities.TriggerResponse + 0, // 3: triggers.CoordinatedEventList.events:type_name -> triggers.CoordinatedEvent + 11, // 4: triggers.ConsensusQueueEvent.observedAt:type_name -> google.protobuf.Timestamp + 11, // 5: triggers.ConsensusQueueEvent.deadline:type_name -> google.protobuf.Timestamp + 3, // 6: triggers.ConsensusQueueEvent.blobs:type_name -> triggers.CoordinatedEventBlobHandle + 2, // 7: triggers.ConsensusQueueEventList.events:type_name -> triggers.ConsensusQueueEvent + 2, // 8: triggers.ConsensusQueueObservation.events:type_name -> triggers.ConsensusQueueEvent + 11, // 9: triggers.ConsensusQueueObservation.expiration:type_name -> google.protobuf.Timestamp + 10, // 10: triggers.ConsensusQueueObservation.workflowCapacities:type_name -> triggers.ConsensusQueueObservation.WorkflowCapacitiesEntry + 8, // 11: triggers.ConsensusQueueObservedEvents.observations:type_name -> triggers.ConsensusQueueObservedEventEntry + 9, // 12: triggers.ConsensusQueueObservedEventEntry.event:type_name -> triggers.ConsensusQueueObservedEvent + 11, // 13: triggers.ConsensusQueueObservedEvent.observedAts:type_name -> google.protobuf.Timestamp + 11, // 14: triggers.ConsensusQueueObservedEvent.deadlines:type_name -> google.protobuf.Timestamp + 3, // 15: triggers.ConsensusQueueObservedEvent.blobs:type_name -> triggers.CoordinatedEventBlobHandle + 16, // [16:16] is the sub-list for method output_type + 16, // [16:16] is the sub-list for method input_type + 16, // [16:16] is the sub-list for extension type_name + 16, // [16:16] is the sub-list for extension extendee + 0, // [0:16] is the sub-list for field type_name } func init() { file_triggers_proto_init() } @@ -269,7 +773,7 @@ func file_triggers_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_triggers_proto_rawDesc), len(file_triggers_proto_rawDesc)), NumEnums: 0, - NumMessages: 3, + NumMessages: 11, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/capabilities/pb/triggers.proto b/pkg/capabilities/pb/triggers.proto index 1042eacb7f..5d955b1732 100644 --- a/pkg/capabilities/pb/triggers.proto +++ b/pkg/capabilities/pb/triggers.proto @@ -13,14 +13,68 @@ message CoordinatedEvent { int64 triggerIndex = 3; google.protobuf.Timestamp observedAt = 4; google.protobuf.Timestamp deadline = 5; - uint64 sequenceNumber = 6; - capabilities.TriggerResponse event = 7; + capabilities.TriggerResponse event = 6; } -message ConsensusQueueObservation { +message CoordinatedEventList { repeated CoordinatedEvent events = 1; } -message ConsensusQueueReport { - repeated CoordinatedEvent events = 1; -} \ No newline at end of file +// ConsensusQueueEvent holds minimal event metadata for queueing and medianization within the plugin, and one or more +// blob reference pointing to the rest. +message ConsensusQueueEvent { + string workflowID = 1; + string triggerEventID = 2; + google.protobuf.Timestamp observedAt = 3; + google.protobuf.Timestamp deadline = 4; + bytes timelessHash = 5; // hash of the blob data, timestamp fields we have extracted for medianization set to nil + repeated CoordinatedEventBlobHandle blobs = 6; +} + +// ConsensusQueueEventsBlobHandle refers to a CoordinatedEvent by blob handle and index within an CoordinatedEventList. +message CoordinatedEventBlobHandle { + uint32 blob = 1; + uint32 element = 2; +} + +// ConsensusQueueEvents holds a list of events. +message ConsensusQueueEventList { + repeated ConsensusQueueEvent events = 1; +} + +// ConsensusQueueBlobHandleList holds a list of blob handles referenced from ConsensusQueueEventList events. +message ConsensusQueueBlobHandleList { + repeated bytes blobHandles = 1; +} + +// ConsensusQueueObservation is a ReportingPlugin observation of events and capacity meta-data. +message ConsensusQueueObservation { + repeated ConsensusQueueEvent events = 1; // sorted + repeated bytes blobHandles = 2; // referenced from events by index + + google.protobuf.Timestamp expiration = 3; // observation time - grace period + uint32 capacity = 4; // Global remaining capacity + map workflowCapacities = 5; // Workflow-specific free executions +} + +// ConsensusQueueObservedEvents holds a working set of observed events that are awaiting consensus. +message ConsensusQueueObservedEvents { + repeated ConsensusQueueObservedEventEntry observations = 1; // map entries since we have []byte keys + repeated bytes blobHandles = 2; // referenced from observations events by index +} + +message ConsensusQueueObservedEventEntry { + bytes timelessHash = 1; // key + ConsensusQueueObservedEvent event = 2; // val +} + +// ConsensusQueueObservedEvent tracks observation metadata for events awaiting consensus. +message ConsensusQueueObservedEvent { + string workflowID = 1; + string triggerEventID = 2; + + bytes nodeIds = 3; // []OracleID + repeated google.protobuf.Timestamp observedAts = 4; + repeated google.protobuf.Timestamp deadlines = 5; + repeated CoordinatedEventBlobHandle blobs = 6; +} diff --git a/pkg/settings/cresettings/README.md b/pkg/settings/cresettings/README.md index 6066836a99..b07a6414a5 100644 --- a/pkg/settings/cresettings/README.md +++ b/pkg/settings/cresettings/README.md @@ -463,7 +463,8 @@ flowchart CentralTriggerQueue.Drain %% TODO placating test for now since this flowchart no longer renders - DonTimeSequencedTimestampsActivePeriod + DonTimeSequencedTimestampsActivePeriod + ConsensusQueueExpirationInterval %% standard capabilities job startup is likewise its own entry point standardcapabilities.Delegate.NewServices diff --git a/pkg/settings/cresettings/defaults.json b/pkg/settings/cresettings/defaults.json index 1b544edf4b..a6a631de7a 100644 --- a/pkg/settings/cresettings/defaults.json +++ b/pkg/settings/cresettings/defaults.json @@ -39,6 +39,7 @@ "DispatcherPerSenderRate": "500rps:1000", "CentralTriggerEventQueueLimit": "1000", "CentralTriggerQueueDrainInterval": "500ms", + "ConsensusQueueExpirationInterval": "1h0m0s", "TriggerRegistrationStatusUpdateTimeout": "0s", "BaseTriggerRetryInterval": "30s", "BaseTriggerMaxRetries": "20", diff --git a/pkg/settings/cresettings/defaults.toml b/pkg/settings/cresettings/defaults.toml index d9be44c4b0..762e5869f1 100644 --- a/pkg/settings/cresettings/defaults.toml +++ b/pkg/settings/cresettings/defaults.toml @@ -38,6 +38,7 @@ DispatcherGlobalRate = '5000rps:5000' DispatcherPerSenderRate = '500rps:1000' CentralTriggerEventQueueLimit = '1000' CentralTriggerQueueDrainInterval = '500ms' +ConsensusQueueExpirationInterval = '1h0m0s' TriggerRegistrationStatusUpdateTimeout = '0s' BaseTriggerRetryInterval = '30s' BaseTriggerMaxRetries = '20' diff --git a/pkg/settings/cresettings/settings.go b/pkg/settings/cresettings/settings.go index 654238eb33..f78f8fe61e 100644 --- a/pkg/settings/cresettings/settings.go +++ b/pkg/settings/cresettings/settings.go @@ -106,6 +106,7 @@ var Default = Schema{ DispatcherPerSenderRate: Rate(rate.Limit(500), 1000), CentralTriggerEventQueueLimit: Int(1000), CentralTriggerQueueDrainInterval: Duration(500 * time.Millisecond), + ConsensusQueueExpirationInterval: Duration(time.Hour), TriggerRegistrationStatusUpdateTimeout: Duration(0 * time.Second), BaseTriggerRetryInterval: Duration(30 * time.Second), BaseTriggerMaxRetries: Int(20), @@ -477,7 +478,9 @@ type Schema struct { CentralTriggerEventQueueLimit Setting[int] `unit:"{trigger}"` // CentralTriggerQueueDrainInterval is how often the central trigger queue is drained. The default // matches the expected OCR round interval. The drainer raises values below 200ms to 200ms. - CentralTriggerQueueDrainInterval Setting[time.Duration] + CentralTriggerQueueDrainInterval Setting[time.Duration] + // How long past their deadline to expire consensus queue entries + ConsensusQueueExpirationInterval Setting[time.Duration] TriggerRegistrationStatusUpdateTimeout Setting[time.Duration] BaseTriggerRetryInterval Setting[time.Duration]