diff --git a/stovepipe/extension/storage/BUILD.bazel b/stovepipe/extension/storage/BUILD.bazel index fc12fdf84..2ebebccaa 100644 --- a/stovepipe/extension/storage/BUILD.bazel +++ b/stovepipe/extension/storage/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ + "build_issue_store.go", "build_store.go", "queue_log_store.go", "queue_store.go", diff --git a/stovepipe/extension/storage/README.md b/stovepipe/extension/storage/README.md index de79c8959..e0777fd6e 100644 --- a/stovepipe/extension/storage/README.md +++ b/stovepipe/extension/storage/README.md @@ -1,6 +1,6 @@ # Storage -Pluggable persistence interfaces for Stovepipe entities (`RequestStore`, `RequestURIStore`, `QueueStore`, `BuildStore`). Implementations live under `extension/storage//`. +Pluggable persistence interfaces for Stovepipe entities (`RequestStore`, `RequestURIStore`, `QueueStore`, `BuildStore`, `BuildIssueStore`). Implementations live under `extension/storage//`. The aggregate is resolved per queue through a factory keyed by queue name, mirroring the extension contract and the [submitqueue storage seam](../../../submitqueue/extension/storage/README.md#queue-scoped-resolution): a resolved instance is bound to its queue, entity arguments whose queue disagrees with the binding are rejected, and reads never surface another queue's records. Stovepipe has no cross-queue read paths, so every store β€” including `QueueStore`, whose row key is the queue name itself β€” lives inside the queue-scoped aggregate; there is no global remainder. @@ -23,3 +23,11 @@ Same design space as [`submitqueue/extension/storage`](../../../submitqueue/exte `RequestAcceptanceStore` is the ordered lookup mapping for List: immutable `(queue, accepted_at_ms, request_id)` keys with bounded primary-key range scans, not a secondary index or a second status projection. The request-log materializer ensures each known acceptance has a mapping after summary persistence, including retries where the summary is unchanged. Unknown acceptance times are not mapped; historical backfill is separate. `QueueStore` owns internal enablement alongside pipeline coordination. `QueueSummaryStore` persists the current read model projected from immutable `QueueLogStore` occurrences. Enablement sequences, operational row versions and projection versions are distinct. These stores use primary-key operations; version arithmetic, duplicate reconciliation and projection repair belong to business logic. See [queue state](../../../doc/rfc/stovepipe/queue-state.md). + +## Build issue observations + +`BuildIssueStore` accumulates immutable observations identified by build, task, and unit within the bound queue. The build ID is supplied separately from the detected issue. Repeated observations return `ErrAlreadyExists`, which callers may treat as successful persistence. Empty unit IDs represent task-wide observations. + +Each observation is stored separately, not in a serialized collection. Observations are historical evidence; they do not track whether an issue is currently open or resolved. No update, deletion, or absence-based recovery is implied by this store. + +The MySQL primary key contains the queue, build ID, task ID, and unit ID directly as binary strings, preserving byte-exact identity. Their limits are 255, 1020, 255, and 1024 bytes respectively, for a combined maximum of 2554 bytes. Creates and reads reject identifiers exceeding these limits before querying the database; identifiers are never truncated. diff --git a/stovepipe/extension/storage/build_issue_store.go b/stovepipe/extension/storage/build_issue_store.go new file mode 100644 index 000000000..8633bc3c9 --- /dev/null +++ b/stovepipe/extension/storage/build_issue_store.go @@ -0,0 +1,34 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +//go:generate mockgen -source=build_issue_store.go -destination=mock/build_issue_store_mock.go -package=mock + +import ( + "context" + + "github.com/uber/submitqueue/stovepipe/entity" +) + +// BuildIssueStore retains immutable observations keyed by build, task, and unit +// within the bound queue. Presence records an observation, not a currently open issue. +type BuildIssueStore interface { + // Create returns ErrAlreadyExists for a previously recorded build/task/unit. + // Callers may treat that duplicate as success when accumulating observations. + Create(ctx context.Context, buildID string, issue entity.BuildIssue) error + + // Get returns ErrNotFound when the build/task/unit has not been recorded. + Get(ctx context.Context, buildID, taskID, unitID string) (entity.BuildIssue, error) +} diff --git a/stovepipe/extension/storage/mock/BUILD.bazel b/stovepipe/extension/storage/mock/BUILD.bazel index dab7cef72..ad9f4504a 100644 --- a/stovepipe/extension/storage/mock/BUILD.bazel +++ b/stovepipe/extension/storage/mock/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ + "build_issue_store_mock.go", "build_store_mock.go", "queue_log_store_mock.go", "queue_store_mock.go", diff --git a/stovepipe/extension/storage/mock/build_issue_store_mock.go b/stovepipe/extension/storage/mock/build_issue_store_mock.go new file mode 100644 index 000000000..e97faa051 --- /dev/null +++ b/stovepipe/extension/storage/mock/build_issue_store_mock.go @@ -0,0 +1,71 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: build_issue_store.go +// +// Generated by this command: +// +// mockgen -source=build_issue_store.go -destination=mock/build_issue_store_mock.go -package=mock +// + +// Package mock is a generated GoMock package. +package mock + +import ( + context "context" + reflect "reflect" + + entity "github.com/uber/submitqueue/stovepipe/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockBuildIssueStore is a mock of BuildIssueStore interface. +type MockBuildIssueStore struct { + ctrl *gomock.Controller + recorder *MockBuildIssueStoreMockRecorder + isgomock struct{} +} + +// MockBuildIssueStoreMockRecorder is the mock recorder for MockBuildIssueStore. +type MockBuildIssueStoreMockRecorder struct { + mock *MockBuildIssueStore +} + +// NewMockBuildIssueStore creates a new mock instance. +func NewMockBuildIssueStore(ctrl *gomock.Controller) *MockBuildIssueStore { + mock := &MockBuildIssueStore{ctrl: ctrl} + mock.recorder = &MockBuildIssueStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockBuildIssueStore) EXPECT() *MockBuildIssueStoreMockRecorder { + return m.recorder +} + +// Create mocks base method. +func (m *MockBuildIssueStore) Create(ctx context.Context, buildID string, issue entity.BuildIssue) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", ctx, buildID, issue) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create. +func (mr *MockBuildIssueStoreMockRecorder) Create(ctx, buildID, issue any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockBuildIssueStore)(nil).Create), ctx, buildID, issue) +} + +// Get mocks base method. +func (m *MockBuildIssueStore) Get(ctx context.Context, buildID, taskID, unitID string) (entity.BuildIssue, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, buildID, taskID, unitID) + ret0, _ := ret[0].(entity.BuildIssue) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockBuildIssueStoreMockRecorder) Get(ctx, buildID, taskID, unitID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockBuildIssueStore)(nil).Get), ctx, buildID, taskID, unitID) +} diff --git a/stovepipe/extension/storage/mock/storage_mock.go b/stovepipe/extension/storage/mock/storage_mock.go index 11d137b25..b39ce22d7 100644 --- a/stovepipe/extension/storage/mock/storage_mock.go +++ b/stovepipe/extension/storage/mock/storage_mock.go @@ -79,6 +79,20 @@ func (m *MockStorage) EXPECT() *MockStorageMockRecorder { return m.recorder } +// GetBuildIssueStore mocks base method. +func (m *MockStorage) GetBuildIssueStore() storage.BuildIssueStore { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBuildIssueStore") + ret0, _ := ret[0].(storage.BuildIssueStore) + return ret0 +} + +// GetBuildIssueStore indicates an expected call of GetBuildIssueStore. +func (mr *MockStorageMockRecorder) GetBuildIssueStore() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBuildIssueStore", reflect.TypeOf((*MockStorage)(nil).GetBuildIssueStore)) +} + // GetBuildStore mocks base method. func (m *MockStorage) GetBuildStore() storage.BuildStore { m.ctrl.T.Helper() diff --git a/stovepipe/extension/storage/mysql/BUILD.bazel b/stovepipe/extension/storage/mysql/BUILD.bazel index 26edd5e8b..25a1344f7 100644 --- a/stovepipe/extension/storage/mysql/BUILD.bazel +++ b/stovepipe/extension/storage/mysql/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ + "build_issue_store.go", "build_store.go", "queue_log_store.go", "queue_store.go", @@ -29,6 +30,7 @@ go_library( go_test( name = "go_default_test", srcs = [ + "build_issue_store_test.go", "build_store_test.go", "queue_store_test.go", "queue_summary_store_test.go", diff --git a/stovepipe/extension/storage/mysql/build_issue_store.go b/stovepipe/extension/storage/mysql/build_issue_store.go new file mode 100644 index 000000000..a7c825b38 --- /dev/null +++ b/stovepipe/extension/storage/mysql/build_issue_store.go @@ -0,0 +1,106 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mysql + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/uber-go/tally" + + "github.com/uber/submitqueue/platform/metrics" + "github.com/uber/submitqueue/stovepipe/entity" + "github.com/uber/submitqueue/stovepipe/extension/storage" +) + +type buildIssueStore struct { + db *sql.DB + scope tally.Scope + queue string +} + +// NewBuildIssueStore binds observations to queue over the shared database. +func NewBuildIssueStore(db *sql.DB, scope tally.Scope, queue string) storage.BuildIssueStore { + return &buildIssueStore{db: db, scope: scope, queue: queue} +} + +func (s *buildIssueStore) Create(ctx context.Context, buildID string, issue entity.BuildIssue) (retErr error) { + op := metrics.Begin(s.scope, "create", metrics.StorageLatencyBuckets) + defer func() { op.Complete(retErr) }() + + if err := validateBuildIssueKey(s.queue, buildID, issue.TaskID, issue.UnitID); err != nil { + return err + } + _, err := s.db.ExecContext(ctx, + `INSERT INTO build_issue (queue, build_id, task_id, unit_id) + VALUES (?, ?, ?, ?)`, + s.queue, buildID, issue.TaskID, issue.UnitID, + ) + if err != nil { + if isDuplicateEntry(err) { + return fmt.Errorf("build issue build=%q task=%q unit=%q: %w", buildID, issue.TaskID, issue.UnitID, storage.ErrAlreadyExists) + } + + return fmt.Errorf("failed to create build issue build=%q task=%q unit=%q: %w", buildID, issue.TaskID, issue.UnitID, err) + } + + return nil +} + +func (s *buildIssueStore) Get(ctx context.Context, buildID, taskID, unitID string) (ret entity.BuildIssue, retErr error) { + op := metrics.Begin(s.scope, "get", metrics.StorageLatencyBuckets) + defer func() { op.Complete(retErr) }() + + if err := validateBuildIssueKey(s.queue, buildID, taskID, unitID); err != nil { + return entity.BuildIssue{}, err + } + + var issue entity.BuildIssue + err := s.db.QueryRowContext(ctx, + `SELECT task_id, unit_id FROM build_issue + WHERE queue = ? AND build_id = ? AND task_id = ? AND unit_id = ?`, + s.queue, buildID, taskID, unitID, + ).Scan(&issue.TaskID, &issue.UnitID) + if errors.Is(err, sql.ErrNoRows) { + return entity.BuildIssue{}, storage.WrapNotFound(err) + } + + if err != nil { + return entity.BuildIssue{}, fmt.Errorf("failed to get build issue build=%q task=%q unit=%q: %w", buildID, taskID, unitID, err) + } + + return issue, nil +} + +func validateBuildIssueKey(queue, buildID, taskID, unitID string) error { + for _, field := range []struct { + name string + value string + maxBytes int + }{ + {"queue", queue, 255}, + {"build ID", buildID, 1020}, + {"task ID", taskID, 255}, + {"unit ID", unitID, 1024}, + } { + if len(field.value) > field.maxBytes { + return fmt.Errorf("build issue %s exceeds %d-byte limit", field.name, field.maxBytes) + } + } + + return nil +} diff --git a/stovepipe/extension/storage/mysql/build_issue_store_test.go b/stovepipe/extension/storage/mysql/build_issue_store_test.go new file mode 100644 index 000000000..47eeddc6f --- /dev/null +++ b/stovepipe/extension/storage/mysql/build_issue_store_test.go @@ -0,0 +1,164 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mysql + +import ( + "context" + "database/sql" + "errors" + "regexp" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/require" + + "github.com/uber/submitqueue/stovepipe/entity" + "github.com/uber/submitqueue/stovepipe/extension/storage" +) + +func TestBuildIssueStoreCreate(t *testing.T) { + dbErr := errors.New("database unavailable") + + for _, tt := range []struct { + name string + unitID string + dbErr error + wantErr error + }{ + {name: "unit observation", unitID: "unit"}, + {name: "task-wide observation"}, + {name: "long unit identifier", unitID: strings.Repeat("u", 1024)}, + {name: "duplicate observation", unitID: "unit", dbErr: &mysql.MySQLError{Number: mysqlErrDuplicateEntry}, wantErr: storage.ErrAlreadyExists}, + {name: "dependency error", unitID: "unit", dbErr: dbErr, wantErr: dbErr}, + } { + t.Run(tt.name, func(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + store := NewBuildIssueStore(db, testMetrics(), "queue") + issue := entity.BuildIssue{TaskID: "task", UnitID: tt.unitID} + + expectation := mock.ExpectExec(regexp.QuoteMeta(`INSERT INTO build_issue (queue, build_id, task_id, unit_id) VALUES (?, ?, ?, ?)`)). + WithArgs("queue", "build", issue.TaskID, issue.UnitID) + if tt.dbErr != nil { + expectation.WillReturnError(tt.dbErr) + } else { + expectation.WillReturnResult(sqlmock.NewResult(0, 1)) + } + + err = store.Create(context.Background(), "build", issue) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + + require.NoError(t, mock.ExpectationsWereMet()) + }) + } +} + +func TestBuildIssueStoreGet(t *testing.T) { + dbErr := errors.New("database unavailable") + + for _, tt := range []struct { + name string + unitID string + queryErr error + scanErr bool + wantErr error + }{ + {name: "unit observation", unitID: "unit"}, + {name: "task-wide observation"}, + {name: "missing observation", unitID: "unit", queryErr: sql.ErrNoRows, wantErr: storage.ErrNotFound}, + {name: "dependency error", unitID: "unit", queryErr: dbErr, wantErr: dbErr}, + {name: "scan error", unitID: "unit", scanErr: true}, + } { + t.Run(tt.name, func(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + store := NewBuildIssueStore(db, testMetrics(), "queue") + issue := entity.BuildIssue{TaskID: "task", UnitID: tt.unitID} + + expectation := mock.ExpectQuery(regexp.QuoteMeta(`SELECT task_id, unit_id FROM build_issue WHERE queue = ? AND build_id = ? AND task_id = ? AND unit_id = ?`)). + WithArgs("queue", "build", issue.TaskID, issue.UnitID) + if tt.queryErr != nil { + expectation.WillReturnError(tt.queryErr) + } else if tt.scanErr { + expectation.WillReturnRows(sqlmock.NewRows([]string{"task_id", "unit_id"}).AddRow(nil, issue.UnitID)) + } else { + expectation.WillReturnRows(sqlmock.NewRows([]string{"task_id", "unit_id"}).AddRow(issue.TaskID, issue.UnitID)) + } + + got, err := store.Get(context.Background(), "build", issue.TaskID, issue.UnitID) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + } else if tt.scanErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, issue, got) + } + + if tt.wantErr != nil || tt.scanErr { + require.Equal(t, entity.BuildIssue{}, got) + } + + require.NoError(t, mock.ExpectationsWereMet()) + }) + } +} + +func TestBuildIssueStoreRejectsOversizedIdentifiers(t *testing.T) { + for _, tt := range []struct { + name string + queue string + buildID string + issue entity.BuildIssue + }{ + {name: "queue", queue: strings.Repeat("q", 256), buildID: "build", issue: entity.BuildIssue{TaskID: "task"}}, + {name: "build ID", queue: "queue", buildID: strings.Repeat("b", 1021), issue: entity.BuildIssue{TaskID: "task"}}, + {name: "task ID", queue: "queue", buildID: "build", issue: entity.BuildIssue{TaskID: strings.Repeat("t", 256)}}, + {name: "unit ID", queue: "queue", buildID: "build", issue: entity.BuildIssue{TaskID: "task", UnitID: strings.Repeat("u", 1025)}}, + {name: "multibyte task ID", queue: "queue", buildID: "build", issue: entity.BuildIssue{TaskID: strings.Repeat("Γ©", 128)}}, + {name: "multibyte unit ID", queue: "queue", buildID: "build", issue: entity.BuildIssue{TaskID: "task", UnitID: strings.Repeat("πŸ˜€", 257)}}, + } { + t.Run(tt.name, func(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + store := NewBuildIssueStore(db, testMetrics(), tt.queue) + + // SQL would succeed, so rejection must come from identifier validation. + mock.MatchExpectationsInOrder(false) + mock.ExpectExec("INSERT INTO build_issue").WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery("SELECT task_id, unit_id FROM build_issue").WillReturnRows( + sqlmock.NewRows([]string{"task_id", "unit_id"}).AddRow(tt.issue.TaskID, tt.issue.UnitID), + ) + + require.Error(t, store.Create(context.Background(), tt.buildID, tt.issue)) + + got, err := store.Get(context.Background(), tt.buildID, tt.issue.TaskID, tt.issue.UnitID) + require.Error(t, err) + require.Equal(t, entity.BuildIssue{}, got) + }) + } +} diff --git a/stovepipe/extension/storage/mysql/schema/build_issue.sql b/stovepipe/extension/storage/mysql/schema/build_issue.sql new file mode 100644 index 000000000..b50fdde18 --- /dev/null +++ b/stovepipe/extension/storage/mysql/schema/build_issue.sql @@ -0,0 +1,11 @@ +-- One immutable observation per (queue, build, task, unit), not an open-issue projection. +-- Binary identifiers preserve byte-exact identity; the full key is at most 2554 bytes. +CREATE TABLE IF NOT EXISTS build_issue ( + queue VARBINARY(255) NOT NULL, + build_id VARBINARY(1020) NOT NULL, -- Accommodates existing VARCHAR(255) utf8mb4 build IDs. + + task_id VARBINARY(255) NOT NULL, + unit_id VARBINARY(1024) NOT NULL, + + PRIMARY KEY (queue, build_id, task_id, unit_id) +) ENGINE=InnoDB; diff --git a/stovepipe/extension/storage/mysql/storage.go b/stovepipe/extension/storage/mysql/storage.go index e3961c95a..5acb6e87d 100644 --- a/stovepipe/extension/storage/mysql/storage.go +++ b/stovepipe/extension/storage/mysql/storage.go @@ -55,6 +55,7 @@ func (s *Storage) For(queueName string) (storage.Storage, error) { queueLogStore: NewQueueLogStore(s.db, queueScope.SubScope("queue_log_store"), queueName), queueStore: NewQueueStore(s.db, queueScope.SubScope("queue_store"), queueName), buildStore: NewBuildStore(s.db, queueScope.SubScope("build_store"), queueName), + buildIssueStore: NewBuildIssueStore(s.db, queueScope.SubScope("build_issue_store"), queueName), validationFactStore: NewValidationFactStore(s.db, queueScope.SubScope("validation_fact_store"), queueName), }, nil } @@ -75,6 +76,7 @@ type mysqlStorage struct { queueSummaryStore storage.QueueSummaryStore queueLogStore storage.QueueLogStore buildStore storage.BuildStore + buildIssueStore storage.BuildIssueStore validationFactStore storage.ValidationFactStore } @@ -126,3 +128,7 @@ func (f *mysqlStorage) GetQueueSummaryStore() storage.QueueSummaryStore { return func (f *mysqlStorage) GetQueueLogStore() storage.QueueLogStore { return f.queueLogStore } + +func (f *mysqlStorage) GetBuildIssueStore() storage.BuildIssueStore { + return f.buildIssueStore +} diff --git a/stovepipe/extension/storage/mysql/storage_test.go b/stovepipe/extension/storage/mysql/storage_test.go index b80f5017c..326070bd1 100644 --- a/stovepipe/extension/storage/mysql/storage_test.go +++ b/stovepipe/extension/storage/mysql/storage_test.go @@ -47,6 +47,7 @@ func TestNewStorage(t *testing.T) { assert.NotNil(t, bound.GetRequestSummaryStore()) assert.NotNil(t, bound.GetQueueStore()) assert.NotNil(t, bound.GetBuildStore()) + assert.NotNil(t, bound.GetBuildIssueStore()) _, err = s.For("") assert.Error(t, err, "resolving an empty queue name must fail") @@ -127,6 +128,14 @@ func TestStorageForQueueMetrics(t *testing.T) { return err }, }, + { + name: "build_issue_store", + op: "get", + read: func(bound storage.Storage, _ string) error { + _, err := bound.GetBuildIssueStore().Get(context.Background(), "build-id", "task", "unit") + return err + }, + }, { name: "validation_fact_store", op: "get", diff --git a/stovepipe/extension/storage/storage.go b/stovepipe/extension/storage/storage.go index f0bb5d079..21b9886b6 100644 --- a/stovepipe/extension/storage/storage.go +++ b/stovepipe/extension/storage/storage.go @@ -94,6 +94,9 @@ type Storage interface { // GetBuildStore returns the BuildStore instance. GetBuildStore() BuildStore + // GetBuildIssueStore returns the BuildIssueStore instance. + GetBuildIssueStore() BuildIssueStore + // GetValidationFactStore returns the ValidationFactStore instance. GetValidationFactStore() ValidationFactStore } diff --git a/test/integration/stovepipe/extension/storage/mysql/BUILD.bazel b/test/integration/stovepipe/extension/storage/mysql/BUILD.bazel index 408c73051..758f35ad9 100644 --- a/test/integration/stovepipe/extension/storage/mysql/BUILD.bazel +++ b/test/integration/stovepipe/extension/storage/mysql/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_test") go_test( name = "go_default_test", srcs = [ + "build_issue_test.go", "queue_state_transitions_test.go", "queue_summary_test.go", "request_acceptance_test.go", diff --git a/test/integration/stovepipe/extension/storage/mysql/build_issue_test.go b/test/integration/stovepipe/extension/storage/mysql/build_issue_test.go new file mode 100644 index 000000000..f877fe7d8 --- /dev/null +++ b/test/integration/stovepipe/extension/storage/mysql/build_issue_test.go @@ -0,0 +1,183 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mysql + +import ( + "context" + "database/sql" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/uber/submitqueue/stovepipe/entity" + "github.com/uber/submitqueue/stovepipe/extension/storage" +) + +func testBuildIssueStore(t *testing.T, ctx context.Context, db *sql.DB, factory storage.Factory) { + t.Helper() + + bound, err := factory.For(storage.Config{QueueName: "issues/main"}) + require.NoError(t, err) + store := bound.GetBuildIssueStore() + + fixtures := []entity.BuildIssue{ + {TaskID: "validation", UnitID: "unit"}, + {TaskID: "validation", UnitID: "Unit"}, + {TaskID: "validation", UnitID: "unit "}, + {TaskID: "Validation", UnitID: "unit"}, + {TaskID: "validation ", UnitID: "unit"}, + {TaskID: "validation"}, + {TaskID: "other", UnitID: "unit"}, + {TaskID: "a\x00b", UnitID: "c"}, + {TaskID: "a", UnitID: "b\x00c"}, + {TaskID: strings.Repeat("怜証", 42), UnitID: strings.Repeat("対豑/", 146)}, + } + + for _, issue := range fixtures { + require.NoError(t, store.Create(ctx, "build", issue)) + require.ErrorIs(t, store.Create(ctx, "build", issue), storage.ErrAlreadyExists) + + got, err := store.Get(ctx, "build", issue.TaskID, issue.UnitID) + require.NoError(t, err) + require.Equal(t, issue, got) + } + + var count int + require.NoError(t, db.QueryRowContext(ctx, "SELECT COUNT(*) FROM build_issue WHERE queue = ? AND build_id = ?", "issues/main", "build").Scan(&count)) + require.Equal(t, len(fixtures), count) + + issue := fixtures[0] + for _, buildID := range []string{"next-build", "Build", "build "} { + _, err := store.Get(ctx, buildID, issue.TaskID, issue.UnitID) + require.ErrorIs(t, err, storage.ErrNotFound) + + require.NoError(t, store.Create(ctx, buildID, issue)) + + got, err := store.Get(ctx, buildID, issue.TaskID, issue.UnitID) + require.NoError(t, err) + require.Equal(t, issue, got) + } + + for _, queue := range []string{"issues/other", "Issues/main", "issues/main "} { + other, err := factory.For(storage.Config{QueueName: queue}) + require.NoError(t, err) + _, err = other.GetBuildIssueStore().Get(ctx, "build", issue.TaskID, issue.UnitID) + require.ErrorIs(t, err, storage.ErrNotFound) + + require.NoError(t, other.GetBuildIssueStore().Create(ctx, "build", issue)) + + got, err := other.GetBuildIssueStore().Get(ctx, "build", issue.TaskID, issue.UnitID) + require.NoError(t, err) + require.Equal(t, issue, got) + } + + for _, missing := range []entity.BuildIssue{ + {TaskID: "missing", UnitID: "unit"}, + {TaskID: "validation", UnitID: "missing"}, + {TaskID: "other"}, + } { + _, err := store.Get(ctx, "build", missing.TaskID, missing.UnitID) + require.ErrorIs(t, err, storage.ErrNotFound) + } + + t.Run("full primary key at byte limits", func(t *testing.T) { + maxQueue := strings.Repeat("q", 255) + maxBuildID := strings.Repeat("πŸ˜€", 255) + maxIssue := entity.BuildIssue{TaskID: strings.Repeat("Γ©", 127) + "t", UnitID: strings.Repeat("πŸ˜€", 256)} + + maxBound, err := factory.For(storage.Config{QueueName: maxQueue}) + require.NoError(t, err) + maxStore := maxBound.GetBuildIssueStore() + + require.NoError(t, maxStore.Create(ctx, maxBuildID, maxIssue)) + require.ErrorIs(t, maxStore.Create(ctx, maxBuildID, maxIssue), storage.ErrAlreadyExists) + + got, err := maxStore.Get(ctx, maxBuildID, maxIssue.TaskID, maxIssue.UnitID) + require.NoError(t, err) + require.Equal(t, maxIssue, got) + + for _, tt := range []struct { + name string + queue string + buildID string + issue entity.BuildIssue + }{ + {name: "queue", queue: maxQueue + "q", buildID: maxBuildID, issue: maxIssue}, + {name: "build ID", queue: maxQueue, buildID: maxBuildID + "b", issue: maxIssue}, + {name: "task ID", queue: maxQueue, buildID: maxBuildID, issue: entity.BuildIssue{TaskID: maxIssue.TaskID + "t", UnitID: maxIssue.UnitID}}, + {name: "unit ID", queue: maxQueue, buildID: maxBuildID, issue: entity.BuildIssue{TaskID: maxIssue.TaskID, UnitID: maxIssue.UnitID + "u"}}, + } { + t.Run("reject oversized "+tt.name, func(t *testing.T) { + invalidBound, err := factory.For(storage.Config{QueueName: tt.queue}) + require.NoError(t, err) + invalidStore := invalidBound.GetBuildIssueStore() + + err = invalidStore.Create(ctx, tt.buildID, tt.issue) + require.Error(t, err) + require.NotErrorIs(t, err, storage.ErrAlreadyExists) + + _, err = invalidStore.Get(ctx, tt.buildID, tt.issue.TaskID, tt.issue.UnitID) + require.Error(t, err) + require.NotErrorIs(t, err, storage.ErrNotFound) + }) + } + + require.NoError(t, db.QueryRowContext(ctx, "SELECT COUNT(*) FROM build_issue WHERE queue = ?", maxQueue).Scan(&count)) + require.Equal(t, 1, count) + + got, err = maxStore.Get(ctx, maxBuildID, maxIssue.TaskID, maxIssue.UnitID) + require.NoError(t, err) + require.Equal(t, maxIssue, got) + }) + + t.Run("accumulate individual observations", func(t *testing.T) { + for i := 0; i < 128; i++ { + issue := entity.BuildIssue{TaskID: "validation", UnitID: fmt.Sprintf("unit-%d", i)} + require.NoError(t, store.Create(ctx, "many-issues", issue)) + + got, err := store.Get(ctx, "many-issues", issue.TaskID, issue.UnitID) + require.NoError(t, err) + require.Equal(t, issue, got) + } + + require.NoError(t, db.QueryRowContext(ctx, "SELECT COUNT(*) FROM build_issue WHERE queue = ? AND build_id = ?", "issues/main", "many-issues").Scan(&count)) + require.Equal(t, 128, count) + }) + + t.Run("concurrent duplicate observations", func(t *testing.T) { + const writers = 8 + results := make(chan error, writers) + for i := 0; i < writers; i++ { + go func() { results <- store.Create(ctx, "concurrent", issue) }() + } + + created := 0 + for i := 0; i < writers; i++ { + if err := <-results; err == nil { + created++ + } else { + require.ErrorIs(t, err, storage.ErrAlreadyExists) + } + } + + require.Equal(t, 1, created) + + got, err := store.Get(ctx, "concurrent", issue.TaskID, issue.UnitID) + require.NoError(t, err) + require.Equal(t, issue, got) + }) +} diff --git a/test/integration/stovepipe/extension/storage/mysql/storage_test.go b/test/integration/stovepipe/extension/storage/mysql/storage_test.go index fc93afcaf..cd734ffb5 100644 --- a/test/integration/stovepipe/extension/storage/mysql/storage_test.go +++ b/test/integration/stovepipe/extension/storage/mysql/storage_test.go @@ -90,6 +90,11 @@ func TestMySQLStorage(t *testing.T) { suite.Run(t, testSuite) }) + t.Run("BuildIssueStore", func(t *testing.T) { + resetStorage(t, db) + testBuildIssueStore(t, ctx, db, factory) + }) + t.Run("BuildStore", func(t *testing.T) { resetStorage(t, db) testSuite := new(MySQLBuildStoreSuite) @@ -151,6 +156,7 @@ func resetStorage(t *testing.T, db *sql.DB) { "TRUNCATE TABLE request_uri", "TRUNCATE TABLE request", "TRUNCATE TABLE build", + "TRUNCATE TABLE build_issue", "TRUNCATE TABLE queue", "TRUNCATE TABLE validation_fact", } {