Skip to content

Commit 87f3ea3

Browse files
committed
feat(stovepipe): Persist build analysis reports
1 parent 2727b89 commit 87f3ea3

18 files changed

Lines changed: 651 additions & 2 deletions

‎stovepipe/entity/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go_library(
44
name = "go_default_library",
55
srcs = [
66
"build.go",
7+
"build_analysis.go",
78
"build_report.go",
89
"ingest.go",
910
"list.go",

‎stovepipe/entity/build_analysis.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Copyright (c) 2026 Uber Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package entity
16+
17+
// BuildAnalysis is the current validation-report snapshot for one build.
18+
// BuildID is immutable; reports are replaced as a complete snapshot.
19+
type BuildAnalysis struct {
20+
// BuildID identifies the build within its validation queue.
21+
BuildID string `json:"build_id"`
22+
// Reports contains the current attempt of each observed validation task.
23+
Reports []BuildReport `json:"reports"`
24+
// ObservedAtMs is the observation time in milliseconds since the Unix epoch.
25+
ObservedAtMs int64 `json:"observed_at_ms"`
26+
// Version is the optimistic storage version, starting at one.
27+
Version int32 `json:"version"`
28+
}

‎stovepipe/extension/storage/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test")
33
go_library(
44
name = "go_default_library",
55
srcs = [
6+
"build_analysis_store.go",
67
"build_store.go",
78
"queue_log_store.go",
89
"queue_store.go",

‎stovepipe/extension/storage/README.md‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
# Storage
22

3-
Pluggable persistence interfaces for Stovepipe entities (`RequestStore`, `RequestURIStore`, `QueueStore`, `BuildStore`). Implementations live under `extension/storage/<impl>/`.
3+
Pluggable persistence interfaces for Stovepipe entities (`RequestStore`, `RequestURIStore`, `QueueStore`, `BuildStore`, `BuildAnalysisStore`). Implementations live under `extension/storage/<impl>/`.
44

55
The aggregate is resolved per queue through a factory keyed by queue name, mirroring the extension contract and the [submitqueue storage seam](../../../submitqueue/extension/storage/README.md#queue-scoped-resolution): a resolved instance is bound to its queue, entity arguments whose queue disagrees with the binding are rejected, and reads never surface another queue's records. Stovepipe has no cross-queue read paths, so every store — including `QueueStore`, whose row key is the queue name itself — lives inside the queue-scoped aggregate; there is no global remainder.
66

77
This is a separate contract from `submitqueue/extension/storage` — same shape and conventions by design, but its own interfaces and its own `ErrNotFound`/`ErrAlreadyExists`/`ErrVersionMismatch` sentinels, since Stovepipe and SubmitQueue are independent domains. `ErrVersionMismatch` is declared as a retryable infrastructure error so callers can return it without reclassifying it.
88

99
## Optimistic locking contract
1010

11-
Entities that support concurrent mutation (`Request`, `Build`) carry an `int32 Version` field. `Update` methods take both `oldVersion` (the where-clause guard) and `newVersion` (the value to write) — the store performs a pure conditional write and never computes `oldVersion + 1` itself. Version arithmetic is owned by the controller: it computes `newVersion`, calls `Update`, and only assigns `entity.Version = newVersion` after the call succeeds. See [AGENTS.md](../../../AGENTS.md) and the [submitqueue storage README](../../../submitqueue/extension/storage/README.md#optimistic-locking-contract) for the full rationale and the caller pattern — the convention is identical here.
11+
Entities that support concurrent mutation (`Request`, `Build`, `BuildAnalysis`) carry an `int32 Version` field. `Update` methods take both `oldVersion` (the where-clause guard) and `newVersion` (the value to write) — the store performs a pure conditional write and never computes `oldVersion + 1` itself. Version arithmetic is owned by the controller: it computes `newVersion`, calls `Update`, and only assigns `entity.Version = newVersion` after the call succeeds. See [AGENTS.md](../../../AGENTS.md) and the [submitqueue storage README](../../../submitqueue/extension/storage/README.md#optimistic-locking-contract) for the full rationale and the caller pattern — the convention is identical here.
1212

1313
## Read-after-write consistency
1414

@@ -23,3 +23,11 @@ Same design space as [`submitqueue/extension/storage`](../../../submitqueue/exte
2323
`RequestAcceptanceStore` is the ordered lookup mapping for List: immutable `(queue, accepted_at_ms, request_id)` keys with bounded primary-key range scans, not a secondary index or a second status projection. The request-log materializer ensures each known acceptance has a mapping after summary persistence, including retries where the summary is unchanged. Unknown acceptance times are not mapped; historical backfill is separate.
2424

2525
`QueueStore` owns internal enablement alongside pipeline coordination. `QueueSummaryStore` persists the current read model projected from immutable `QueueLogStore` occurrences. Enablement sequences, operational row versions and projection versions are distinct. These stores use primary-key operations; version arithmetic, duplicate reconciliation and projection repair belong to business logic. See [queue state](../../../doc/rfc/stovepipe/queue-state.md).
26+
27+
## Build-analysis snapshots
28+
29+
`BuildAnalysisStore` retains one current report snapshot per `(queue, build ID)`. It replaces the full report list under the caller's version guard rather than accumulating failed units across attempts. Observations retain task identity, attempt identity, report availability, retry disposition, and the observation timestamp. Reader output uses the shared domain report values directly, without a second report representation.
30+
31+
`ObservedAtMs` records when the evidence was inspected; it is not a storage version or a provider ordering guarantee. Callers own freshness decisions and must reload and reinspect after a conflicting write rather than applying an old provider response against a newer version. The store does not interpret opaque attempt IDs or infer build completion, recovery, or validation coverage.
32+
33+
The MySQL implementation uses bytewise queue/build keys of up to 1020 bytes. It adds only the `build_analysis` table; existing tables are unchanged. Apply the new schema before enabling a consumer of this store. Nil and empty report or failed-unit slices retain their JSON representations. No pipeline controller reads or writes these snapshots yet.
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Copyright (c) 2026 Uber Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package storage
16+
17+
//go:generate mockgen -source=build_analysis_store.go -destination=mock/build_analysis_store_mock.go -package=mock
18+
19+
import (
20+
"context"
21+
22+
"github.com/uber/submitqueue/stovepipe/entity"
23+
)
24+
25+
// BuildAnalysisStore persists queue-scoped report snapshots keyed by build ID.
26+
// Updates replace all reports; they do not merge failures from prior attempts.
27+
// Callers own version arithmetic and provider-specific observation freshness.
28+
type BuildAnalysisStore interface {
29+
// Create requires a nonempty BuildID and returns ErrAlreadyExists for an existing snapshot.
30+
Create(ctx context.Context, analysis entity.BuildAnalysis) error
31+
32+
// Get returns ErrNotFound when the build has no analysis snapshot.
33+
Get(ctx context.Context, buildID string) (entity.BuildAnalysis, error)
34+
35+
// Update replaces the snapshot when its stored version matches oldVersion,
36+
// writing newVersion without mutating analysis. A missing or stale snapshot
37+
// returns ErrVersionMismatch.
38+
Update(ctx context.Context, analysis entity.BuildAnalysis, oldVersion, newVersion int32) error
39+
}

‎stovepipe/extension/storage/mock/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library")
33
go_library(
44
name = "go_default_library",
55
srcs = [
6+
"build_analysis_store_mock.go",
67
"build_store_mock.go",
78
"queue_log_store_mock.go",
89
"queue_store_mock.go",

‎stovepipe/extension/storage/mock/build_analysis_store_mock.go‎

Lines changed: 85 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎stovepipe/extension/storage/mock/storage_mock.go‎

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎stovepipe/extension/storage/mysql/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test")
33
go_library(
44
name = "go_default_library",
55
srcs = [
6+
"build_analysis_store.go",
67
"build_store.go",
78
"queue_log_store.go",
89
"queue_store.go",
@@ -29,6 +30,7 @@ go_library(
2930
go_test(
3031
name = "go_default_test",
3132
srcs = [
33+
"build_analysis_store_test.go",
3234
"build_store_test.go",
3335
"queue_store_test.go",
3436
"queue_summary_store_test.go",
Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
// Copyright (c) 2026 Uber Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// Unless required by applicable law or agreed to in writing, software
8+
// distributed under the License is distributed on an "AS IS" BASIS,
9+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
// See the License for the specific language governing permissions and
11+
// limitations under the License.
12+
13+
package mysql
14+
15+
import (
16+
"context"
17+
"database/sql"
18+
"encoding/json"
19+
"errors"
20+
"fmt"
21+
22+
"github.com/uber-go/tally"
23+
"github.com/uber/submitqueue/platform/metrics"
24+
"github.com/uber/submitqueue/stovepipe/entity"
25+
"github.com/uber/submitqueue/stovepipe/extension/storage"
26+
)
27+
28+
type buildAnalysisStore struct {
29+
db *sql.DB
30+
scope tally.Scope
31+
queue string
32+
}
33+
34+
// NewBuildAnalysisStore creates a queue-scoped MySQL report snapshot store.
35+
func NewBuildAnalysisStore(db *sql.DB, scope tally.Scope, queue string) storage.BuildAnalysisStore {
36+
return &buildAnalysisStore{db: db, scope: scope, queue: queue}
37+
}
38+
39+
func (s *buildAnalysisStore) Create(ctx context.Context, analysis entity.BuildAnalysis) (retErr error) {
40+
op := metrics.Begin(s.scope, "create", metrics.StorageLatencyBuckets)
41+
defer func() { op.Complete(retErr) }()
42+
43+
if analysis.BuildID == "" {
44+
return fmt.Errorf("build analysis requires a nonempty build ID")
45+
}
46+
reportsJSON, err := json.Marshal(analysis.Reports)
47+
if err != nil {
48+
return fmt.Errorf("marshal build analysis reports: %w", err)
49+
}
50+
_, err = s.db.ExecContext(ctx,
51+
`INSERT INTO build_analysis (queue, build_id, reports, observed_at_ms, version)
52+
VALUES (?, ?, ?, ?, ?)`,
53+
s.queue, analysis.BuildID, reportsJSON, analysis.ObservedAtMs, analysis.Version,
54+
)
55+
if err != nil {
56+
if isDuplicateEntry(err) {
57+
return fmt.Errorf("build analysis %q: %w", analysis.BuildID, storage.ErrAlreadyExists)
58+
}
59+
return fmt.Errorf("insert build analysis %q: %w", analysis.BuildID, err)
60+
}
61+
return nil
62+
}
63+
64+
func (s *buildAnalysisStore) Get(ctx context.Context, buildID string) (ret entity.BuildAnalysis, retErr error) {
65+
op := metrics.Begin(s.scope, "get", metrics.StorageLatencyBuckets)
66+
defer func() { op.Complete(retErr) }()
67+
68+
var analysis entity.BuildAnalysis
69+
var reportsJSON []byte
70+
err := s.db.QueryRowContext(ctx,
71+
`SELECT build_id, reports, observed_at_ms, version
72+
FROM build_analysis WHERE queue = ? AND build_id = ?`,
73+
s.queue, buildID,
74+
).Scan(&analysis.BuildID, &reportsJSON, &analysis.ObservedAtMs, &analysis.Version)
75+
if errors.Is(err, sql.ErrNoRows) {
76+
return entity.BuildAnalysis{}, storage.WrapNotFound(err)
77+
}
78+
if err != nil {
79+
return entity.BuildAnalysis{}, fmt.Errorf("get build analysis %q: %w", buildID, err)
80+
}
81+
if err := json.Unmarshal(reportsJSON, &analysis.Reports); err != nil {
82+
return entity.BuildAnalysis{}, fmt.Errorf("decode build analysis reports for %q: %w", buildID, err)
83+
}
84+
return analysis, nil
85+
}
86+
87+
func (s *buildAnalysisStore) Update(ctx context.Context, analysis entity.BuildAnalysis, oldVersion, newVersion int32) (retErr error) {
88+
op := metrics.Begin(s.scope, "update", metrics.StorageLatencyBuckets)
89+
defer func() { op.Complete(retErr) }()
90+
91+
if analysis.BuildID == "" {
92+
return fmt.Errorf("build analysis requires a nonempty build ID")
93+
}
94+
reportsJSON, err := json.Marshal(analysis.Reports)
95+
if err != nil {
96+
return fmt.Errorf("marshal build analysis reports: %w", err)
97+
}
98+
result, err := s.db.ExecContext(ctx,
99+
`UPDATE build_analysis SET reports = ?, observed_at_ms = ?, version = ?
100+
WHERE queue = ? AND build_id = ? AND version = ?`,
101+
reportsJSON, analysis.ObservedAtMs, newVersion, s.queue, analysis.BuildID, oldVersion,
102+
)
103+
if err != nil {
104+
return fmt.Errorf("update build analysis %q: %w", analysis.BuildID, err)
105+
}
106+
rowsAffected, err := result.RowsAffected()
107+
if err != nil {
108+
return fmt.Errorf("get rows affected for build analysis %q: %w", analysis.BuildID, err)
109+
}
110+
if rowsAffected != 1 {
111+
return fmt.Errorf("build analysis %q expected version %d: %w", analysis.BuildID, oldVersion, storage.ErrVersionMismatch)
112+
}
113+
return nil
114+
}

0 commit comments

Comments
 (0)