Skip to content

Commit 32200d9

Browse files
prathshenoyprathshenoy
committed
feat(stovepipe): Persist build analysis reports
1 parent 722c358 commit 32200d9

19 files changed

Lines changed: 710 additions & 35 deletions

‎stovepipe/entity/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go_library(
44
name = "go_default_library",
55
srcs = [
66
"build.go",
7+
"build_analysis.go",
78
"ingest.go",
89
"list.go",
910
"project_status.go",

‎stovepipe/entity/build_analysis.go‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// Copyright (c) 2026 Uber Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package entity
16+
17+
// BuildReportState describes the availability of a validation report.
18+
type BuildReportState string
19+
20+
const (
21+
// BuildReportStateUnknown means report availability has not been established.
22+
BuildReportStateUnknown BuildReportState = ""
23+
// BuildReportStatePending means a final report is not available yet.
24+
BuildReportStatePending BuildReportState = "pending"
25+
// BuildReportStateAvailable means a final validation report can be interpreted.
26+
BuildReportStateAvailable BuildReportState = "available"
27+
// BuildReportStateUnavailable means the attempt ended without a usable final report.
28+
BuildReportStateUnavailable BuildReportState = "unavailable"
29+
)
30+
31+
// BuildRetryDisposition describes the automatic retry decision for an attempt of a
32+
// validation task, not manual reruns or the retries used to read its evidence.
33+
type BuildRetryDisposition string
34+
35+
const (
36+
// BuildRetryDispositionUnknown means the automatic retry decision is not known.
37+
BuildRetryDispositionUnknown BuildRetryDisposition = ""
38+
// BuildRetryDispositionPending means an automatic retry is scheduled or running.
39+
BuildRetryDispositionPending BuildRetryDisposition = "pending"
40+
// BuildRetryDispositionExhausted means no automatic retry remains under the applicable policy.
41+
BuildRetryDispositionExhausted BuildRetryDisposition = "exhausted"
42+
)
43+
44+
// BuildReport contains validation evidence from the current attempt of a validation task.
45+
// A validation task is work executed and retried together; a unit is the
46+
// granularity of reported outcomes. All identifiers are opaque.
47+
type BuildReport struct {
48+
// TaskID identifies the logical validation task within the build, stable across attempts.
49+
TaskID string `json:"task_id"`
50+
// AttemptID identifies this execution of the validation task; empty if none exists yet.
51+
AttemptID string `json:"attempt_id"`
52+
// State describes whether this attempt's final validation report is available.
53+
State BuildReportState `json:"state"`
54+
// FailedUnits lists failed validation-unit identifiers when State is available.
55+
// An empty list establishes no failure within the report's coverage, not build success.
56+
FailedUnits []string `json:"failed_units"`
57+
// RetryDisposition describes the automatic retry decision for this attempt.
58+
RetryDisposition BuildRetryDisposition `json:"retry_disposition"`
59+
}
60+
61+
// BuildAnalysis is the current validation-report snapshot for one build.
62+
// BuildID is immutable; reports are replaced as a complete snapshot.
63+
type BuildAnalysis struct {
64+
// BuildID identifies the build within its validation queue.
65+
BuildID string `json:"build_id"`
66+
// Reports contains the current attempt of each observed validation task.
67+
Reports []BuildReport `json:"reports"`
68+
// ObservedAtMs is the observation time in milliseconds since the Unix epoch.
69+
ObservedAtMs int64 `json:"observed_at_ms"`
70+
// Version is the optimistic storage version, starting at one.
71+
Version int32 `json:"version"`
72+
}

‎stovepipe/extension/buildanalysis/buildanalysis.go‎

Lines changed: 11 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -25,48 +25,26 @@ import (
2525
)
2626

2727
// ReportState describes the availability of a validation report.
28-
type ReportState string
28+
type ReportState = entity.BuildReportState
2929

3030
const (
31-
// ReportStateUnknown means report availability has not been established.
32-
ReportStateUnknown ReportState = ""
33-
// ReportStatePending means a final report is not available yet.
34-
ReportStatePending ReportState = "pending"
35-
// ReportStateAvailable means a final validation report can be interpreted.
36-
ReportStateAvailable ReportState = "available"
37-
// ReportStateUnavailable means the attempt ended without a usable final report.
38-
ReportStateUnavailable ReportState = "unavailable"
31+
ReportStateUnknown = entity.BuildReportStateUnknown
32+
ReportStatePending = entity.BuildReportStatePending
33+
ReportStateAvailable = entity.BuildReportStateAvailable
34+
ReportStateUnavailable = entity.BuildReportStateUnavailable
3935
)
4036

41-
// RetryDisposition describes the automatic retry decision for an attempt of a
42-
// validation task, not manual reruns or the retries used to read its evidence.
43-
type RetryDisposition string
37+
// RetryDisposition describes automatic retries for an attempt of a validation task.
38+
type RetryDisposition = entity.BuildRetryDisposition
4439

4540
const (
46-
// RetryDispositionUnknown means the automatic retry decision is not known.
47-
RetryDispositionUnknown RetryDisposition = ""
48-
// RetryDispositionPending means an automatic retry is scheduled or running.
49-
RetryDispositionPending RetryDisposition = "pending"
50-
// RetryDispositionExhausted means no automatic retry remains under the applicable policy.
51-
RetryDispositionExhausted RetryDisposition = "exhausted"
41+
RetryDispositionUnknown = entity.BuildRetryDispositionUnknown
42+
RetryDispositionPending = entity.BuildRetryDispositionPending
43+
RetryDispositionExhausted = entity.BuildRetryDispositionExhausted
5244
)
5345

5446
// Report contains validation evidence from the current attempt of a validation task.
55-
// A validation task is work executed and retried together; a unit is the
56-
// granularity of reported outcomes. All identifiers are opaque.
57-
type Report struct {
58-
// TaskID identifies the logical validation task within the build, stable across attempts.
59-
TaskID string
60-
// AttemptID identifies this execution of the validation task; empty if none exists yet.
61-
AttemptID string
62-
// State describes whether this attempt's final validation report is available.
63-
State ReportState
64-
// FailedUnits lists failed validation-unit identifiers when State is available.
65-
// An empty list establishes no failure within the report's coverage, not build success.
66-
FailedUnits []string
67-
// RetryDisposition describes the automatic retry decision for this attempt.
68-
RetryDisposition RetryDisposition
69-
}
47+
type Report = entity.BuildReport
7048

7149
// Reader inspects every relevant task currently discoverable in a build,
7250
// returning one report per task for its current attempt without waiting.

‎stovepipe/extension/storage/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test")
33
go_library(
44
name = "go_default_library",
55
srcs = [
6+
"build_analysis_store.go",
67
"build_store.go",
78
"queue_log_store.go",
89
"queue_store.go",

‎stovepipe/extension/storage/README.md‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
# Storage
22

3-
Pluggable persistence interfaces for Stovepipe entities (`RequestStore`, `RequestURIStore`, `QueueStore`, `BuildStore`). Implementations live under `extension/storage/<impl>/`.
3+
Pluggable persistence interfaces for Stovepipe entities (`RequestStore`, `RequestURIStore`, `QueueStore`, `BuildStore`, `BuildAnalysisStore`). Implementations live under `extension/storage/<impl>/`.
44

55
The aggregate is resolved per queue through a factory keyed by queue name, mirroring the extension contract and the [submitqueue storage seam](../../../submitqueue/extension/storage/README.md#queue-scoped-resolution): a resolved instance is bound to its queue, entity arguments whose queue disagrees with the binding are rejected, and reads never surface another queue's records. Stovepipe has no cross-queue read paths, so every store — including `QueueStore`, whose row key is the queue name itself — lives inside the queue-scoped aggregate; there is no global remainder.
66

77
This is a separate contract from `submitqueue/extension/storage` — same shape and conventions by design, but its own interfaces and its own `ErrNotFound`/`ErrAlreadyExists`/`ErrVersionMismatch` sentinels, since Stovepipe and SubmitQueue are independent domains. `ErrVersionMismatch` is declared as a retryable infrastructure error so callers can return it without reclassifying it.
88

99
## Optimistic locking contract
1010

11-
Entities that support concurrent mutation (`Request`, `Build`) carry an `int32 Version` field. `Update` methods take both `oldVersion` (the where-clause guard) and `newVersion` (the value to write) — the store performs a pure conditional write and never computes `oldVersion + 1` itself. Version arithmetic is owned by the controller: it computes `newVersion`, calls `Update`, and only assigns `entity.Version = newVersion` after the call succeeds. See [AGENTS.md](../../../AGENTS.md) and the [submitqueue storage README](../../../submitqueue/extension/storage/README.md#optimistic-locking-contract) for the full rationale and the caller pattern — the convention is identical here.
11+
Entities that support concurrent mutation (`Request`, `Build`, `BuildAnalysis`) carry an `int32 Version` field. `Update` methods take both `oldVersion` (the where-clause guard) and `newVersion` (the value to write) — the store performs a pure conditional write and never computes `oldVersion + 1` itself. Version arithmetic is owned by the controller: it computes `newVersion`, calls `Update`, and only assigns `entity.Version = newVersion` after the call succeeds. See [AGENTS.md](../../../AGENTS.md) and the [submitqueue storage README](../../../submitqueue/extension/storage/README.md#optimistic-locking-contract) for the full rationale and the caller pattern — the convention is identical here.
1212

1313
## Read-after-write consistency
1414

@@ -23,3 +23,11 @@ Same design space as [`submitqueue/extension/storage`](../../../submitqueue/exte
2323
`RequestAcceptanceStore` is the ordered lookup mapping for List: immutable `(queue, accepted_at_ms, request_id)` keys with bounded primary-key range scans, not a secondary index or a second status projection. The request-log materializer ensures each known acceptance has a mapping after summary persistence, including retries where the summary is unchanged. Unknown acceptance times are not mapped; historical backfill is separate.
2424

2525
`QueueStore` owns internal enablement alongside pipeline coordination. `QueueSummaryStore` persists the current read model projected from immutable `QueueLogStore` occurrences. Enablement sequences, operational row versions and projection versions are distinct. These stores use primary-key operations; version arithmetic, duplicate reconciliation and projection repair belong to business logic. See [queue state](../../../doc/rfc/stovepipe/queue-state.md).
26+
27+
## Build-analysis snapshots
28+
29+
`BuildAnalysisStore` retains one current report snapshot per `(queue, build ID)`. It replaces the full report list under the caller's version guard rather than accumulating failed units across attempts. Observations retain task identity, attempt identity, report availability, retry disposition, and the observation timestamp. The reader's report types alias the shared domain values, so reader output can be persisted without a second report representation.
30+
31+
`ObservedAtMs` records when the evidence was inspected; it is not a storage version or a provider ordering guarantee. Callers own freshness decisions and must reload and reinspect after a conflicting write rather than applying an old provider response against a newer version. The store does not interpret opaque attempt IDs or infer build completion, recovery, or validation coverage.
32+
33+
The MySQL implementation uses bytewise queue/build keys of up to 1020 bytes. It adds only the `build_analysis` table; existing tables are unchanged. Apply the new schema before enabling a consumer of this store. Nil and empty report or failed-unit slices retain their JSON representations. No pipeline controller reads or writes these snapshots yet.

‎stovepipe/extension/storage/build_analysis_store.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Copyright (c) 2026 Uber Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package storage
16+
17+
//go:generate mockgen -source=build_analysis_store.go -destination=mock/build_analysis_store_mock.go -package=mock
18+
19+
import (
20+
"context"
21+
22+
"github.com/uber/submitqueue/stovepipe/entity"
23+
)
24+
25+
// BuildAnalysisStore persists queue-scoped report snapshots keyed by build ID.
26+
// Updates replace all reports; they do not merge failures from prior attempts.
27+
// Callers own version arithmetic and provider-specific observation freshness.
28+
type BuildAnalysisStore interface {
29+
// Create requires a nonempty BuildID and returns ErrAlreadyExists for an existing snapshot.
30+
Create(ctx context.Context, analysis entity.BuildAnalysis) error
31+
32+
// Get returns ErrNotFound when the build has no analysis snapshot.
33+
Get(ctx context.Context, buildID string) (entity.BuildAnalysis, error)
34+
35+
// Update replaces the snapshot when its stored version matches oldVersion,
36+
// writing newVersion without mutating analysis. A missing or stale snapshot
37+
// returns ErrVersionMismatch.
38+
Update(ctx context.Context, analysis entity.BuildAnalysis, oldVersion, newVersion int32) error
39+
}

‎stovepipe/extension/storage/mock/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library")
33
go_library(
44
name = "go_default_library",
55
srcs = [
6+
"build_analysis_store_mock.go",
67
"build_store_mock.go",
78
"queue_log_store_mock.go",
89
"queue_store_mock.go",

‎stovepipe/extension/storage/mock/build_analysis_store_mock.go‎

Lines changed: 85 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎stovepipe/extension/storage/mock/storage_mock.go‎

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎stovepipe/extension/storage/mysql/BUILD.bazel‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test")
33
go_library(
44
name = "go_default_library",
55
srcs = [
6+
"build_analysis_store.go",
67
"build_store.go",
78
"queue_log_store.go",
89
"queue_store.go",
@@ -29,6 +30,7 @@ go_library(
2930
go_test(
3031
name = "go_default_test",
3132
srcs = [
33+
"build_analysis_store_test.go",
3234
"build_store_test.go",
3335
"queue_store_test.go",
3436
"queue_summary_store_test.go",
@@ -43,6 +45,7 @@ go_test(
4345
embed = [":go_default_library"],
4446
deps = [
4547
"//stovepipe/entity:go_default_library",
48+
"//stovepipe/extension/buildanalysis:go_default_library",
4649
"//stovepipe/extension/storage:go_default_library",
4750
"@com_github_data_dog_go_sqlmock//:go_default_library",
4851
"@com_github_go_sql_driver_mysql//:go_default_library",

0 commit comments

Comments
 (0)