feat(stovepipe): Add project ordering index

Author: prathshenoy

doc/rfc/stovepipe/project-validation-index.md

@@ -0,0 +1,17 @@
+# Project validation ordering index
+
+The project validation index is an immutable mapping from `(queue, URI, degree_order, project)` to the fact keyed by `(queue, URI, project)`. It contains no duplicate validation payload. Its primary key supports bounded range reads in descending breakage degree and ascending bytewise project ID, without sorting a non-key attribute or scanning every fact.
+
+`degree_order` reverses nonnegative IEEE-754 degree bit patterns into an ascending integer key without rounding. Positive and negative zero share one key; invalid degrees and whole-repository facts cannot produce mappings. Binary identifiers preserve bytewise tie order.
+
+## Persistence and repair
+
+The record controller persists the winning immutable fact before ensuring its mapping. Redelivery repairs missing mappings even when the fact already exists, and derives their keys from the stored degree rather than a losing candidate. Project-recording logs and downstream terminal notifications follow successful mapping writes. No cross-store transaction is required.
+
+## Rollout
+
+The ordering table is additive; the original fact identity and primary key are unchanged. Install the table before deploying index writers, then adopt index-based readers after the writers are deployed.
+
+The table starts empty. There is no historical backfill or fallback scan: older requests may have empty project listings, and requests spanning the rollout may have partial listings. Existing facts remain available through explicit project lookups. This historical-coverage limitation is intentional; no data migration or compatibility state is required.
+
+Keep the additive table when reverting application adoption. Facts recorded while index writers are disabled remain outside index-based listings unless the normal record step processes them again.

stovepipe/controller/record/record.go

@@ -197,7 +197,7 @@ func (c *Controller) recordProjectFacts(ctx context.Context, store storage.Stora
 		return err
 	}
 	for _, result := range results {
-		_, _, err := c.recordValidationFact(ctx, store.GetValidationFactStore(), entity.ValidationFact{
+		fact, _, err := c.recordValidationFact(ctx, store.GetValidationFactStore(), entity.ValidationFact{
 			URI:       request.URI,
 			Project:   result.Project,
 			Degree:    result.Degree,
@@ -207,6 +207,15 @@ func (c *Controller) recordProjectFacts(ctx context.Context, store storage.Stora
 		if err != nil {
 			return err
 		}
+
+		entry, err := entity.NewProjectValidationIndex(fact)
+		if err != nil {
+			return err
+		}
+
+		if err := store.GetProjectValidationIndexStore().Create(ctx, entry); err != nil && !errors.Is(err, storage.ErrAlreadyExists) {
+			return fmt.Errorf("failed to index project fact for request %q and project %q: %w", request.ID, fact.Project, err)
+		}
 	}
 	if len(results) == 0 {
 		return nil

stovepipe/controller/record/record_test.go

@@ -74,6 +74,7 @@ type recordMocks struct {
 	reqStore      *storagemock.MockRequestStore
 	queueStore    *storagemock.MockQueueStore
 	factStore     *storagemock.MockValidationFactStore
+	indexStore    *storagemock.MockProjectValidationIndexStore
 	store         *storagemock.MockStorage
 	materializer  *requestlogmock.MockMaterializer
 	sourceControl *sourcecontrolmock.MockSourceControl
@@ -148,6 +149,7 @@ func newControllerForTopic(t *testing.T, ctrl *gomock.Controller, topicKey consu
 		reqStore:      storagemock.NewMockRequestStore(ctrl),
 		queueStore:    storagemock.NewMockQueueStore(ctrl),
 		factStore:     storagemock.NewMockValidationFactStore(ctrl),
+		indexStore:    storagemock.NewMockProjectValidationIndexStore(ctrl),
 		store:         storagemock.NewMockStorage(ctrl),
 		materializer:  requestlogmock.NewMockMaterializer(ctrl),
 		sourceControl: sourcecontrolmock.NewMockSourceControl(ctrl),
@@ -158,6 +160,7 @@ func newControllerForTopic(t *testing.T, ctrl *gomock.Controller, topicKey consu
 	m.store.EXPECT().GetRequestStore().Return(m.reqStore).AnyTimes()
 	m.store.EXPECT().GetQueueStore().Return(m.queueStore).AnyTimes()
 	m.store.EXPECT().GetValidationFactStore().Return(m.factStore).AnyTimes()
+	m.store.EXPECT().GetProjectValidationIndexStore().Return(m.indexStore).AnyTimes()
 	m.materializer.EXPECT().PersistLog(gomock.Any(), m.store, gomock.Any()).DoAndReturn(
 		func(_ context.Context, _ storage.Storage, log entity.RequestLog) error {
 			assert.Equal(t, entity.RequestEventValidationFactRecorded, log.Event)
@@ -402,6 +405,13 @@ func TestProcess_RecordsNamedProjectStatusResults(t *testing.T) {
 		}).
 		Times(3)
 
+	var indexes []entity.ProjectValidationIndex
+	m.indexStore.EXPECT().Create(gomock.Any(), gomock.Any()).
+		DoAndReturn(func(_ context.Context, entry entity.ProjectValidationIndex) error {
+			indexes = append(indexes, entry)
+			return nil
+		}).Times(2)
+
 	var logs []entity.RequestLog
 	eventMaterializer.EXPECT().PersistLog(gomock.Any(), m.store, gomock.Any()).
 		DoAndReturn(func(_ context.Context, _ storage.Storage, log entity.RequestLog) error {
@@ -412,6 +422,12 @@ func TestProcess_RecordsNamedProjectStatusResults(t *testing.T) {
 
 	require.NoError(t, c.Process(queueContext(), delivery(t, ctrl, recordPayload(t, testID))))
 	require.Len(t, facts, 3)
+	require.Len(t, indexes, 2)
+	for i, fact := range facts[1:] {
+		entry, err := entity.NewProjectValidationIndex(fact)
+		require.NoError(t, err)
+		assert.Equal(t, entry, indexes[i])
+	}
 	assert.Equal(t, []string{"", "project-a", "project-b"}, []string{facts[0].Project, facts[1].Project, facts[2].Project})
 	for _, fact := range facts {
 		assert.Equal(t, testURI, fact.URI)
@@ -1217,3 +1233,43 @@ func TestProcess_QueueMismatchFails(t *testing.T) {
 
 	require.Error(t, c.Process(queueContext(), d))
 }
+
+func TestRecordProjectFactsRepairsIndexOnRetry(t *testing.T) {
+	for _, retryErr := range []error{nil, storage.ErrAlreadyExists} {
+		name := "index repaired"
+		if retryErr != nil {
+			name = "index already repaired"
+		}
+		t.Run(name, func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			c, m := newController(t, ctrl)
+			request := requestWithState(entity.RequestStateFailed)
+			resolverFactory := projectstatusmock.NewMockResolverFactory(ctrl)
+			resolver := projectstatusmock.NewMockResolver(ctrl)
+			c.projectStatusResolverFactory = resolverFactory
+			resolverFactory.EXPECT().For(projectstatus.ResolverConfig{QueueName: testQueue}).Return(resolver, nil).Times(2)
+			stored := entity.ValidationFact{URI: request.URI, Project: "p", Degree: 0.25, RequestID: request.ID, CreatedAt: 10}
+			entry, err := entity.NewProjectValidationIndex(stored)
+			require.NoError(t, err)
+			writeErr := errors.New("index write unavailable")
+			materializer := requestlogmock.NewMockMaterializer(ctrl)
+			c.materializer = materializer
+
+			gomock.InOrder(
+				resolver.EXPECT().Resolve(gomock.Any(), request, request.TerminalBuildID).Return([]projectstatus.Result{{Project: "p", Degree: 0.25}}, nil),
+				m.factStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil),
+				m.indexStore.EXPECT().Create(gomock.Any(), entry).Return(writeErr),
+				resolver.EXPECT().Resolve(gomock.Any(), request, request.TerminalBuildID).Return([]projectstatus.Result{{Project: "p", Degree: 0.75}}, nil),
+				m.factStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(storage.ErrAlreadyExists),
+				m.factStore.EXPECT().Get(gomock.Any(), request.URI, "p").Return(stored, nil),
+				m.indexStore.EXPECT().Create(gomock.Any(), entry).Return(retryErr),
+				materializer.EXPECT().PersistLog(gomock.Any(), m.store, gomock.Any()).DoAndReturn(func(_ context.Context, _ storage.Storage, log entity.RequestLog) error {
+					assert.Equal(t, entity.RequestEventProjectFactsRecorded, log.Event)
+					return nil
+				}),
+			)
+			require.ErrorIs(t, c.recordProjectFacts(queueContext(), m.store, request), writeErr)
+			require.NoError(t, c.recordProjectFacts(queueContext(), m.store, request))
+		})
+	}
+}

stovepipe/entity/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
         "ingest.go",
         "list.go",
         "project_status.go",
+        "project_validation_index.go",
         "queue.go",
         "queue_config.go",
         "queue_log.go",
@@ -31,6 +32,7 @@ go_test(
     name = "go_default_test",
     srcs = [
         "build_test.go",
+        "project_validation_index_test.go",
         "queue_state_test.go",
         "request_id_test.go",
         "request_log_test.go",

stovepipe/entity/project_validation_index.go

@@ -0,0 +1,45 @@
+// Copyright (c) 2026 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package entity
+
+import (
+	"fmt"
+	"math"
+)
+
+// ProjectValidationIndex maps an ordered key to an immutable project fact within a queue.
+type ProjectValidationIndex struct {
+	// URI is the commit containing the project result.
+	URI string
+	// DegreeOrder is the ascending integer key for descending breakage degree.
+	DegreeOrder int64
+	// Project identifies the fact and breaks equal-degree ties in ascending bytewise order.
+	Project string
+}
+
+// NewProjectValidationIndex derives an exact ordering key from a named project's fact.
+// Invalid degrees, empty URIs and whole-repository facts return an error.
+func NewProjectValidationIndex(fact ValidationFact) (ProjectValidationIndex, error) {
+	if fact.URI == "" || fact.Project == "" || math.IsNaN(fact.Degree) || fact.Degree < DegreeGreen || fact.Degree > DegreeBroken {
+		return ProjectValidationIndex{}, fmt.Errorf("project validation index requires a URI, project and degree in [0, 1]")
+	}
+	degree := fact.Degree
+	if degree == 0 {
+		degree = 0 // Canonicalize negative zero to the same key as positive zero.
+	}
+	// Nonnegative IEEE-754 bit patterns increase with degree; integer subtraction reverses that order without rounding.
+	order := int64(math.Float64bits(DegreeBroken) - math.Float64bits(degree))
+	return ProjectValidationIndex{URI: fact.URI, DegreeOrder: order, Project: fact.Project}, nil
+}

stovepipe/entity/project_validation_index_test.go

@@ -0,0 +1,52 @@
+// Copyright (c) 2026 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package entity
+
+import (
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestProjectValidationIndexOrdersDegreesExactly(t *testing.T) {
+	degrees := []float64{1, math.Nextafter(1, 0), 0.75, math.Nextafter(0.5, 1), 0.5, math.SmallestNonzeroFloat64, 0}
+	var previous ProjectValidationIndex
+	for i, degree := range degrees {
+		entry, err := NewProjectValidationIndex(ValidationFact{URI: "commit", Project: "project", Degree: degree})
+		require.NoError(t, err)
+		if i > 0 {
+			assert.Less(t, previous.DegreeOrder, entry.DegreeOrder)
+		}
+		previous = entry
+	}
+	negativeZero, err := NewProjectValidationIndex(ValidationFact{URI: "commit", Project: "project", Degree: math.Copysign(0, -1)})
+	require.NoError(t, err)
+	assert.Equal(t, previous, negativeZero)
+}
+
+func TestProjectValidationIndexRejectsInvalidFacts(t *testing.T) {
+	for _, fact := range []ValidationFact{
+		{Project: "p"}, {URI: "commit"},
+		{URI: "commit", Project: "p", Degree: math.NaN()},
+		{URI: "commit", Project: "p", Degree: math.Inf(1)},
+		{URI: "commit", Project: "p", Degree: -0.1},
+		{URI: "commit", Project: "p", Degree: 1.1},
+	} {
+		_, err := NewProjectValidationIndex(fact)
+		require.Error(t, err)
+	}
+}

stovepipe/extension/storage/BUILD.bazel

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "build_store.go",
+        "project_validation_index_store.go",
         "queue_log_store.go",
         "queue_store.go",
         "queue_summary_store.go",

stovepipe/extension/storage/README.md

@@ -23,3 +23,5 @@ Same design space as [`submitqueue/extension/storage`](../../../submitqueue/exte
 `RequestAcceptanceStore` is the ordered lookup mapping for List: immutable `(queue, accepted_at_ms, request_id)` keys with bounded primary-key range scans, not a secondary index or a second status projection. The request-log materializer ensures each known acceptance has a mapping after summary persistence, including retries where the summary is unchanged. Unknown acceptance times are not mapped; historical backfill is separate.
 
 `QueueStore` owns internal enablement alongside pipeline coordination. `QueueSummaryStore` persists the current read model projected from immutable `QueueLogStore` occurrences. Enablement sequences, operational row versions and projection versions are distinct. These stores use primary-key operations; version arithmetic, duplicate reconciliation and projection repair belong to business logic. See [queue state](../../../doc/rfc/stovepipe/queue-state.md).
+
+`ProjectValidationIndexStore` is the ordered lookup mapping for named project facts. Its immutable `(queue, URI, degree_order, project)` primary key encodes descending degree and ascending bytewise project ID, so listing is a bounded key-range read. It holds no duplicate validation payload; readers hydrate facts by their original keys and verify the mapping. The record controller ensures mappings from persisted winning facts, including retries after partial writes. The table starts empty: only results processed by index-aware writers receive mappings, with no historical population or read-time fallback.

stovepipe/extension/storage/mock/BUILD.bazel

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "build_store_mock.go",
+        "project_validation_index_store_mock.go",
         "queue_log_store_mock.go",
         "queue_store_mock.go",
         "queue_summary_store_mock.go",