feat(stovepipe): Show most broken projects first

doc/rfc/stovepipe/project-status-api.md

@@ -17,7 +17,7 @@ The request and response fields have the following API-level meaning. The publis
 | `queue` | Yes | Identifies the queue in which to find the validation request. |
 | `request_id` | ID lookup | Canonical positive decimal request ID returned by `Ingest`. |
 | `change_uri` | URI lookup | Identifies the exact commit URI under validation. |
-| `projects` | No | Consumer-defined project IDs whose results are requested. Omitted or empty returns all recorded project results. |
+| `projects` | No | Consumer-defined project IDs whose results are requested. When omitted or empty, returns recorded project results, most broken first. |
 | `page_size` | No | Limits one page of results for the requested projects. |
 | `page_token` | No | Continues a previous page of results for the requested projects. |
 
@@ -29,7 +29,7 @@ The request and response fields have the following API-level meaning. The publis
 | `updated_at_ms` | Records, in Unix milliseconds, the newest durable lifecycle or result record represented by the response. |
 | `repository_breakage_degree` | Returns the whole-repository result when it is durably recorded. |
 | `project_results_complete` | Indicates whether the implementation has durably finished producing results for the supplied project IDs. |
-| `projects` | Returns recorded results for requested project IDs, or all recorded project results when none are supplied. |
+| `projects` | Returns recorded results for requested project IDs in the supplied order, or recorded project results ordered by descending breakage degree, then ascending project ID when none are supplied. |
 | `next_page_token` | Continues requested-project result pagination when another page exists. |
 
 `request_state` is a stable public lifecycle vocabulary: `accepted`, `processing`, `succeeded`, `failed`, `cancelled`, or `superseded`. Clients must tolerate a future value. A terminal request state does not by itself mean that project results are complete.
@@ -64,15 +64,17 @@ The caller may supply the project IDs for which it wants recorded results. The A
 
 When project IDs are supplied, the response includes only results for those IDs, in the supplied order, and paginates that list. An implementation may record results for every requested project, only requested projects with a non-green result, or another documented subset.
 
-With no project IDs supplied, the response lists all recorded project results in project-ID order, excluding the repository fact. Results are a live view; restart pagination to discover facts recorded behind the cursor.
+With no project IDs supplied, the response lists recorded project results by descending breakage degree, then ascending project ID, excluding the repository fact. If the result for the cursor project is missing, the API returns an error. Results are a live view; restart pagination to discover facts recorded before the cursor.
 
 Each project result is an immutable validation fact keyed by queue, commit URI, and project ID. A result must identify the selected request. `project_results_complete` is false until the implementation has durably finished producing results for the supplied project IDs; a completed empty set is valid.
 
 Callers use `project_results_complete`, rather than the presence or absence of an individual project result, to determine whether the implementation has finished producing results for the requested projects. `next_page_token` is empty only on the final page. A terminal lifecycle notification is emitted only after the repository result and any applicable project completion record are durable. If notification delivery fails, consumers can recover the same result through either endpoint.
 
 ## Implementation
 
-Both endpoints return durable lifecycle summaries, the whole-repository fact when present, and recorded facts for requested projects in paginated request order, or all recorded project results in project-ID order when none are requested. `project_results_complete` remains false until a durable completion projection is available. Missing facts are omitted; an explicitly recorded zero degree remains present. `updated_at_ms` is the maximum timestamp of the summary and the facts represented on the page.
+Both endpoints return durable lifecycle summaries, the whole-repository fact when present, and recorded facts for requested projects in paginated request order, or recorded project results by descending breakage degree then ascending project ID when none are requested. `project_results_complete` remains false until a durable completion projection is available. Missing facts are omitted; an explicitly recorded zero degree remains present. `updated_at_ms` is the maximum timestamp of the summary and the facts represented on the page.
+
+A dedicated ordering mapping makes listing a bounded primary-key range read, not a sort over all facts. Listing reads at most `page_size + 1` ordering mappings and hydrates only the returned facts, plus the cursor fact on continuations. The continuation token identifies the last project; its immutable fact supplies the degree needed to resume this ordering.
 
 ## Errors and Authorization
 

stovepipe/controller/project_status.go

@@ -161,7 +161,11 @@ func (c *projectStatusController) GetProjectStatus(ctx context.Context, req enti
 		}
 	}
 
-	page, err := loadProjectStatusPage(ctx, req, requestSummary, factStore)
+	var indexStore storage.ProjectFactByDegreeStore
+	if len(req.Projects) == 0 {
+		indexStore = store.GetProjectFactByDegreeStore()
+	}
+	page, err := loadProjectStatusPage(ctx, req, requestSummary, factStore, indexStore)
 	if err != nil {
 		return entity.GetProjectStatusResult{}, err
 	}
@@ -221,6 +225,7 @@ func loadProjectStatusPage(
 	req entity.GetProjectStatusRequest,
 	summary entity.RequestSummary,
 	store storage.ValidationFactStore,
+	indexStore storage.ProjectFactByDegreeStore,
 ) (projectStatusPage, error) {
 	token, err := parseProjectStatusPageToken(req, summary)
 	if err != nil {
@@ -233,7 +238,7 @@ func loadProjectStatusPage(
 	}
 
 	if len(req.Projects) == 0 {
-		return listProjectStatusPage(ctx, summary, store, token, pageSize)
+		return listProjectStatusPage(ctx, summary, store, indexStore, token, pageSize)
 	}
 
 	return getRequestedProjectStatusPage(ctx, summary, store, token, pageSize)
@@ -282,32 +287,61 @@ func listProjectStatusPage(
 	ctx context.Context,
 	summary entity.RequestSummary,
 	store storage.ValidationFactStore,
+	indexStore storage.ProjectFactByDegreeStore,
 	token projectStatusPageToken,
 	pageSize int,
 ) (projectStatusPage, error) {
-	facts, err := store.ListProjects(ctx, summary.URI, token.AfterProject, pageSize+1)
-	if err != nil {
-		return projectStatusPage{}, fmt.Errorf("GetProjectStatus failed to list project facts for request %q: %w", summary.RequestID, err)
-	}
+	var after entity.ProjectFactByDegree
+	if token.AfterProject != "" {
+		fact, err := store.Get(ctx, summary.URI, token.AfterProject)
+		if err != nil {
+			return projectStatusPage{}, fmt.Errorf("GetProjectStatus failed to load cursor project %q: %w", token.AfterProject, err)
+		}
 
-	for _, fact := range facts {
-		if err := validateProjectFact(fact, summary, fact.Project); err != nil {
+		if err := validateProjectFact(fact, summary, token.AfterProject); err != nil {
 			return projectStatusPage{}, err
 		}
 
-		if fact.Project == "" {
-			return projectStatusPage{}, &ProjectStatusConsistencyError{message: "project listing contains a repository fact"}
+		after, err = entity.NewProjectFactByDegree(fact)
+		if err != nil {
+			return projectStatusPage{}, err
 		}
 	}
 
-	page := projectStatusPage{facts: facts}
-	if len(facts) > pageSize {
-		page.facts = facts[:pageSize]
-		token.AfterProject = page.facts[pageSize-1].Project
+	entries, err := indexStore.List(ctx, summary.URI, after, pageSize+1)
+	if err != nil {
+		return projectStatusPage{}, fmt.Errorf("GetProjectStatus failed to list validation facts by degree for request %q: %w", summary.RequestID, err)
+	}
+
+	var page projectStatusPage
+	if len(entries) > pageSize {
+		entries = entries[:pageSize]
+		token.AfterProject = entries[pageSize-1].Project
 		page.nextToken, err = encodeProjectStatusPageToken(token)
+		if err != nil {
+			return projectStatusPage{}, err
+		}
 	}
 
-	return page, err
+	for _, entry := range entries {
+		fact, err := store.Get(ctx, summary.URI, entry.Project)
+		if err != nil {
+			return projectStatusPage{}, fmt.Errorf("GetProjectStatus failed to load indexed project %q: %w", entry.Project, err)
+		}
+
+		if err := validateProjectFact(fact, summary, entry.Project); err != nil {
+			return projectStatusPage{}, err
+		}
+
+		expected, err := entity.NewProjectFactByDegree(fact)
+		if err != nil || entry != expected {
+			return projectStatusPage{}, &ProjectStatusConsistencyError{message: "degree-ordered fact lookup disagrees with stored fact"}
+		}
+
+		page.facts = append(page.facts, fact)
+	}
+
+	return page, nil
 }
 
 func encodeProjectStatusPageToken(token projectStatusPageToken) (string, error) {

stovepipe/controller/project_status_test.go

@@ -111,6 +111,7 @@ func TestGetProjectStatus(t *testing.T) {
 			uriStore := storagemock.NewMockRequestURIStore(mockCtrl)
 			summaryStore := storagemock.NewMockRequestSummaryStore(mockCtrl)
 			factStore := storagemock.NewMockValidationFactStore(mockCtrl)
+			indexStore := storagemock.NewMockProjectFactByDegreeStore(mockCtrl)
 			if !(tt.wantInvalid && tt.request.PageToken == "") {
 				factory.EXPECT().For(storage.Config{QueueName: projectStatusQueue}).Return(store, nil)
 				store.EXPECT().GetRequestURIStore().Return(uriStore)
@@ -123,7 +124,14 @@ func TestGetProjectStatus(t *testing.T) {
 						factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "").Return(tt.fact, tt.factErr)
 						if !tt.noProjectLookup && !(tt.factErr == nil && tt.wantConsistency) {
 							if len(tt.request.Projects) == 0 {
-								factStore.EXPECT().ListProjects(gomock.Any(), projectStatusURI, "", defaultProjectStatusPageSize+1).Return(tt.projectFacts, tt.wantErr)
+								store.EXPECT().GetProjectFactByDegreeStore().Return(indexStore)
+								indexStore.EXPECT().List(gomock.Any(), projectStatusURI, entity.ProjectFactByDegree{}, defaultProjectStatusPageSize+1).
+									Return(projectStatusIndexEntries(t, tt.projectFacts), tt.wantErr)
+								if tt.wantErr == nil {
+									for _, fact := range tt.projectFacts {
+										factStore.EXPECT().Get(gomock.Any(), projectStatusURI, fact.Project).Return(fact, nil)
+									}
+								}
 							}
 							for _, project := range tt.request.Projects {
 								factStore.EXPECT().Get(gomock.Any(), projectStatusURI, project).Return(entity.ValidationFact{}, storage.ErrNotFound)
@@ -300,9 +308,9 @@ func TestGetProjectStatusReturnsProjectFactsInRequestedOrder(t *testing.T) {
 	summaryStore.EXPECT().Get(gomock.Any(), projectStatusID).Return(requestSummary, nil)
 	store.EXPECT().GetValidationFactStore().Return(factStore)
 	factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "").Return(entity.ValidationFact{}, storage.ErrNotFound)
-	factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "project-b").Return(entity.ValidationFact{URI: projectStatusURI, RequestID: projectStatusID, Project: "project-b", Degree: entity.DegreeBroken, CreatedAt: 20}, nil)
+	factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "project-b").Return(entity.ValidationFact{URI: projectStatusURI, RequestID: projectStatusID, Project: "project-b", Degree: entity.DegreeGreen, CreatedAt: 20}, nil)
 	factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "missing").Return(entity.ValidationFact{}, storage.ErrNotFound)
-	factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "project-a").Return(entity.ValidationFact{URI: projectStatusURI, RequestID: projectStatusID, Project: "project-a", Degree: entity.DegreeGreen, CreatedAt: 30}, nil)
+	factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "project-a").Return(entity.ValidationFact{URI: projectStatusURI, RequestID: projectStatusID, Project: "project-a", Degree: entity.DegreeBroken, CreatedAt: 30}, nil)
 
 	controller := NewProjectStatusController(zap.NewNop().Sugar(), tally.NoopScope, factory)
 	got, err := controller.GetProjectStatus(context.Background(), entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI, Projects: []string{"project-b", "missing", "project-a"}})
@@ -452,32 +460,38 @@ func TestParseProjectStatusPageTokenRejectsMismatches(t *testing.T) {
 func TestGetProjectStatusPaginatesStoredProjects(t *testing.T) {
 	summary := entity.RequestSummary{RequestID: projectStatusID, Queue: projectStatusQueue, URI: projectStatusURI, StateTimestampMs: 10}
 	stored := []entity.ValidationFact{
+		{URI: projectStatusURI, RequestID: projectStatusID, Project: "c", Degree: 1, CreatedAt: 60},
 		{URI: projectStatusURI, RequestID: projectStatusID, Project: "a", Degree: 0.5, CreatedAt: 20},
 		{URI: projectStatusURI, RequestID: projectStatusID, Project: "b", Degree: 0, CreatedAt: 50},
-		{URI: projectStatusURI, RequestID: projectStatusID, Project: "c", Degree: 1, CreatedAt: 60},
 	}
 
 	ctrl := gomock.NewController(t)
 	factory := storagemock.NewMockFactory(ctrl)
 	store := storagemock.NewMockStorage(ctrl)
 	summaries := storagemock.NewMockRequestSummaryStore(ctrl)
 	facts := storagemock.NewMockValidationFactStore(ctrl)
+	indexStore := storagemock.NewMockProjectFactByDegreeStore(ctrl)
+	entries := projectStatusIndexEntries(t, stored)
 
 	factory.EXPECT().For(storage.Config{QueueName: projectStatusQueue}).Return(store, nil).Times(2)
 	store.EXPECT().GetRequestSummaryStore().Return(summaries).Times(2)
 	summaries.EXPECT().Get(gomock.Any(), projectStatusID).Return(summary, nil).Times(2)
 	store.EXPECT().GetValidationFactStore().Return(facts).Times(2)
+	store.EXPECT().GetProjectFactByDegreeStore().Return(indexStore).Times(2)
 	facts.EXPECT().Get(gomock.Any(), projectStatusURI, "").Return(entity.ValidationFact{}, storage.ErrNotFound).Times(2)
-	facts.EXPECT().ListProjects(gomock.Any(), projectStatusURI, "", 3).Return(stored, nil)
-	facts.EXPECT().ListProjects(gomock.Any(), projectStatusURI, "b", defaultProjectStatusPageSize+1).Return(stored[2:], nil)
+	indexStore.EXPECT().List(gomock.Any(), projectStatusURI, entity.ProjectFactByDegree{}, 3).Return(entries, nil)
+	indexStore.EXPECT().List(gomock.Any(), projectStatusURI, entries[1], defaultProjectStatusPageSize+1).Return(entries[2:], nil)
+	facts.EXPECT().Get(gomock.Any(), projectStatusURI, "c").Return(stored[0], nil)
+	facts.EXPECT().Get(gomock.Any(), projectStatusURI, "a").Return(stored[1], nil).Times(2)
+	facts.EXPECT().Get(gomock.Any(), projectStatusURI, "b").Return(stored[2], nil)
 
 	controller := NewProjectStatusController(zap.NewNop().Sugar(), tally.NoopScope, factory)
 	req := entity.GetProjectStatusRequest{Queue: projectStatusQueue, RequestID: projectStatusID, PageSize: 2}
 
 	first, err := controller.GetProjectStatus(context.Background(), req)
 	require.NoError(t, err)
 	assert.Equal(t, stored[:2], first.ProjectValidationFacts)
-	assert.Equal(t, int64(50), first.UpdatedAtMs)
+	assert.Equal(t, int64(60), first.UpdatedAtMs)
 	require.NotEmpty(t, first.NextPageToken)
 
 	req.PageToken = first.NextPageToken
@@ -487,10 +501,69 @@ func TestGetProjectStatusPaginatesStoredProjects(t *testing.T) {
 	second, err := controller.GetProjectStatus(context.Background(), req)
 	require.NoError(t, err)
 	assert.Equal(t, stored[2:], second.ProjectValidationFacts)
-	assert.Equal(t, int64(60), second.UpdatedAtMs)
+	assert.Equal(t, int64(50), second.UpdatedAtMs)
 	assert.Empty(t, second.NextPageToken)
 }
 
+func projectStatusIndexEntries(t *testing.T, facts []entity.ValidationFact) []entity.ProjectFactByDegree {
+	t.Helper()
+	var entries []entity.ProjectFactByDegree
+	for _, fact := range facts {
+		entry, err := entity.NewProjectFactByDegree(fact)
+		require.NoError(t, err)
+		entries = append(entries, entry)
+	}
+	return entries
+}
+
+func TestListProjectStatusPageRejectsMissingCursor(t *testing.T) {
+	for _, readErr := range []error{storage.ErrNotFound, errors.New("database unavailable")} {
+		t.Run(readErr.Error(), func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			facts := storagemock.NewMockValidationFactStore(ctrl)
+			index := storagemock.NewMockProjectFactByDegreeStore(ctrl)
+			facts.EXPECT().Get(gomock.Any(), projectStatusURI, "cursor").Return(entity.ValidationFact{}, readErr)
+			_, err := listProjectStatusPage(context.Background(), entity.RequestSummary{URI: projectStatusURI}, facts, index, projectStatusPageToken{AfterProject: "cursor"}, 2)
+			require.ErrorIs(t, err, readErr)
+		})
+	}
+}
+
+func TestListProjectStatusPageRejectsInconsistentIndex(t *testing.T) {
+	fact := entity.ValidationFact{URI: projectStatusURI, Project: "p", Degree: 0.5, RequestID: projectStatusID}
+	entry, err := entity.NewProjectFactByDegree(fact)
+	require.NoError(t, err)
+	for _, tt := range []struct {
+		name    string
+		modify  func(*entity.ProjectFactByDegree, *entity.ValidationFact)
+		factErr error
+	}{
+		{name: "missing fact", factErr: storage.ErrNotFound, modify: func(*entity.ProjectFactByDegree, *entity.ValidationFact) {}},
+		{name: "read failure", factErr: errors.New("database unavailable"), modify: func(*entity.ProjectFactByDegree, *entity.ValidationFact) {}},
+		{name: "wrong degree", modify: func(e *entity.ProjectFactByDegree, _ *entity.ValidationFact) { e.Degree = 0.75 }},
+		{name: "wrong URI", modify: func(e *entity.ProjectFactByDegree, _ *entity.ValidationFact) { e.URI = "other" }},
+		{name: "wrong request", modify: func(_ *entity.ProjectFactByDegree, f *entity.ValidationFact) { f.RequestID = "other" }},
+		{name: "repository fact", modify: func(e *entity.ProjectFactByDegree, f *entity.ValidationFact) { e.Project = ""; f.Project = "" }},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			ctrl := gomock.NewController(t)
+			facts := storagemock.NewMockValidationFactStore(ctrl)
+			index := storagemock.NewMockProjectFactByDegreeStore(ctrl)
+			e, f := entry, fact
+			tt.modify(&e, &f)
+			index.EXPECT().List(gomock.Any(), projectStatusURI, entity.ProjectFactByDegree{}, 3).Return([]entity.ProjectFactByDegree{e}, nil)
+			facts.EXPECT().Get(gomock.Any(), projectStatusURI, e.Project).Return(f, tt.factErr)
+			_, err := listProjectStatusPage(context.Background(), entity.RequestSummary{URI: projectStatusURI, RequestID: projectStatusID}, facts, index, projectStatusPageToken{}, 2)
+			require.Error(t, err)
+			if tt.factErr != nil {
+				assert.ErrorIs(t, err, tt.factErr)
+			} else {
+				assert.True(t, IsProjectStatusConsistency(err))
+			}
+		})
+	}
+}
+
 func TestValidateProjectFact(t *testing.T) {
 	requestSummary := entity.RequestSummary{RequestID: projectStatusID, URI: projectStatusURI}
 	assert.NoError(t, validateProjectFact(entity.ValidationFact{URI: projectStatusURI, RequestID: projectStatusID, Project: "project-a", Degree: entity.DegreeGreen}, requestSummary, "project-a"))

stovepipe/entity/project_status.go

@@ -40,7 +40,7 @@ type GetProjectStatusResult struct {
 	HasRepositoryValidationFact bool
 	// ProjectResultsComplete reports whether the implementation has finished its project-result set.
 	ProjectResultsComplete bool
-	// ProjectValidationFacts contains recorded results in request order, or project-ID order when Projects is empty.
+	// ProjectValidationFacts contains recorded results in request order, or descending degree then ascending project ID when Projects is empty.
 	ProjectValidationFacts []ValidationFact
 	// NextPageToken continues requested-project result pagination when another page exists.
 	NextPageToken string

test/integration/stovepipe/BUILD.bazel

@@ -16,11 +16,14 @@ go_test(
     ],
     deps = [
         "//api/stovepipe/protopb:go_default_library",
+        "//stovepipe/entity:go_default_library",
+        "//stovepipe/extension/storage/mysql:go_default_library",
         "//test/testutil:go_default_library",
         "@com_github_go_sql_driver_mysql//:go_default_library",
         "@com_github_stretchr_testify//assert:go_default_library",
         "@com_github_stretchr_testify//require:go_default_library",
         "@com_github_stretchr_testify//suite:go_default_library",
+        "@com_github_uber_go_tally//:go_default_library",
         "@org_golang_google_grpc//:go_default_library",
         "@org_golang_google_protobuf//proto:go_default_library",
     ],