feat(speculation): add enumerator and selector extensions

Makefile

@@ -336,7 +336,7 @@ local-stovepipe-gateway-start: build-stovepipe-gateway-linux ## Start Stovepipe
 
 mocks: ## Generate mock files using mockgen
 	@echo "Generating mocks..."
-	@$(BAZEL) run @rules_go//go -- generate ./submitqueue/extension/storage/... ./submitqueue/extension/buildrunner/... ./submitqueue/extension/changeprovider/... ./extension/counter/... ./extension/messagequeue/... ./submitqueue/extension/queueconfig/... ./submitqueue/extension/mergechecker/... ./submitqueue/extension/pusher/... ./submitqueue/extension/scorer/... ./submitqueue/extension/conflict/... ./submitqueue/core/consumer/... ./submitqueue/core/changeset/...
+	@$(BAZEL) run @rules_go//go -- generate ./submitqueue/extension/storage/... ./submitqueue/extension/buildrunner/... ./submitqueue/extension/changeprovider/... ./extension/counter/... ./extension/messagequeue/... ./submitqueue/extension/queueconfig/... ./submitqueue/extension/mergechecker/... ./submitqueue/extension/pusher/... ./submitqueue/extension/scorer/... ./submitqueue/extension/conflict/... ./submitqueue/extension/speculation/enumerator/... ./submitqueue/extension/speculation/selector/... ./submitqueue/core/consumer/... ./submitqueue/core/changeset/...
 	@echo "Mocks generated successfully!"
 
 proto: ## Generate protobuf files from .proto definitions

doc/rfc/index.md

@@ -10,8 +10,9 @@ Design documents and technical proposals, grouped by scope. Shared/cross-cutting
 
 - [Orchestrator Workflow](submitqueue/workflow.md) - Queue-driven controller pipeline from gateway entry through batching, scoring, build, merge, and conclude
 - [Build Runner](submitqueue/build-runner.md) - Vendor-agnostic BuildRunner interface, provider-neutral BuildStatus lifecycle, and how the orchestrator wires it into the build stage
+- [Build Runner](submitqueue/build-runner.md) - Vendor-agnostic BuildRunner interface, provider-neutral BuildStatus lifecycle, and how the orchestrator wires it into the build stage
 - [Extension Contract](submitqueue/extension-contract.md) - When extensions take orchestrator identity (request/batch) and resolve granular content themselves vs. take controller-resolved data; revises the BuildRunner base/head contract
-
+- [Speculation](submitqueue/speculation.md) - Why SubmitQueue speculates, the path/tree model, and the two pluggable seams: speculation-tree enumeration and path selection
 
 ## Stovepipe
 

submitqueue/entity/build.go

@@ -53,12 +53,6 @@ func (s BuildStatus) IsTerminal() bool {
 	return s == BuildStatusSucceeded || s == BuildStatusFailed || s == BuildStatusCancelled
 }
 
-// SpeculationPathInfo represents the base and head commits of a speculation path used in a build.
-type SpeculationPathInfo struct {
-	// Base is a list of batchIDs(in order) that form the base of this speculation path.
-	Base []string
-}
-
 // Build represents a build scheduled for a batch along a specific speculation path.
 // All fields except the Status are immutable after creation.
 type Build struct {
@@ -69,8 +63,9 @@ type Build struct {
 	BatchID string
 	// SpeculationPath is the speculation path that represents this build. For
 	// a given batch this path is crafted from the graph that is generated from the
-	// dependencies of this batch.
-	SpeculationPath SpeculationPathInfo
+	// dependencies of this batch. Its Head is the batch being verified (equal to
+	// BatchID) and its Base is the assumed-good prefix of predecessor batches.
+	SpeculationPath SpeculationPath
 	// Score represents the build prediction score for this speculation path.
 	Score float32
 	// Status represents the state of the build lifecycle this build is in.

submitqueue/entity/build_test.go

@@ -70,8 +70,9 @@ func TestBuild_ToBytes(t *testing.T) {
 	build := Build{
 		ID:      "build-1",
 		BatchID: "batch-1",
-		SpeculationPath: SpeculationPathInfo{
+		SpeculationPath: SpeculationPath{
 			Base: []string{"batch-0", "batch-prev"},
+			Head: "batch-1",
 		},
 		Score:  0.85,
 		Status: BuildStatusAccepted,
@@ -92,8 +93,9 @@ func TestBuildFromBytes(t *testing.T) {
 	original := Build{
 		ID:      "build-42",
 		BatchID: "batch-7",
-		SpeculationPath: SpeculationPathInfo{
+		SpeculationPath: SpeculationPath{
 			Base: []string{"batch-5", "batch-6"},
+			Head: "batch-7",
 		},
 		Score:  0.92,
 		Status: BuildStatusAccepted,
@@ -145,8 +147,9 @@ func TestBuild_SerializationRoundTrip(t *testing.T) {
 			build: Build{
 				ID:      "build-100",
 				BatchID: "batch-50",
-				SpeculationPath: SpeculationPathInfo{
+				SpeculationPath: SpeculationPath{
 					Base: []string{"batch-48", "batch-49"},
+					Head: "batch-50",
 				},
 				Score:  0.75,
 				Status: BuildStatusAccepted,
@@ -166,8 +169,9 @@ func TestBuild_SerializationRoundTrip(t *testing.T) {
 			build: Build{
 				ID:      "build-300",
 				BatchID: "batch-70",
-				SpeculationPath: SpeculationPathInfo{
+				SpeculationPath: SpeculationPath{
 					Base: []string{"batch-65"},
+					Head: "batch-70",
 				},
 				Score:  0,
 				Status: BuildStatusFailed,

submitqueue/entity/speculation_tree.go

@@ -14,38 +14,112 @@
 
 package entity
 
-// SpeculationPathAction defines the possible actions for a speculation path.
+// SpeculationPath is a single speculation path: an assumed-good prefix of
+// predecessor batches (Base) on top of which the batch under verification
+// (Head) is built and validated.
+//
+// This is the unit the build stage consumes: Base maps to the build runner's
+// base changes (an assumed-good prefix to apply) and Head maps to the changes
+// being validated.
+type SpeculationPath struct {
+	// Base is the ordered list of predecessor batch IDs assumed to have passed.
+	// Empty means the path builds the head batch directly on the target branch.
+	Base []string
+	// Head is the batch ID being verified by this path.
+	Head string
+}
+
+// SpeculationPathStatus is the observed lifecycle state of a speculation path.
+// It is written only by the orchestrator's speculate controller (into the
+// speculation tree store) and read by the path selector as input; enumerators
+// and selectors never write it.
+type SpeculationPathStatus string
+
+const (
+	// SpeculationPathStatusUnknown is the unreachable zero value, set by default
+	// on init. A persisted path always carries a real status (candidate onward),
+	// so this should never be seen in the store.
+	SpeculationPathStatusUnknown SpeculationPathStatus = ""
+	// SpeculationPathStatusCandidate is a freshly enumerated path the controller
+	// has persisted but not yet sent to build.
+	SpeculationPathStatusCandidate SpeculationPathStatus = "candidate"
+	// SpeculationPathStatusSelected is a path the controller has sent to the build
+	// controller (in response to a selector Build action) but for which no build
+	// signal has arrived yet — the build system may not have started it
+	// (resource-gated), so whether it is actually building is not yet known.
+	SpeculationPathStatusSelected SpeculationPathStatus = "selected"
+	// SpeculationPathStatusBuilding is a path a build signal has confirmed is in
+	// flight; its BuildID is known.
+	SpeculationPathStatusBuilding SpeculationPathStatus = "building"
+	// SpeculationPathStatusPassed is a path whose build succeeded.
+	SpeculationPathStatusPassed SpeculationPathStatus = "passed"
+	// SpeculationPathStatusFailed is a path whose build failed.
+	SpeculationPathStatusFailed SpeculationPathStatus = "failed"
+	// SpeculationPathStatusCancelled is a path that is no longer pursued — its
+	// base was invalidated, its build was cancelled, or the selector dropped it.
+	SpeculationPathStatusCancelled SpeculationPathStatus = "cancelled"
+)
+
+// SpeculationPathAction is the action a path selector asks the controller to
+// take for a path. It is the selector's only output: ephemeral (recomputed
+// every time the selector runs) and never persisted. The controller enacts it
+// and records the resulting SpeculationPathStatus.
 type SpeculationPathAction string
 
 const (
-	// SpeculationPathActionUnknown is the default zero value for SpeculationPathAction.
+	// SpeculationPathActionUnknown is the unreachable zero value. A real decision
+	// always carries Build or Cancel; the selector expresses "leave this path
+	// as-is" by omitting it from its decisions, not by returning this.
 	SpeculationPathActionUnknown SpeculationPathAction = ""
-	// TODO: Add comprehensive list of actions
+	// SpeculationPathActionBuild asks the controller to send this path to the
+	// build controller (which triggers a build subject to resources). The path moves
+	// to Selected on send, then Building once a build signal confirms it.
+	SpeculationPathActionBuild SpeculationPathAction = "build"
+	// SpeculationPathActionCancel asks the controller to drop this path and
+	// cancel any build in flight for it.
+	SpeculationPathActionCancel SpeculationPathAction = "cancel"
 )
 
-// SpeculationInfo represents metadata about a single speculation path, including the path through the dependency graph, its current state, and the predicted build score.
-type SpeculationInfo struct {
-	// Path represents the speculation path; which is an ordered list of batches.
-	Path []string
-	// Action is a state that this path is in.
-	Action SpeculationPathAction
-	// Score is score for this speculation path.
+// SpeculationPathInfo is the per-path entry in a speculation tree: a path, the
+// enumerator's predicted score for it, its controller-owned status, and a link
+// to the build dispatched for it (if any).
+type SpeculationPathInfo struct {
+	// Path is the Base/Head split this entry covers.
+	Path SpeculationPath
+	// Score is the enumerator's predicted success score for this path.
 	Score float32
+	// Status is the observed lifecycle state of the path. Written only by the
+	// controller; read by the selector.
+	Status SpeculationPathStatus
+	// BuildID links this path to its build. Empty until a build signal confirms
+	// the build and the controller records it (Selected -> Building); the
+	// controller never knows the ID at send time.
+	BuildID string
+}
+
+// SpeculationPathDecision is a path selector's decision for a single path: the
+// action the controller should take for it. It is the selector's output and is
+// not persisted.
+type SpeculationPathDecision struct {
+	// Path identifies the speculation path the action applies to.
+	Path SpeculationPath
+	// Action is what the controller should do for the path.
+	Action SpeculationPathAction
 }
 
-// SpeculationTree represents the set of speculation paths constructed for a batch based on its dependency graph.
+// SpeculationTree is the set of candidate speculation paths for a batch, built
+// from its dependency graph.
 type SpeculationTree struct {
 	// BatchID is the batch for which this speculation tree is constructed.
 	BatchID string
-	// Speculations is a list of speculation paths for this batch based on a graph of its
-	// dependents.
+	// Paths is the candidate speculation paths for this batch, derived from a
+	// graph of its dependencies.
 	//
 	// For e.g - Consider batches - queueA/batch/1, queueA/batch/2, queueA/batch/3
 	// such that - queueA/batch/2 and queueA/batch/3 depend on queueA/batch/1
 	//
-	// Speculations for queueA/batch/1 - [{Path: []string{"queueA/batch/1"}, State: "scheduled", Score: 0.1}]
-	// Speculations for queueA/batch/2 - [{Path: []string{"queueA/batch/2"}, State: "scheduled", Score: 0.9}, {Path: []string{"queueA/batch/1", "queueA/batch/2"}, State: "scheduled", Score: 0.3}]
-	// Speculations for queueA/batch/3 - [{Path: []string{"queueA/batch/3"}, State: "scheduled", Score: 0.9}, {Path: []string{"queueA/batch/1", "queueA/batch/3"}, State: "scheduled", Score: 0.3}]
+	// Paths for queueA/batch/2 - [{Path: {Base: [], Head: "queueA/batch/2"}, Score: 0.9, Status: "candidate"}, {Path: {Base: ["queueA/batch/1"], Head: "queueA/batch/2"}, Score: 0.3, Status: "candidate"}]
+	// Paths for queueA/batch/3 - [{Path: {Base: [], Head: "queueA/batch/3"}, Score: 0.9, Status: "candidate"}, {Path: {Base: ["queueA/batch/1"], Head: "queueA/batch/3"}, Score: 0.3, Status: "candidate"}]
 	//
-	Speculations []SpeculationInfo
+	Paths []SpeculationPathInfo
 }

submitqueue/extension/speculation/enumerator/BUILD.bazel

@@ -0,0 +1,9 @@
+load("@rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "enumerator",
+    srcs = ["enumerator.go"],
+    importpath = "github.com/uber/submitqueue/submitqueue/extension/speculation/enumerator",
+    visibility = ["//visibility:public"],
+    deps = ["//submitqueue/entity"],
+)

submitqueue/extension/speculation/enumerator/README.md

@@ -0,0 +1,19 @@
+# Speculation Tree Enumerator
+
+Vendor-agnostic interface for enumerating the **speculation tree** of a batch — the set of candidate speculation paths the orchestrator may build, each scored with its predicted probability of success.
+
+See the [Speculation RFC](../../../../doc/rfc/submitqueue/speculation.md) for the end-to-end design and how enumeration fits into the orchestrator pipeline.
+
+## Enumerator
+
+An enumerator is deliberately **dumb**: *given a batch and its dependency batches, it mechanically lists the candidate paths and scores them.* It does **not** decide which paths to build — that is the [selector](../selector)'s job — it does **not** set path status, and it does **not** decide how far back to speculate. Speculation depth is the controller's responsibility: the controller trims the dependency list before calling the enumerator, which then enumerates over exactly the list it is handed.
+
+Each candidate is a path: an assumed-good prefix of predecessor batches (the base) on top of which the batch under verification (the head) is built. The base maps directly onto the build stage's base changes and the head onto the changes being validated.
+
+Enumeration is **pure and deterministic**: the same batch and dependency list always produce the same tree. This lets the controller regenerate a tree whenever the dependency graph changes without tracking incremental state in the enumerator. Keeping enumeration tractable for a very wide dependency list is the enumerator's only real concern.
+
+Scores ride in on the inputs. Each dependency is passed as a full `entity.Batch`, which already carries its per-batch success probability (`Batch.Score`) from the score stage; the enumerator combines the scores of a path's base batches into the path's score. No separate scoring backend or injected probability source is needed, and tests just set `.Score` on literal batches. The head is passed as an ID — its score is constant across all of its own paths.
+
+## Factory
+
+`Factory.For(Config) (Enumerator, error)` returns the enumerator for a queue, following the repo's extension contract (`conflict.Analyzer` is the reference shape). `Config` carries only the queue identity (`QueueName`); the system hands the factory nothing else. Everything an implementation needs — including behavioral knobs like speculation depth — is injected at construction by the integrator in the wiring layer, which resolves per-queue settings through `queueconfig`. `Enumerate` itself stays config-free.

submitqueue/extension/speculation/enumerator/enumerator.go

@@ -0,0 +1,65 @@
+// Copyright (c) 2025 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package enumerator
+
+//go:generate mockgen -source=enumerator.go -destination=mock/enumerator_mock.go -package=mock
+
+import (
+	"context"
+
+	"github.com/uber/submitqueue/submitqueue/entity"
+)
+
+// Enumerator builds the speculation tree for a batch: the set of candidate
+// speculation paths to consider, each scored with its predicted success
+// probability.
+//
+// Enumeration answers "what futures are possible" for a batch. It is
+// deliberately dumb: it mechanically lists candidate paths from the dependency
+// batches it is handed and attaches a Score to each. It does not decide which
+// paths to build — that is the selector's job (see
+// extension/speculation/selector) — and it does not decide how far back to
+// speculate: the controller trims the dependency list by speculation depth
+// before calling Enumerate.
+type Enumerator interface {
+	// Enumerate returns the speculation tree for the batch identified by batchID,
+	// given its dependency batches in arrival order. Each returned path carries a
+	// Base/Head split and a predicted success Score; the returned paths leave
+	// Status unset (the controller stamps it on persist).
+	//
+	// Path scores are derived from the dependency batches' Score field (the
+	// per-batch success probability set by the score stage), so no separate
+	// scoring backend is needed. The combination formula is the implementation's
+	// concern.
+	//
+	// Enumeration is pure and deterministic: the same (batchID, deps) always
+	// yields the same tree, so callers may regenerate safely.
+	Enumerate(ctx context.Context, batchID string, deps []entity.Batch) (entity.SpeculationTree, error)
+}
+
+// Config carries the per-queue identity handed to a Factory. The system knows
+// only the queue name; everything an implementation needs (including behavioral
+// knobs such as speculation depth) is injected at construction by the integrator.
+type Config struct {
+	// QueueName identifies the queue this Enumerator serves.
+	QueueName string
+}
+
+// Factory builds the Enumerator for a queue. Implementations are provided by
+// integrators (and tests) and inject whatever they need at construction.
+type Factory interface {
+	// For returns the Enumerator for the given queue.
+	For(cfg Config) (Enumerator, error)
+}

submitqueue/extension/speculation/enumerator/mock/BUILD.bazel

@@ -0,0 +1,13 @@
+load("@rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "mock",
+    srcs = ["enumerator_mock.go"],
+    importpath = "github.com/uber/submitqueue/submitqueue/extension/speculation/enumerator/mock",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//submitqueue/entity",
+        "//submitqueue/extension/speculation/enumerator",
+        "@org_uber_go_mock//gomock",
+    ],
+)