perf(chunksfts): rank the whole workspace in one FTS5 query

doc/openapi.yaml

@@ -6606,14 +6606,16 @@ components:
             The slowest single project's dense search. May belong to a
             project whose query failed; the fan-out logs a warning of its
             own for those.
-        bm25_sum_ms:
+        bm25_ms:
           type: integer
           description: |
-            FTS5/BM25 search summed across projects, on the same terms as
-            dense_sum_ms.
-        bm25_max_ms:
-          type: integer
-          description: The slowest single project's BM25 search.
+            The workspace's BM25 search. One FTS5 statement covering every
+            project, partitioned per project by a window function — not a
+            sum over projects, which is why it has no matching `_max`
+            field. `MATCH` is evaluated over the whole server's index
+            whatever the scope, so asking once per project repeated the
+            same global work N times and the N queries contended over one
+            index on top of that.
         fuse_ms:
           type: integer
           description: Normalisation, candidacy blending and thresholding.

server/internal/chunksfts/chunksfts.go

@@ -199,6 +199,25 @@ func DeleteByProject(ctx context.Context, db *sql.DB, projectPath string) error
 //
 // Empty or all-tokens-too-short queries return a nil slice without
 // hitting the DB — there is nothing to match.
+//
+// NOTE: nothing in production calls this any more — workspace search asks
+// SearchProjects for every project at once, and a single-project workspace
+// goes through the same path. It is kept for two reasons, both worth more
+// than the ~40 lines it costs:
+//
+//  1. it is the independent oracle for SearchProjects. The two are
+//     structurally different statements that must return byte-identical
+//     rankings, because the per-project BM25 signal feeds project candidacy
+//     in workspace search — a divergence would silently re-rank the projects
+//     panel with no error and no failed_repos.
+//     TestSearchProjects_MatchesPerProjectQueries is that check, and it is
+//     only worth anything while this stays a separate implementation.
+//     Collapsing it into SearchProjects([]string{p}) would make the test
+//     compare a function to itself;
+//  2. it is the fallback if a single-project regression ever shows up in the
+//     partitioned form.
+//
+// Do not delete it as unused.
 func SearchProject(ctx context.Context, db *sql.DB, projectPath, query string, limit int) ([]Hit, error) {
 	if limit <= 0 {
 		limit = 20
@@ -214,7 +233,7 @@ func SearchProject(ctx context.Context, db *sql.DB, projectPath, query string, l
 		 FROM chunks_fts cf
 		 JOIN chunks_meta cm ON cm.rowid = cf.rowid
 		 WHERE chunks_fts MATCH ? AND cm.project_path = ?
-		 ORDER BY bm ASC
+		 ORDER BY bm ASC, cm.rowid ASC
 		 LIMIT ?`,
 		fts5Q, projectPath, limit,
 	)
@@ -224,23 +243,10 @@ func SearchProject(ctx context.Context, db *sql.DB, projectPath, query string, l
 	defer rows.Close()
 	var out []Hit
 	for rows.Next() {
-		var (
-			h        Hit
-			chunkT   sql.NullString
-			symName  sql.NullString
-			language sql.NullString
-			bm       float64
-		)
-		if err := rows.Scan(&h.FilePath, &h.StartLine, &h.EndLine,
-			&chunkT, &symName, &language, &h.Content, &bm); err != nil {
-			return nil, fmt.Errorf("scan chunks_fts row: %w", err)
+		h, err := scanHit(rows)
+		if err != nil {
+			return nil, err
 		}
-		h.ChunkType = chunkT.String
-		h.SymbolName = symName.String
-		h.Language = language.String
-		// SQLite returns more-negative bm25 for better matches. Flip so
-		// callers can blend with cosine-style "higher is better" scores.
-		h.Score = -bm
 		out = append(out, h)
 	}
 	if err := rows.Err(); err != nil {
@@ -249,6 +255,187 @@ func SearchProject(ctx context.Context, db *sql.DB, projectPath, query string, l
 	return out, nil
 }
 
+// searchProjectsBatch caps how many project_path values go into one IN
+// list. SQLite's default bound-variable ceiling is 999; 500 leaves room
+// for the query and limit parameters and matches the batch size the
+// vector store already uses for its own IN lists.
+const searchProjectsBatch = 500
+
+// SearchProjects answers the same question as SearchProject for many
+// projects at once, returning each project's top `perProject` hits keyed
+// by project_path. Projects with no match are absent from the map rather
+// than present with an empty slice — a caller distinguishing "nothing
+// matched" from "not asked about" gets that for free, and "BM25 found
+// nothing here" is a signal this package exists to produce.
+//
+// Why this is not a loop over SearchProject: FTS5 drives the query. It
+// evaluates MATCH over the WHOLE chunks_fts table — every project on the
+// server — joins each hit to chunks_meta, and only then discards the rows
+// belonging to other projects. So the per-project cost barely depends on
+// the project's size, and asking N times does the same global work N
+// times. Measured on a 43-project workspace, BM25 was 78-80% of the
+// fan-out's total work, and each query slowed from ~400 ms standalone to
+// as much as 7 s when 43 of them ran against the index at once.
+//
+// The window function does the partitioning SQLite would otherwise make
+// us do with N queries: rank within each project, keep the top rows of
+// each.
+//
+// Both forms order by (bm ASC, rowid ASC). The rowid is defensive, not a
+// fix for an observed bug: bm25 ties are the norm rather than the
+// exception in a trigram index over real code — in this package's own
+// test corpus 14 of 16 hits share a score with another hit — and today
+// SQLite happens to return tied rows in rowid order for both the LIMIT
+// and the window form, so they agree without being told to. That is
+// unspecified behaviour of the sorter. Naming the tiebreak makes the
+// agreement a property of the queries instead of a coincidence that a
+// future planner is free to break.
+func SearchProjects(ctx context.Context, db *sql.DB, projectPaths []string, query string, perProject int) (map[string][]Hit, error) {
+	if perProject <= 0 {
+		perProject = 20
+	}
+	fts5Q := buildFTS5Query(query)
+	if fts5Q == "" || len(projectPaths) == 0 {
+		return nil, nil
+	}
+
+	out := make(map[string][]Hit, len(projectPaths))
+	for start := 0; start < len(projectPaths); start += searchProjectsBatch {
+		end := start + searchProjectsBatch
+		if end > len(projectPaths) {
+			end = len(projectPaths)
+		}
+		if err := searchProjectsBatchInto(ctx, db, projectPaths[start:end], fts5Q, perProject, out); err != nil {
+			return nil, err
+		}
+	}
+	return out, nil
+}
+
+func searchProjectsBatchInto(ctx context.Context, db *sql.DB, projectPaths []string,
+	fts5Q string, perProject int, dst map[string][]Hit) error {
+
+	args := make([]any, 0, len(projectPaths)+2)
+	args = append(args, fts5Q)
+	for _, pp := range projectPaths {
+		args = append(args, pp)
+	}
+	args = append(args, perProject)
+
+	rows, err := db.QueryContext(ctx, workspaceRankQuery(placeholders(len(projectPaths))), args...)
+	if err != nil {
+		return fmt.Errorf("chunks_fts workspace search: %w", err)
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var pp string
+		h, err := scanRankedHit(rows, &pp)
+		if err != nil {
+			return err
+		}
+		dst[pp] = append(dst[pp], h)
+	}
+	if err := rows.Err(); err != nil {
+		return fmt.Errorf("iterate chunks_fts: %w", err)
+	}
+	return nil
+}
+
+// workspaceRankQuery builds the partitioned statement for a placeholder list.
+//
+// Split out so the test that pins the query PLAN builds the same string this
+// does. A copy in the test would drift, and drifting is the entire failure
+// mode being guarded against.
+//
+// Rank on the cheap columns, then fetch the payload for the rows that
+// survived. The obvious form — selecting file_path, content and the rest
+// inside the CTE — makes SQLite materialise all of it for EVERY matched
+// row before ROW_NUMBER trims to `perProject` per project, and the match
+// set is the whole server's index. On the load-test fixture a four-common-
+// word query matched 263,515 rows to return 2,300: carrying the payload
+// through cost 15.1 s against 2.8 s for this form, and a 624k-row match
+// cost 29.6 s — worse than the 46 per-project queries this replaced. The
+// rowid round-trip is the cheap half of the trade.
+//
+// Those numbers came from a bench harness that is NOT in this repository:
+// /loadtests/ is gitignored, corpus and tools alike. To recreate it, time
+// this statement against a corpus with a large match set alongside the
+// same statement with the payload columns moved into `hits` — the gap only
+// appears when the match set is orders of magnitude larger than the result.
+// TestSearchProjects_FetchesPayloadAfterTheTrim is the in-repo guard.
+func workspaceRankQuery(ph string) string {
+	return fmt.Sprintf(`
+		WITH hits AS (
+		  SELECT cm.project_path AS pp, cm.rowid AS rid, bm25(chunks_fts) AS bm
+		    FROM chunks_fts cf
+		    JOIN chunks_meta cm ON cm.rowid = cf.rowid
+		   WHERE chunks_fts MATCH ? AND cm.project_path IN (%s)
+		),
+		ranked AS (
+		  SELECT pp, rid, bm,
+		         ROW_NUMBER() OVER (PARTITION BY pp ORDER BY bm ASC, rid ASC) AS rn
+		    FROM hits
+		)
+		SELECT r.pp, cm.file_path, cm.start_line, cm.end_line,
+		       cm.chunk_type, cm.symbol_name, cm.language, cf.content, r.bm
+		  FROM ranked r
+		  JOIN chunks_meta cm ON cm.rowid = r.rid
+		  JOIN chunks_fts cf ON cf.rowid = r.rid
+		 WHERE r.rn <= ?
+		 ORDER BY r.pp, r.rn`, ph)
+}
+
+// placeholders builds "?,?,?" for an IN list. n is always >= 1: SearchProjects
+// returns early on an empty slice and the batching loop never produces an empty
+// batch. There is deliberately no n == 0 branch — an empty list used to render
+// as IN (NULL), which matches nothing and is indistinguishable from "nothing
+// matched". A syntax error from IN () is the better failure: it is loud, and it
+// happens at the call that is wrong.
+func placeholders(n int) string {
+	return strings.TrimSuffix(strings.Repeat("?,", n), ",")
+}
+
+// scanHit reads one single-project ranking row.
+func scanHit(rows *sql.Rows) (Hit, error) { return scanRankedHit(rows, nil) }
+
+// scanRankedHit reads one ranking row, optionally preceded by a project_path
+// column (pass nil for the single-project query, which does not select one).
+//
+// Both queries share this because they must produce byte-identical Hits and
+// their only structural difference is that leading column. Two hand-written
+// scans agreeing is exactly the kind of thing that stays true until it
+// quietly does not — and the equivalence test compares the two paths against
+// EACH OTHER, so a mistake made symmetrically in both would pass.
+func scanRankedHit(rows *sql.Rows, pp *string) (Hit, error) {
+	var (
+		h        Hit
+		chunkT   sql.NullString
+		symName  sql.NullString
+		language sql.NullString
+		bm       float64
+	)
+	// Built in one allocation rather than prepending to a finished slice: the
+	// workspace query returns up to len(projects) * perProject rows — ~2,150
+	// on the 43-project load-test fixture — and this runs per row, inside the
+	// path the rest of this change exists to speed up.
+	dest := make([]any, 0, 9)
+	if pp != nil {
+		dest = append(dest, pp)
+	}
+	dest = append(dest, &h.FilePath, &h.StartLine, &h.EndLine,
+		&chunkT, &symName, &language, &h.Content, &bm)
+	if err := rows.Scan(dest...); err != nil {
+		return Hit{}, fmt.Errorf("scan chunks_fts row: %w", err)
+	}
+	h.ChunkType = chunkT.String
+	h.SymbolName = symName.String
+	h.Language = language.String
+	// SQLite returns more-negative bm25 for better matches. Flip so
+	// callers can blend with cosine-style "higher is better" scores.
+	h.Score = -bm
+	return h, nil
+}
+
 // buildFTS5Query turns a free-text query into a safe FTS5 expression:
 // each whitespace-separated word becomes a double-quoted phrase, all
 // phrases are OR-joined. Single-character tokens are dropped (trigram