Skip to content

shards: simplify branch repository shard selection #1136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
keegancsmith merged 1 commit into from
Aug 14, 2026
+85 −321
Merged

shards: simplify branch repository shard selection #1136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 9 additions & 86 deletions search/branchesrepos_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,32 +184,18 @@ func TestSelectRepoSetBranchesReposBuildsBloom(t *testing.T) {
// checked below.
shards[missShards].repos = append(shards[missShards].repos, &zoekt.Repository{ID: ids[0][0]})

if !branchesReposMaySelect(shards, len(q.List), 2) {
t.Fatal("preflight did not recognize a filterable scan")
}

snapshots := snapshotBranchesRepos(q)
sel := newBranchesReposSelector(q.List, 2)
for _, shard := range shards[:missShards] {
if sel.containsDirect(shard.repos[0].ID) {
t.Fatalf("miss shard %d matched", shard.repos[0].ID)
}
}
if got := sel.missProbes; got != branchesReposMinimumMissProbes {
t.Fatalf("direct miss probes = %d, want %d", got, branchesReposMinimumMissProbes)
}
sel.missProbes = branchesReposMinimumMissProbes

var bloom branchesReposBloom
if !sel.maybeBuildBloom(shards[missShards:], &bloom) {
bloom := sel.maybeBuildBloom(shards[missShards:])
if bloom == nil {
t.Fatal("selector did not build a membership filter")
}
if bloom.mayContain(ids[0][0]) && sel.containsDirect(ids[0][0]) {
t.Fatalf("miss repository %d matched after filter setup", ids[0][0])
}
for _, shard := range shards[missShards:] {
id := shard.repos[0].ID
if !bloom.mayContain(id) || !sel.containsDirect(id) {
t.Fatalf("matching shard %d did not match after filter setup", id)
if !bloom.mayContain(id) {
t.Fatalf("matching shard %d was rejected by filter", id)
}
}
assertBranchesReposUnchanged(t, "bloom filter", q, snapshots)
Expand All @@ -219,28 +205,6 @@ func TestSelectRepoSetBranchesReposBuildsBloom(t *testing.T) {
}
}

func TestBranchesReposMayReachSelector(t *testing.T) {
shards, _ := newBranchesReposShards(32, 1)
if branchesReposMayReachSelector(shards, branchesReposBenchmarkBranches) {
t.Fatal("short scan reached selector threshold")
}

shards, _ = newBranchesReposShards(144, 1)
if !branchesReposMayReachSelector(shards, branchesReposBenchmarkBranches) {
t.Fatal("long scan did not reach selector threshold")
}
}

func TestBranchesReposMaySelectRequiresRemainingWork(t *testing.T) {
shards, _ := newBranchesReposShards(128, branchesReposBenchmarkRepos)
if !branchesReposMayReachSelector(shards, branchesReposBenchmarkBranches) {
t.Fatal("preflight did not reach selector threshold")
}
if branchesReposMaySelect(shards, branchesReposBenchmarkBranches, 128*100) {
t.Fatal("preflight selected a selector with too little remaining miss work")
}
}

func TestBranchesReposCanBuildBloomRejectsSaturatedFilter(t *testing.T) {
if !branchesReposCanBuildBloom(branchesReposBloomMaxCardinality) {
t.Fatal("filter capacity was rejected")
Expand All @@ -261,14 +225,9 @@ func TestBranchesReposSelectorSkipsNonProbingShards(t *testing.T) {
q.List[0].Repos.Add(100_000)
q.List[branchesReposBenchmarkBranches-1].Repos.Add(200_000)
sel := newBranchesReposSelector(q.List, 2)
for _, shard := range shards[:missShards] {
if sel.containsDirect(shard.repos[0].ID) {
t.Fatalf("miss shard %d matched", shard.repos[0].ID)
}
}
sel.missProbes = branchesReposMinimumMissProbes

var bloom branchesReposBloom
if sel.maybeBuildBloom(shards[missShards:], &bloom) {
if sel.maybeBuildBloom(shards[missShards:]) != nil {
t.Fatal("selector built a filter despite only unlisted remaining shards")
}
if !sel.settled {
Expand All @@ -283,48 +242,12 @@ func TestBranchesReposSelectorSkipsNonProbingShards(t *testing.T) {
shard.repos = []*zoekt.Repository{}
}
emptySelector := newBranchesReposSelector(q.List, 2)
for _, shard := range shards[:missShards] {
emptySelector.containsDirect(shard.repos[0].ID)
}
var emptyBloom branchesReposBloom
if emptySelector.maybeBuildBloom(shards[missShards:], &emptyBloom) {
emptySelector.missProbes = branchesReposMinimumMissProbes
if emptySelector.maybeBuildBloom(shards[missShards:]) != nil {
t.Fatal("selector built a filter despite only empty remaining shard lists")
}
}

func TestBranchesReposSelectorRetainsFirstBranch(t *testing.T) {
shards, ids := newBranchesReposShards(128, 1)
q := newBranchesReposQuery(branchesReposBenchmarkBranches)
for _, shardIDs := range ids {
addBranchesReposIDs(q, 0, shardIDs)
}
q.List[1].Repos.Add(100_000)

sel := newBranchesReposSelector(q.List, 129)
for _, shard := range shards {
if !sel.containsDirect(shard.repos[0].ID) {
t.Fatalf("matching shard %d did not match", shard.repos[0].ID)
}
}
if got, want := sel.preferred, -1; got != want {
t.Fatalf("preferred branch = %d, want %d", got, want)
}
if got := sel.missProbes; got != 0 {
t.Fatalf("direct miss probes = %d, want 0", got)
}
if branchesReposFirstMatchIsLater(shards, q.List) {
t.Fatal("first-branch-only query established a later preference")
}

late := newBranchesReposQuery(branchesReposBenchmarkBranches)
for _, shardIDs := range ids {
addBranchesReposIDs(late, branchesReposBenchmarkBranches-1, shardIDs)
}
if !branchesReposFirstMatchIsLater(shards, late.List) {
t.Fatal("later-branch query did not establish a preference")
}
}

func TestSelectRepoSetBranchesReposEmptyBitmaps(t *testing.T) {
shards, _ := newBranchesReposShards(branchesReposBenchmarkBranches+100, 1)
q := newBranchesReposQuery(branchesReposBenchmarkBranches)
Expand Down
Loading
Loading