diff --git a/.gitignore b/.gitignore index e9f580c5..f9dcd5a7 100644 --- a/.gitignore +++ b/.gitignore @@ -93,3 +93,6 @@ server/internal/httpapi/dashboard/dist/* portainer_mcp/ portainer-mcp* tools.yaml + +# Local load-test corpus + throwaway server instance (never committed) +/loadtests/ diff --git a/doc/CONFIG_REFERENCE.md b/doc/CONFIG_REFERENCE.md index 4bbb1c30..8cd09ca6 100644 --- a/doc/CONFIG_REFERENCE.md +++ b/doc/CONFIG_REFERENCE.md @@ -37,6 +37,7 @@ the DB. | `CIX_CHROMA_PERSIST_DIR` | `/data/chroma` | Legacy chromem-go store. Read on startup for the one-time import into the SQLite vector store, then left untouched as the rollback path. See [VECTORSTORE.md](VECTORSTORE.md). | | `CIX_VECTORS_DIR` | sibling of `CIX_CHROMA_PERSIST_DIR` (`/data/vectors`) | Vector store directory: one SQLite database per embedding namespace. | | `CIX_VECTOR_MMAP_SIZE` | `0` (off) | `PRAGMA mmap_size` for the vector store, in bytes. Roughly 40% lower search latency in exchange for resident memory — mapped database pages count in RSS. | +| `CIX_VECTOR_SCAN_QUANT` | `true` | Scan a compact int8 copy of each vector instead of the float32 original, rescoring the shortlist against the originals. Every score returned is the exact cosine; which documents reach the shortlist is an approximation, measured at recall 1.000 against exact search (see `doc/VECTORSTORE.md`). 3.4x fewer bytes read per query at 2048 dimensions, in exchange for roughly a quarter more disk. Set `false` if the volume cannot take it; existing copies are then ignored, and writes remove the copies of rows they touch so re-enabling rebuilds instead of trusting stale data. | | `CIX_GGUF_CACHE_DIR` | `/data/models` | Where downloaded GGUF files live. | | `CIX_PUBLIC_URL` | — | Externally-reachable URL used to build GitHub webhook delivery URLs. Empty disables webhook URL display. | diff --git a/doc/VECTORSTORE.md b/doc/VECTORSTORE.md index bc95c451..a37a9d93 100644 --- a/doc/VECTORSTORE.md +++ b/doc/VECTORSTORE.md @@ -150,21 +150,96 @@ on disk, deliberately: it keeps the package self-contained and live in `vectors`. A multi-kilobyte `TEXT` column pushes a row past SQLite's local-payload limit, and SQLite then keeps only ~1 kB of the row in the table page and spills the rest — *including the embedding* — into an overflow chain, -roughly doubling the pages a scan touches. Kept apart, a `vectors` row is -~3.2 kB and two of them share an 8 KiB page. Content is read only for the K +roughly doubling the pages a scan touches. Kept apart, a 768-dim `vectors` row +is ~3.2 kB and two of them share an 8 KiB page. Content is read only for the K winners of a search: one extra lookup per result. +**Why the scan reads a second copy of every vector.** The paragraph above stops +being true once the model is bigger than 1024 dimensions. A 2048-dim float32 +embedding is 8192 bytes on its own, past the 8157-byte local-payload limit, so +every `vectors` row spills into an overflow page and the scan is back to the +layout splitting out the content was meant to avoid. Measured with `dbstat` +over 400 rows, bytes a full scan must read per vector: + +| dimensions | representation | leaf | overflow | bytes/vector | +|---|---|---|---|---| +| 768 | float32 | 200 | 0 | 4096 | +| 1024 | float32 | 400 | 0 | 8192 | +| 2048 | float32 | 50 | 400 | 9216 | +| 768 | int8 | 40 | 0 | 819 | +| 1024 | int8 | 58 | 0 | 1188 | +| 2048 | int8 | 134 | 0 | 2744 | + +The pathological line is 1024, not 2048: nothing overflows there and the scan +still reads 8192 bytes to obtain 4096, because two 4.1 kB rows cannot share an +8 KiB page. Halving `output_dimension` to save time bought half the vector +quality for 89% of the I/O. + +`vectors_q8` removes the whole step function by scanning one byte per component +instead of four. `TestScanPackingEfficiency` and `TestScanBytesPerVectorBudget` +assert those numbers — as pages, not milliseconds, so they mean the same thing +in CI, on a laptop, and on the production box. + ## Search ``` -SELECT rowid, embedding FROM vectors INDEXED BY idx_vec_coll - WHERE collection_id = ? [AND ] +SELECT doc_id, scale, embedding FROM vectors_q8 INDEXED BY idx_q8_coll + WHERE collection_id = ? [AND language = ?] ``` -Rows stream past a dot product (embeddings are stored L2-normalised, so cosine -similarity *is* the dot product) into a top-K min-heap that rejects a losing -row with one comparison. Metadata and chunk text are fetched afterwards, for -the winners only. +Rows stream past an integer dot product into a top-K min-heap that rejects a +losing row with one comparison. The heap is wider than the caller's limit — the +int8 ranking chooses a shortlist, it does not produce the answer. The shortlist +is then rescored against the exact float32 vectors in `vectors`, and metadata +and chunk text are fetched for the winners only. + +**What the approximation costs.** Measured on 60k vectors of the load-test +fixture's largest collection (`ziglang/zig`, voyage-code-3 @2048) against 50 +real query-side embeddings, recall of the exact float32 top-K: + +| shortlist | k=10 | k=20 | +|---|---|---| +| 20 | 0.998 | 0.994 | +| 40 | 0.998 | 0.999 | +| **60** | **1.000** | **1.000** | +| 200 | 1.000 | 1.000 | + +Without rescoring at all the int8 ranking alone gives 0.994 at both k — the +quantisation misorders near-ties, it does not lose the documents, which is why +re-reading a few dozen exact vectors recovered every one of them here. +`q8Shortlist` therefore uses a floor of 64 and 4x the limit above it. Scan CPU +in the same run: 127 ms per query float32, 42 ms int8 (3.0x). + +Two different guarantees, worth keeping apart. **Scores are exact by +construction**: every number a caller sees is the cosine against the float32 +vector, computed by the rescore. **The result set is an approximation** whose +error was measured at zero on this corpus and is not proved at zero in general +— the shortlist is a fixed size and `topK` rejects boundary ties strictly, so a +collection holding more than `shortlist` documents inside one quantisation step +of each other (a file vendored a hundred times, say) can truncate a tie in scan +order, and the rescore cannot recover a document that was never shortlisted. +Widening the shortlist to swallow boundary ties would close that; it has not +been needed on any corpus measured so far. + +Scores returned to callers are always the exact cosine, never the int8 +estimate. That is load-bearing beyond cosmetics: `min_score` thresholds on it, +the workspace fan-out normalises across projects with it, and hybrid search +blends it with BM25 — an approximate score would move results between projects +in a way no single-project test would catch. `TestSearchScoresAreExact` pins it. + +**Building and rebuilding the copy.** Writes maintain `vectors_q8` in the same +transaction as `vectors`, so a collection created by this code is complete by +construction, and `q8_state` records that at creation — the readiness check is +a primary-key lookup, never a `COUNT`. A store written before the table existed +is converted by a background pass at open, largest collection first, in 2000-row +transactions at a 50% duty cycle; until a collection is covered its searches +take the float32 scan, which is correct and simply slower. Nothing is ever +marked complete before it is: the flag is written in the same transaction as +the batch that proves it. Set `CIX_VECTOR_SCAN_QUANT=false` to opt out — the +copy is roughly a quarter of the float32 bytes on top of an already large +store, so an operator short of disk needs a way to say no. Turning it off also +withdraws the completion flag from anything written while it is off, so turning +it back on rebuilds rather than trusting a stale copy. `INDEXED BY` is not an optimisation hint, it is a guarantee, and *which* index matters. Measured on the real index, scanning its largest (74k-row) collection: @@ -188,6 +263,10 @@ The metadata filter (`where`) mirrors chromem's semantics exactly, including the two odd cases: an unknown key with a non-empty value matches nothing, and an unknown key with an empty value matches everything. +`TestQ8ScanUsesCollectionIndex` pins the same guarantee for the compact table: +`idx_q8_coll`'s keys are `(collection_id, rowid)` for the same reason, and the +language filter must not change the driving index. + **Concurrency.** One scan per query, and a process-wide semaphore caps concurrent scans at `NumCPU`. Splitting a single query across workers was measured to buy nothing in the low-memory configuration (109 ms at 1 worker vs diff --git a/doc/openapi.yaml b/doc/openapi.yaml index f915431d..6243d5a3 100644 --- a/doc/openapi.yaml +++ b/doc/openapi.yaml @@ -2630,6 +2630,20 @@ paths: minimum: 0 maximum: 1 default: 0.4 + - name: timings + in: query + required: false + description: | + Attach a per-phase breakdown of where the query spent its + time (see WorkspaceSearchTimings). Diagnostic, not API + surface: it exists so a slow workspace query can be taken + apart, and it is off unless asked for. The server logs the + same breakdown by itself whenever a query is slow, so + catching a regression does not depend on someone having + passed this flag at the right moment. + schema: + type: boolean + default: false responses: "200": description: Search results @@ -4137,6 +4151,12 @@ components: Size of the tree. Absent when it could not be walked, which keeps "unreadable" distinguishable from "empty". The SQLite entry includes the -wal and -shm sidecars. + partial: + type: boolean + description: | + True when used_bytes undercounts because some entries inside the + tree were unreadable and skipped. Absent means the sum is + complete. fs_total_bytes: { type: integer, format: int64 } fs_free_bytes: { type: integer, format: int64 } @@ -6525,6 +6545,95 @@ components: under the new schema. items: $ref: "#/components/schemas/WorkspaceSearchStaleFTSRepo" + timings: + $ref: "#/components/schemas/WorkspaceSearchTimings" + + WorkspaceSearchTimings: + type: object + description: | + Where this query spent its time, in milliseconds. Returned only + when the request passes `timings=true` AND the query actually ran + a search — a workspace with no queryable project reports nothing + rather than a block of zeroes that would read as "instant". + + The fan-out phases report a sum AND a max, and both are needed: the + sum is how much work the query did across every project, the max is + how long it waited for the slowest one. With perfect parallelism the + wall time is the max; with none it is the sum; in practice it is + between them, and one number alone cannot say which. + + `projects_scanned` versus `projects_returned` is the ratio that says + how much of the work was discarded: the fan-out runs dense and BM25 + over every project in the workspace and then thresholds the answer + down to the relevant ones. `projects_in_panel` is a separate, + smaller question — how many of those the caller was actually shown, + after the `top_projects` cap. + + The named phases do not sum to `wall_ms`. The remainder is the + workspace visibility check, assembling the projects panel, the + round-robin interleave and writing the response — all in memory and, + on the load-test fixture, ~19 ms of ~9,900 ms. + properties: + wall_ms: + type: integer + description: The whole handler, from its first line to its last. + embed_ms: + type: integer + description: Round-trip to the embedding provider for the query text. + resolve_ms: + type: integer + description: | + Loading the workspace's project memberships and applying the + per-user access filter. Separate from the rest because it is the + one pre-fan-out step that grows with how many projects the + caller can see, rather than with the workspace. + stale_fts_ms: + type: integer + description: The pre-fan-out probe for repos with no BM25 mirror. + fanout_ms: + type: integer + description: Wall time of the parallel per-project phase. + dense_sum_ms: + type: integer + description: | + Vector-store search summed across projects, including hydration + of each project's winning rows, and including projects whose + query failed — the time was spent either way, and omitting it + would put the sums permanently below the wall time they explain. + dense_max_ms: + type: integer + description: | + The slowest single project's dense search. May belong to a + project whose query failed; the fan-out logs a warning of its + own for those. + bm25_ms: + type: integer + description: | + The workspace's BM25 search. One FTS5 statement covering every + project, partitioned per project by a window function — not a + sum over projects, which is why it has no matching `_max` + field. `MATCH` is evaluated over the whole server's index + whatever the scope, so asking once per project repeated the + same global work N times and the N queries contended over one + index on top of that. + fuse_ms: + type: integer + description: Normalisation, candidacy blending and thresholding. + projects_scanned: + type: integer + description: Projects the fan-out searched. + projects_returned: + type: integer + description: | + Projects that survived the relevance threshold — NOT the number + the caller was shown. Capping this at `top_projects` would peg + the scanned:returned ratio to a request parameter instead of + measuring how much of the fan-out's work was discarded. + projects_in_panel: + type: integer + description: | + Projects present in the response's `projects` array, i.e. + `min(projects_returned, top_projects)`. WorkspaceSearchPendingRepo: type: object diff --git a/server/cmd/cix-server/main.go b/server/cmd/cix-server/main.go index a2b8e7c6..8943e477 100644 --- a/server/cmd/cix-server/main.go +++ b/server/cmd/cix-server/main.go @@ -411,6 +411,7 @@ func run() (restart bool, err error) { Dir: cfg.VectorDirFor(comps), LegacyChromaDir: cfg.ChromaDirFor(comps), MMapBytes: cfg.VectorMMapSize, + ScanQuant: cfg.VectorScanQuantEnabled, Logger: logger, }) } @@ -437,6 +438,7 @@ func run() (restart bool, err error) { idx := indexer.New(database, vsHolder, embedSvc, logger) idx.SetEmbedIncludePath(cfg.EmbedIncludePath) + idx.SetMaxChunkTokens(cfg.MaxChunkTokens) // Record the active embedding model on every indexed project so the // dashboard can highlight stale vectors when the runtime provider / // model changes. Wire it as a live lookup so a runtime provider diff --git a/server/dashboard/src/modules/search/SearchPage.tsx b/server/dashboard/src/modules/search/SearchPage.tsx index 2c9cd04c..f61d8cae 100644 --- a/server/dashboard/src/modules/search/SearchPage.tsx +++ b/server/dashboard/src/modules/search/SearchPage.tsx @@ -44,18 +44,15 @@ export default function SearchPage() { const queryParam = params.get('q') ?? ''; const [draft, setDraft] = useState(queryParam); - // Debounce input → URL after 250ms idle; Enter commits immediately. - useEffect(() => { - const id = setTimeout(() => { - if (draft === queryParam) return; - const next = new URLSearchParams(params); - if (draft.trim()) next.set('q', draft); - else next.delete('q'); - setParams(next, { replace: true }); - }, 250); - return () => clearTimeout(id); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [draft]); + // Typing changes `draft` and nothing else. The query in the URL — which is + // what actually runs a search — moves only on submit. + // + // This used to debounce draft into the URL after 250ms idle. That is the + // usual pattern and it is wrong here: a semantic search embeds the query + // through the configured provider, so every pause while typing spent a real + // API call and a full fan-out to answer a half-written question. "retry with + // exponential backoff" typed at a normal pace fires on "retry", "retry with", + // "retry with expo" — three searches nobody asked for and one they did. // Follow the URL when it changes from outside (a pasted link, back button). useEffect(() => { @@ -172,7 +169,11 @@ function Results({ ); } if (query.trim().length < 2) { - return At least two characters, then results appear here.; + return ( + + At least two characters, then press Enter to search. + + ); } switch (mode) { case 'semantic': diff --git a/server/dashboard/src/modules/search/components/SearchBar.tsx b/server/dashboard/src/modules/search/components/SearchBar.tsx index d6570bfc..92977e31 100644 --- a/server/dashboard/src/modules/search/components/SearchBar.tsx +++ b/server/dashboard/src/modules/search/components/SearchBar.tsx @@ -13,7 +13,7 @@ export function SearchBar({ }: { value: string; onChange: (v: string) => void; - /** Fired on Enter — bypasses the debounce and commits immediately. */ + /** Fired on Enter. This is the ONLY thing that runs a search — typing does not. */ onSubmit?: (v: string) => void; placeholder?: string; className?: string; diff --git a/server/internal/chunker/chunker.go b/server/internal/chunker/chunker.go index 6391ba1b..5b147971 100644 --- a/server/internal/chunker/chunker.go +++ b/server/internal/chunker/chunker.go @@ -10,7 +10,10 @@ package chunker import ( + "errors" + "github.com/dvcdsys/code-index/server/internal/tokenizer" "log/slog" + "math" "path/filepath" "strings" "sync" @@ -457,15 +460,88 @@ type Reference struct { // falls back to sliding-window chunking for unsupported languages. The maxSize // parameter controls per-chunk character limit; pass 0 to use the default. func ChunkFile(filePath, content, language string, maxSize int) ([]Chunk, []Reference, error) { + return ChunkFileTokens(filePath, content, language, maxSize, nil, 0) +} + +// ChunkFileTokens is ChunkFile with a token budget. +// +// maxSize (bytes) has always been a stand-in for a token limit: the default +// 4500 is "1500 tokens x 3 bytes", a ratio that holds for dense ASCII code and +// falls apart everywhere else — Cyrillic or CJK comments cost two to three +// bytes per character, so a byte-sized chunk carries far fewer tokens than +// intended, while minified JavaScript packs far more. +// +// When budget is non-nil and reports exact counts, the size decision is made +// in tokens instead, and an over-budget chunk is cut on real token boundaries +// (via Budget.SplitPoints) rather than on a byte count. maxTokens <= 0 uses +// DefaultMaxChunkTokens. A nil or estimating budget keeps the byte path +// unchanged, so nothing about existing behaviour depends on a provider +// having a tokenizer. +func ChunkFileTokens(filePath, content, language string, maxSize int, budget tokenizer.Budget, maxTokens int) ([]Chunk, []Reference, error) { if maxSize <= 0 { maxSize = maxChunkSize } - chunks, refs, err := chunkWithTreesitter(filePath, content, language, maxSize) - if err != nil { - // Fallback: sliding window, no references. - return chunkFallback(filePath, content, language), nil, nil + if budget != nil && !budget.ExactCounts() { + // An estimate is what the byte path already is; do not pretend + // otherwise by routing through the token splitter. + budget = nil + } + if budget != nil { + if maxTokens <= 0 { + maxTokens = DefaultMaxChunkTokens + } + if lim := budget.MaxInputTokens(); lim > 0 && maxTokens > lim { + maxTokens = lim + } + } + // With a token budget the byte cap must not fire first: it is the very + // bias being removed (a byte limit cuts Cyrillic or CJK three times + // sooner than ASCII for the same token cost). Let the inner path emit + // whole semantic units and bound them in tokens afterwards. + innerMax := maxSize + if budget != nil { + innerMax = math.MaxInt32 + } + + chunks, refs, err := chunkWithTreesitter(filePath, content, language, innerMax) + if err != nil || len(chunks) == 0 { + // Tree-sitter declined (no grammar, parse failure, minified input) or + // produced nothing. Either way the fallback runs here, where the + // budget is known, rather than inside the tree-sitter path where it + // is not. + return chunkFallbackTokens(filePath, content, language, budget, maxTokens), nil, nil + } + return boundTokens(chunks, budget, maxTokens), refs, nil +} + +// boundTokens enforces the token budget over chunks from ANY path — the +// tree-sitter one, the bash regex extractor, or the sliding-window fallback. +// Applying it here rather than inside the tree-sitter path is deliberate: +// minified JavaScript and files with no grammar are exactly the inputs that +// reach the fallback, and they are also the ones most likely to blow the +// model's input limit. A budget that only covered the happy path would miss +// them. +func boundTokens(chunks []Chunk, budget tokenizer.Budget, maxTokens int) []Chunk { + if budget == nil || maxTokens <= 0 { + return chunks + } + out := make([]Chunk, 0, len(chunks)) + for _, c := range chunks { + // A byte-level BPE token can never cover less than one byte, so a + // chunk shorter than the budget provably fits and needs no counting + // at all. Most chunks are, so this skips the tokenizer on the hot + // path rather than paying for an answer already known. + if len(c.Content) <= maxTokens { + out = append(out, c) + continue + } + if budget.CountTokens(c.Content) > maxTokens { + out = append(out, splitChunkTokens(c, budget, maxTokens)...) + continue + } + out = append(out, c) } - return chunks, refs, nil + return out } // chunkFallback returns reasonable chunks for content that the tree-sitter @@ -476,6 +552,17 @@ func ChunkFile(filePath, content, language string, maxSize int) ([]Chunk, []Refe // `block` ones, which is much more useful for semantic search. If the // extractor returns nil (no symbols found), we fall through to the universal // sliding-window strategy so the file content is still indexed. +// errUseFallback tells the caller that the tree-sitter path declined this +// file and the fallback chunker must run instead. +// +// It exists because chunkWithTreesitter used to CALL chunkFallback itself and +// return the result as success. That made the caller's fallback branch +// unreachable — which is exactly how a token-aware fallback shipped as dead +// code: no-grammar, minified, parse-failure and empty-AST files all came back +// as byte windows wearing a success return, and no budget ever reached them. +// The decision belongs to whoever knows whether a token budget is in play. +var errUseFallback = errors.New("chunker: tree-sitter declined, use fallback") + func chunkFallback(filePath, content, language string) []Chunk { if language == "bash" { if c := bashRegexChunks(filePath, content); len(c) > 0 { @@ -485,6 +572,54 @@ func chunkFallback(filePath, content, language string) []Chunk { return chunkSlidingWindow(filePath, content, language) } +// chunkFallbackTokens is chunkFallback with a token budget: the sliding window +// walks token boundaries instead of a fixed byte count. +// +// The byte window is 4000 bytes regardless of what those bytes contain, so a +// file of Cyrillic or CJK prose — two to three bytes per character — produced +// windows worth a third of the intended tokens, and this is the path such +// files take, because "no grammar" and "not ASCII" go together often enough to +// matter. boundTokens alone could not fix it: it splits chunks that are too +// large and has no way to grow ones that are too small. +func chunkFallbackTokens(filePath, content, language string, budget tokenizer.Budget, maxTokens int) []Chunk { + if budget == nil || maxTokens <= 0 { + return chunkFallback(filePath, content, language) + } + if language == "bash" { + if c := bashRegexChunks(filePath, content); len(c) > 0 { + return boundTokens(c, budget, maxTokens) + } + } + if len(content) == 0 { + return nil + } + // One chunk, then let the token splitter cut it — same code path, same + // guarantees (pieces are substrings, none over budget, line numbers + // tracked) as every other over-budget chunk in this package. + // + // Note this drops the byte window's 500-byte overlap. That overlap existed + // so a match spanning a window boundary would still be found in one of the + // two windows, and nothing replaces it here: pieces are contiguous. The + // trade is deliberate for now — overlap has to be expressed in tokens to + // coexist with a token budget (size pieces at budget minus overlap, then + // extend each start back into the previous piece), which is a change worth + // making on its own rather than smuggling into a correctness fix. The + // tree-sitter path has never overlapped, so this makes the two paths + // consistent rather than making the fallback worse than its neighbours. + whole := Chunk{ + Content: content, + ChunkType: "block", + FilePath: filePath, + StartLine: 1, + EndLine: countNewlines(content) + 1, + Language: language, + } + if budget.CountTokens(content) <= maxTokens { + return []Chunk{whole} + } + return splitChunkTokens(whole, budget, maxTokens) +} + // --------------------------------------------------------------------------- // Tree-sitter path // --------------------------------------------------------------------------- @@ -529,11 +664,11 @@ func chunkWithTreesitter(filePath, content, language string, maxSize int) ([]Chu registryMu.RUnlock() if !ok { - return chunkFallback(filePath, content, language), nil, nil + return nil, nil, errUseFallback } if nodeKinds == nil { // Grammar exists but we don't have node definitions → sliding window. - return chunkFallback(filePath, content, language), nil, nil + return nil, nil, errUseFallback } if looksMinified(filePath, content, language) { // Minified/bundled sources are the parser's pathological case: a @@ -541,7 +676,7 @@ func chunkWithTreesitter(filePath, content, language string, maxSize int) ([]Chu // instance to its memory cap, and forces a pool recycle — all to // produce AST chunks with near-zero semantic-search value. Skip // straight to the sliding window. - return chunkFallback(filePath, content, language), nil, nil + return nil, nil, errUseFallback } // Build flat target → kind map. @@ -559,10 +694,10 @@ func chunkWithTreesitter(filePath, content, language string, maxSize int) ([]Chu // fall back to sliding window so the file is still indexed. slog.Warn("chunker: wasm parse failed, falling back to sliding window", "path", filePath, "language", language, "err", err) - return chunkFallback(filePath, content, language), nil, nil + return nil, nil, errUseFallback } if len(nodes) == 0 { - return chunkFallback(filePath, content, language), nil, nil + return nil, nil, errUseFallback } tree := buildFlatTree(nodes) @@ -604,7 +739,7 @@ func chunkWithTreesitter(filePath, content, language string, maxSize int) ([]Chu } if len(finalChunks) == 0 { - return chunkFallback(filePath, content, language), nil, nil + return nil, nil, errUseFallback } return finalChunks, refs, nil } @@ -1060,3 +1195,113 @@ func sortRanges(ranges [][2]int) { } } } + +// DefaultMaxChunkTokens is the token equivalent of maxChunkSize. The byte +// default was written as 1500*3 — a 1500-token target at three bytes each — +// so the token target is that same 1500, now expressed in the unit that +// actually matters. +// +// Exported so config.go can use it as the CIX_MAX_CHUNK_TOKENS default rather +// than repeating the number: two copies of a chunk-size default drifting apart +// is the exact failure this change removes for maxChunkSize. +const DefaultMaxChunkTokens = 1500 + +// splitChunkTokens cuts an over-budget chunk into pieces of <= maxTokens. +// +// It keeps splitChunk's attribution rule: only the first piece inherits +// SymbolName/ChunkType, the rest become `block`, so one long function does not +// produce N symbol rows all claiming to be it. +// +// The cut positions come from Budget.SplitPoints over the WHOLE content rather +// than from summing per-line counts. Per-line summing is off by the separators +// — joining lines reinserts newlines, and a newline plus the next line's +// indentation forms its own pre-token — so a budget of 1500 produced chunks of +// up to 1546 tokens on real files, roughly one extra token per line boundary. +// SplitPoints is exact by construction, so the bound actually holds. +// +// Each exact cut is then pulled BACK to the nearest line start, because a chunk +// that begins mid-line reads badly in search results and its line range lies. +// Moving a cut backwards only ever shrinks the piece before it, so the budget +// survives the adjustment. A line longer than the whole budget has no earlier +// boundary to snap to; there the exact cut stands and the line is split +// internally — which is the case the byte splitter could not handle at all. +func splitChunkTokens(chunk Chunk, budget tokenizer.Budget, maxTokens int) []Chunk { + content := chunk.Content + var out []Chunk + pos, line := 0, chunk.StartLine + var cuts []int // absolute offsets into content; nil means "recompute" + + for pos < len(content) { + if cuts == nil { + rel, _ := budget.SplitPoints(content[pos:], maxTokens) + if len(rel) == 0 { + out = append(out, mkPiece(chunk, content[pos:], line, len(out) == 0)) + break + } + cuts = make([]int, 0, len(rel)) + for _, r := range rel { + cuts = append(cuts, pos+r) + } + } + + at := cuts[0] + // Pull the cut back to a line start so a piece never begins mid-line: + // search results and the stored line range both lie otherwise. Moving + // backwards only shrinks this piece, so it stays inside the budget. A + // line wider than the whole budget has no earlier boundary — there the + // exact cut stands and the line is split internally, which is the case + // the byte splitter could not handle at all. + snapped := at + if nl := strings.LastIndexByte(content[pos:at], '\n'); nl >= 0 { + snapped = pos + nl + 1 + } + if snapped <= pos { + snapped = at + } + + piece := content[pos:snapped] + out = append(out, mkPiece(chunk, piece, line, len(out) == 0)) + line += strings.Count(piece, "\n") + pos = snapped + + if snapped == at { + // The boundary landed where the tokenizer put it, so the cuts + // after it are still valid and can be consumed without another + // pass. This is what keeps a 66 KB single line — where the + // newline snap never fires — from costing one full scan per + // piece. + cuts = cuts[1:] + if len(cuts) == 0 { + cuts = nil + } + continue + } + // Snapping moved the boundary: every later cut was measured from a + // start that no longer exists, and reusing them would hand the next + // piece the tokens this one gave up. Recompute. + cuts = nil + } + return out +} + +// mkPiece builds one output chunk, preserving splitChunk's attribution rule: +// only the first piece keeps SymbolName/ChunkType, so a long function does not +// produce N symbol rows all claiming to be it. +func mkPiece(src Chunk, content string, startLine int, first bool) Chunk { + c := Chunk{ + Content: content, + FilePath: src.FilePath, + StartLine: startLine, + EndLine: startLine + strings.Count(strings.TrimSuffix(content, "\n"), "\n"), + Language: src.Language, + ParentName: src.ParentName, + } + if first { + c.ChunkType = src.ChunkType + c.SymbolName = src.SymbolName + c.SymbolSignature = src.SymbolSignature + } else { + c.ChunkType = "block" + } + return c +} diff --git a/server/internal/chunker/chunker_tokens_test.go b/server/internal/chunker/chunker_tokens_test.go new file mode 100644 index 00000000..552ddeb5 --- /dev/null +++ b/server/internal/chunker/chunker_tokens_test.go @@ -0,0 +1,299 @@ +package chunker + +import ( + "strings" + "testing" + + "github.com/dvcdsys/code-index/server/internal/tokenizer" +) + +// fakeBudget is deliberately ADVERSARIAL: a newline costs a token, exactly +// like it does in the real tokenizer, where a line break plus the next line's +// indentation forms its own pre-token. +// +// The first version of this double counted whitespace-separated words and let +// newlines be free. That made the sum of per-line counts equal the count of +// the joined text — which is precisely the assumption the implementation got +// wrong, so the double agreed with the bug and the tests passed while real +// files came out 3% over budget. A test double that cannot express the +// failure mode cannot catch it. +// +// Counting rule: one token per word start, one per newline. Sum over pieces +// therefore does NOT equal the count of the concatenation unless the pieces +// are substrings — which is the property the splitter must have. +type fakeBudget struct{ maxInput int } + +func (f fakeBudget) MaxInputTokens() int { return f.maxInput } +func (f fakeBudget) ExactCounts() bool { return true } + +func (f fakeBudget) CountTokens(s string) int { + n, inWord := 0, false + for i := 0; i < len(s); i++ { + switch c := s[i]; { + case c == '\n': + n++ + inWord = false + case c == ' ' || c == '\t' || c == '\r': + inWord = false + default: + if !inWord { + inWord = true + n++ + } + } + } + return n +} + +// SplitPoints cuts before the token that would overflow the budget, so every +// piece it produces costs at most budget under CountTokens above. +func (f fakeBudget) SplitPoints(s string, budget int) ([]int, int) { + var offsets []int + total, since := 0, 0 + inWord := false + cut := func(at int) { + offsets = append(offsets, at) + since = 1 + } + for i := 0; i < len(s); i++ { + switch c := s[i]; { + case c == '\n': + total++ + since++ + if since > budget { + cut(i) + } + inWord = false + case c == ' ' || c == '\t' || c == '\r': + inWord = false + default: + if !inWord { + inWord = true + total++ + since++ + if since > budget { + cut(i) + } + } + } + } + return offsets, total +} + +var _ tokenizer.Budget = fakeBudget{} + +// TestTokenBudgetBoundsEveryChunk is the property the integration exists for: +// with a budget in hand, no emitted chunk may exceed it. +func TestTokenBudgetBoundsEveryChunk(t *testing.T) { + var sb strings.Builder + sb.WriteString("func run() {\n") + for i := 0; i < 300; i++ { + sb.WriteString("\tdo something with several words on this line\n") + } + sb.WriteString("}\n") + + b := fakeBudget{maxInput: 4096} + chunks, _, err := ChunkFileTokens("x.go", sb.String(), "go", 0, b, 50) + if err != nil { + t.Fatalf("chunk: %v", err) + } + if len(chunks) < 2 { + t.Fatalf("expected the body to be split, got %d chunk(s)", len(chunks)) + } + for i, c := range chunks { + if n := b.CountTokens(c.Content); n > 50 { + t.Errorf("chunk %d is %d tokens, over budget 50", i, n) + } + } +} + +// TestLongSingleLineIsSplit covers the hole the byte splitter had: its loop +// requires more than one line, so a minified file arrived at the embedder as +// one enormous chunk. On the reference corpus that produced a 65 KB chunk +// whose vector was an average of byte windows. +func TestLongSingleLineIsSplit(t *testing.T) { + line := strings.TrimSpace(strings.Repeat("token ", 5000)) + b := fakeBudget{maxInput: 4096} + + chunks, _, err := ChunkFileTokens("min.js", line, "javascript", 0, b, 100) + if err != nil { + t.Fatalf("chunk: %v", err) + } + for i, c := range chunks { + if n := b.CountTokens(c.Content); n > 100 { + t.Errorf("chunk %d is %d tokens, over budget 100 — long line not cut", i, n) + } + } + if len(chunks) < 2 { + t.Fatalf("expected the single line to be cut, got %d chunk(s)", len(chunks)) + } +} + +// TestBudgetCappedByModelContext — a chunk target above the model's own input +// window is meaningless; the smaller of the two must win. +func TestBudgetCappedByModelContext(t *testing.T) { + src := strings.TrimSpace(strings.Repeat("word ", 400)) + b := fakeBudget{maxInput: 40} + + chunks, _, err := ChunkFileTokens("x.txt", src, "text", 0, b, 10000) + if err != nil { + t.Fatalf("chunk: %v", err) + } + for i, c := range chunks { + if n := b.CountTokens(c.Content); n > 40 { + t.Errorf("chunk %d is %d tokens, over the model's %d-token window", i, n, 40) + } + } +} + +// TestEstimatingBudgetKeepsBytePath — a provider without a real tokenizer must +// not be routed through the token splitter: its numbers are the same guess the +// byte path already makes, and pretending otherwise hides that from the caller. +func TestEstimatingBudgetKeepsBytePath(t *testing.T) { + src := strings.Repeat("x := 1\n", 2000) + got, _, err := ChunkFileTokens("x.go", src, "go", 0, estimatingBudget{}, 10) + if err != nil { + t.Fatalf("chunk: %v", err) + } + want, _, err := ChunkFile("x.go", src, "go", 0) + if err != nil { + t.Fatalf("chunk: %v", err) + } + if len(got) != len(want) { + t.Errorf("estimating budget changed chunking: %d chunks vs %d on the byte path", + len(got), len(want)) + } +} + +type estimatingBudget struct{ fakeBudget } + +func (estimatingBudget) ExactCounts() bool { return false } + +// TestNilBudgetUnchanged pins that the default path is untouched. +func TestNilBudgetUnchanged(t *testing.T) { + src := strings.Repeat("func f() { return 1 }\n", 500) + a, _, err := ChunkFileTokens("x.go", src, "go", 0, nil, 0) + if err != nil { + t.Fatalf("chunk: %v", err) + } + b, _, err := ChunkFile("x.go", src, "go", 0) + if err != nil { + t.Fatalf("chunk: %v", err) + } + if len(a) != len(b) { + t.Errorf("nil budget diverged from ChunkFile: %d vs %d chunks", len(a), len(b)) + } +} + +// TestTokenSplitPreservesContent pins the invariant that makes the token +// splitter exact: its pieces are SUBSTRINGS of the chunk it was given, so +// concatenating them reproduces it byte for byte — nothing lost, nothing +// duplicated. +// +// The first implementation instead re-joined lines it had counted separately, +// and the newlines it reinserted cost tokens the running total never saw. A +// 1500-token budget produced 1546-token chunks on real files. Slicing the +// original removes that class of error rather than compensating for it. +// +// Asserted on splitChunkTokens directly, not through ChunkFileTokens: the +// sliding-window fallback deliberately overlaps its windows for recall, so +// whole-pipeline output is not expected to concatenate back. +func TestTokenSplitPreservesContent(t *testing.T) { + var sb strings.Builder + for i := 0; i < 200; i++ { + sb.WriteString("some line with a handful of words in it\n") + } + src := Chunk{ + Content: sb.String(), + FilePath: "x.txt", + StartLine: 1, + EndLine: 200, + ChunkType: "function", + SymbolName: strPtr("run"), + } + b := fakeBudget{maxInput: 4096} + + pieces := splitChunkTokens(src, b, 40) + var rebuilt strings.Builder + for i, c := range pieces { + rebuilt.WriteString(c.Content) + if n := b.CountTokens(c.Content); n > 40 { + t.Errorf("piece %d is %d tokens, over budget 40", i, n) + } + } + if rebuilt.String() != src.Content { + t.Errorf("concatenated pieces differ from the source (%d bytes vs %d)", + rebuilt.Len(), len(src.Content)) + } + if pieces[0].SymbolName == nil || *pieces[0].SymbolName != "run" || pieces[0].ChunkType != "function" { + t.Error("first piece must inherit the symbol") + } + for i, c := range pieces[1:] { + if c.SymbolName != nil || c.ChunkType != "block" { + t.Errorf("piece %d must be an anonymous block, got type %q", i+1, c.ChunkType) + } + } +} + +// TestTokenSplitLineNumbers — a piece that starts mid-file must report the +// line it actually starts on, or `cix search` sends the reader to the wrong +// place. +func TestTokenSplitLineNumbers(t *testing.T) { + var sb strings.Builder + for i := 0; i < 100; i++ { + sb.WriteString("word word word word word\n") + } + b := fakeBudget{maxInput: 4096} + + chunks := splitChunkTokens(Chunk{Content: sb.String(), FilePath: "x.txt", StartLine: 1}, b, 20) + line := 1 + for i, c := range chunks { + if c.StartLine != line { + t.Errorf("chunk %d starts at line %d, expected %d", i, c.StartLine, line) + } + line += strings.Count(c.Content, "\n") + } +} + +func strPtr(s string) *string { return &s } + +// TestFallbackFillsTheBudget covers the path a file with no grammar takes. +// The byte-sized sliding window cut every 4000 bytes regardless of content, so +// multi-byte text produced windows worth a fraction of the intended tokens — +// and boundTokens could not repair that, since it only splits chunks that are +// too big and cannot merge ones that are too small. +func TestFallbackFillsTheBudget(t *testing.T) { + // Two-bytes-per-character text, well past one byte window. + src := strings.Repeat("привіт світ це коментар українською\n", 400) + b := fakeBudget{maxInput: 4096} + // The budget must exceed one byte window's token worth, or the test + // passes on arithmetic: a 4000-byte window of this text is ~358 fake + // tokens, so at budget 200 boundTokens splits every window into 200+158 + // and both halves clear half-budget without the fallback ever being + // token-aware. At 800 the raw window is BELOW half the budget, so an + // under-filled chunk can only come from a byte-sized window. + const budget = 800 + + chunks, _, err := ChunkFileTokens("notes.unknownlang", src, "unknownlang", 0, b, budget) + if err != nil { + t.Fatalf("chunk: %v", err) + } + if len(chunks) < 2 { + t.Fatalf("expected several chunks, got %d", len(chunks)) + } + var under int + for i, c := range chunks { + n := b.CountTokens(c.Content) + if n > budget { + t.Errorf("chunk %d is %d tokens, over budget %d", i, n, budget) + } + // The last chunk is a remainder and may legitimately be short. + if i < len(chunks)-1 && n < budget/2 { + under++ + } + } + if under > 0 { + t.Errorf("%d of %d chunks are under half the budget — the window is still byte-sized", + under, len(chunks)) + } +} diff --git a/server/internal/chunker/corpus_property_test.go b/server/internal/chunker/corpus_property_test.go new file mode 100644 index 00000000..f414af6a --- /dev/null +++ b/server/internal/chunker/corpus_property_test.go @@ -0,0 +1,208 @@ +package chunker + +import ( + "math/rand" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/dvcdsys/code-index/server/internal/tokenizer" + "github.com/dvcdsys/code-index/server/internal/tokenizer/bpecount" +) + +// Property test over a real corpus. +// +// Chunk splitting under a token budget has no external oracle: unlike the +// tokenizer, which can be checked against HuggingFace's implementation and +// against Voyage's own billing, "where should a chunk be cut" is our decision +// and there is nothing to compare it to. What can be checked is that the +// properties we chose actually hold on inputs we did not write — and the +// bugs this file exists to catch were all found by real files rather than by +// hand-written cases: +// +// - a 65 KB single line (a Zig integer literal) that the byte splitter left +// whole, at 65,553 tokens against a 32K context; +// - minified JavaScript, which has no grammar and therefore reaches the +// sliding-window fallback rather than the tree-sitter path; +// - per-line token counting, which was 3% under the truth because joining +// lines reinserts newlines that cost tokens. +// +// The corpus is not in the repository — it is a local fixture of cloned +// repositories, tens of gigabytes. Point the test at one: +// +// CIX_TEST_CORPUS_DIR=…/loadtests/data/repos/repos \ +// CIX_TEST_TOKENIZER=…/voyage-code-3.tokenizer.json \ +// go test ./internal/chunker/ -run Corpus +// +// Without those it skips, so a clean checkout and CI stay green. + +type realBudget struct{ tk *bpecount.Counter } + +func (realBudget) MaxInputTokens() int { return 32000 } +func (realBudget) ExactCounts() bool { return true } +func (b realBudget) CountTokens(s string) int { return b.tk.Count(s) } +func (b realBudget) SplitPoints(s string, n int) ([]int, int) { return b.tk.SplitPoints(s, n) } + +var _ tokenizer.Budget = realBudget{} + +var extLang = map[string]string{ + ".go": "go", ".py": "python", ".ts": "typescript", ".tsx": "tsx", + ".js": "javascript", ".jsx": "javascript", ".java": "java", ".rs": "rust", + ".c": "c", ".h": "c", ".cpp": "cpp", ".rb": "ruby", ".php": "php", + ".kt": "kotlin", ".swift": "swift", ".ex": "elixir", ".zig": "zig", + ".lua": "lua", ".sh": "bash", ".md": "markdown", ".json": "json", +} + +// sampleCorpus walks the fixture and returns up to n files, deterministically +// shuffled so a failure is reproducible. +func sampleCorpus(t *testing.T, root string, n int) []string { + t.Helper() + var files []string + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return nil //nolint:nilerr // an unreadable entry is not this test's problem + } + if d.IsDir() { + if d.Name() == ".git" { + return filepath.SkipDir + } + return nil + } + if _, ok := extLang[strings.ToLower(filepath.Ext(path))]; ok { + files = append(files, path) + } + return nil + }) + if err != nil { + t.Fatalf("walk corpus: %v", err) + } + sort.Strings(files) + rng := rand.New(rand.NewSource(20260818)) + rng.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] }) + if len(files) > n { + files = files[:n] + } + return files +} + +func corpusBudget(t *testing.T) (realBudget, string) { + t.Helper() + dir := os.Getenv("CIX_TEST_CORPUS_DIR") + tok := os.Getenv("CIX_TEST_TOKENIZER") + if dir == "" || tok == "" { + t.Skip("set CIX_TEST_CORPUS_DIR and CIX_TEST_TOKENIZER to run corpus property tests") + } + tk, err := bpecount.Load(tok) + if err != nil { + t.Fatalf("load tokenizer: %v", err) + } + return realBudget{tk}, dir +} + +// TestCorpusChunksRespectBudget is the property that matters to the API: no +// chunk may cost more tokens than the budget, whatever path produced it. +func TestCorpusChunksRespectBudget(t *testing.T) { + b, dir := corpusBudget(t) + const budget = 1500 + + files := sampleCorpus(t, dir, 400) + if len(files) == 0 { + t.Skip("corpus contains no recognised source files") + } + + var checked, chunks, worst int + for _, f := range files { + src, err := os.ReadFile(f) + if err != nil || len(src) == 0 { + continue + } + lang := extLang[strings.ToLower(filepath.Ext(f))] + got, _, err := ChunkFileTokens(f, string(src), lang, 0, b, budget) + if err != nil { + t.Errorf("%s: %v", f, err) + continue + } + checked++ + chunks += len(got) + for i, c := range got { + n := b.CountTokens(c.Content) + if n > worst { + worst = n + } + if n > budget { + t.Errorf("%s chunk %d: %d tokens, over budget %d", f, i, n, budget) + } + } + } + t.Logf("%d files, %d chunks, largest %d tokens (budget %d)", checked, chunks, worst, budget) +} + +// TestCorpusSplitPreservesContent asserts the splitter loses and duplicates +// nothing, on chunks taken from real files rather than constructed ones. Run +// against splitChunkTokens directly: the sliding-window fallback overlaps its +// windows by design, so whole-pipeline output is not expected to concatenate +// back to the source. +func TestCorpusSplitPreservesContent(t *testing.T) { + b, dir := corpusBudget(t) + const budget = 300 + + var split int + for _, f := range sampleCorpus(t, dir, 200) { + src, err := os.ReadFile(f) + if err != nil || len(src) == 0 { + continue + } + whole := Chunk{ + Content: string(src), + FilePath: f, + StartLine: 1, + ChunkType: "file", + } + if b.CountTokens(whole.Content) <= budget { + continue + } + pieces := splitChunkTokens(whole, b, budget) + split++ + + var rebuilt strings.Builder + for i, p := range pieces { + rebuilt.WriteString(p.Content) + if n := b.CountTokens(p.Content); n > budget { + t.Errorf("%s piece %d: %d tokens, over budget %d", f, i, n, budget) + } + } + if rebuilt.String() != whole.Content { + t.Errorf("%s: pieces do not reconstruct the file (%d bytes vs %d)", + f, rebuilt.Len(), len(whole.Content)) + } + } + t.Logf("%d files exceeded the budget and were split", split) +} + +// TestCorpusLineNumbers — a chunk's StartLine must point at the line its text +// actually begins on, or search results send the reader to the wrong place. +func TestCorpusLineNumbers(t *testing.T) { + b, dir := corpusBudget(t) + const budget = 300 + + for _, f := range sampleCorpus(t, dir, 150) { + src, err := os.ReadFile(f) + if err != nil || len(src) == 0 { + continue + } + whole := Chunk{Content: string(src), FilePath: f, StartLine: 1} + if b.CountTokens(whole.Content) <= budget { + continue + } + line := 1 + for i, p := range splitChunkTokens(whole, b, budget) { + if p.StartLine != line { + t.Errorf("%s piece %d starts at line %d, expected %d", f, i, p.StartLine, line) + break + } + line += strings.Count(p.Content, "\n") + } + } +} diff --git a/server/internal/chunksfts/chunksfts.go b/server/internal/chunksfts/chunksfts.go index 73da2b6f..cf662f00 100644 --- a/server/internal/chunksfts/chunksfts.go +++ b/server/internal/chunksfts/chunksfts.go @@ -28,6 +28,7 @@ import ( "context" "database/sql" "fmt" + "sort" "strings" ) @@ -199,6 +200,25 @@ func DeleteByProject(ctx context.Context, db *sql.DB, projectPath string) error // // Empty or all-tokens-too-short queries return a nil slice without // hitting the DB — there is nothing to match. +// +// NOTE: nothing in production calls this any more — workspace search asks +// SearchProjects for every project at once, and a single-project workspace +// goes through the same path. It is kept for two reasons, both worth more +// than the ~40 lines it costs: +// +// 1. it is the independent oracle for SearchProjects. The two are +// structurally different statements that must return byte-identical +// rankings, because the per-project BM25 signal feeds project candidacy +// in workspace search — a divergence would silently re-rank the projects +// panel with no error and no failed_repos. +// TestSearchProjects_MatchesPerProjectQueries is that check, and it is +// only worth anything while this stays a separate implementation. +// Collapsing it into SearchProjects([]string{p}) would make the test +// compare a function to itself; +// 2. it is the fallback if a single-project regression ever shows up in the +// partitioned form. +// +// Do not delete it as unused. func SearchProject(ctx context.Context, db *sql.DB, projectPath, query string, limit int) ([]Hit, error) { if limit <= 0 { limit = 20 @@ -214,7 +234,7 @@ func SearchProject(ctx context.Context, db *sql.DB, projectPath, query string, l FROM chunks_fts cf JOIN chunks_meta cm ON cm.rowid = cf.rowid WHERE chunks_fts MATCH ? AND cm.project_path = ? - ORDER BY bm ASC + ORDER BY bm ASC, cm.rowid ASC LIMIT ?`, fts5Q, projectPath, limit, ) @@ -224,23 +244,10 @@ func SearchProject(ctx context.Context, db *sql.DB, projectPath, query string, l defer rows.Close() var out []Hit for rows.Next() { - var ( - h Hit - chunkT sql.NullString - symName sql.NullString - language sql.NullString - bm float64 - ) - if err := rows.Scan(&h.FilePath, &h.StartLine, &h.EndLine, - &chunkT, &symName, &language, &h.Content, &bm); err != nil { - return nil, fmt.Errorf("scan chunks_fts row: %w", err) + h, err := scanHit(rows) + if err != nil { + return nil, err } - h.ChunkType = chunkT.String - h.SymbolName = symName.String - h.Language = language.String - // SQLite returns more-negative bm25 for better matches. Flip so - // callers can blend with cosine-style "higher is better" scores. - h.Score = -bm out = append(out, h) } if err := rows.Err(); err != nil { @@ -249,6 +256,387 @@ func SearchProject(ctx context.Context, db *sql.DB, projectPath, query string, l return out, nil } +// searchProjectsBatch caps how many project_path values go into one IN list, +// and payloadFetchBatch does the same for the rowid list of the second +// statement. +// +// 500 is NOT a headroom number. Measured through this driver, `rowid IN (...)` +// takes 32,766 placeholders and fails at 32,767 — SQLite raised +// SQLITE_MAX_VARIABLE_NUMBER from 999 to 32,766 in 3.32.0 and modernc tracks a +// recent upstream, so the ceiling is two orders of magnitude away and neither +// constant is anywhere near it. The reason for 500 is consistency: it is +// hydrateBatch (internal/vectorstore/search.go), the batch size the vector +// store already uses for its own IN lists, and one number for both is worth +// more than a tuned one for each. Raising it would want a measurement, and +// 43 x 50 = 2,150 rowids in five statements is nothing against a multi-second +// BM25 scan, so there is nothing to gain by measuring. +const ( + searchProjectsBatch = 500 + payloadFetchBatch = 500 +) + +// SearchProjects answers the same question as SearchProject for many +// projects at once, returning each project's top `perProject` hits keyed +// by project_path. Projects with no match are absent from the map rather +// than present with an empty slice — a caller distinguishing "nothing +// matched" from "not asked about" gets that for free, and "BM25 found +// nothing here" is a signal this package exists to produce. +// +// Why this is not a loop over SearchProject: FTS5 drives the query. It +// evaluates MATCH over the WHOLE chunks_fts table — every project on the +// server — joins each hit to chunks_meta, and only then discards the rows +// belonging to other projects. So the per-project cost barely depends on +// the project's size, and asking N times does the same global work N +// times. Measured on a 43-project workspace, BM25 was 78-80% of the +// fan-out's total work, and each query slowed from ~400 ms standalone to +// as much as 7 s when 43 of them ran against the index at once. +// +// The per-project trim is a bounded heap in Go, not a window function in +// SQL: a window has to sort the ENTIRE match set to find N rows per +// project, and the match set here is the whole server's index. See +// workspaceScanQuery for why that is the dominant cost and what it +// measured. +// +// Both paths order by (bm ASC, rowid ASC) — the per-project query in its +// ORDER BY, the workspace path in rankedRow.betterThan. The rowid is +// defensive, not a fix for an observed bug: bm25 ties are the norm rather +// than the exception in a trigram index over real code — in this package's +// own test corpus 14 of 16 hits share a score with another hit — and today +// SQLite happens to return tied rows in rowid order for the LIMIT form, so +// the two would agree without being told to. That is unspecified behaviour +// of the sorter. Naming the tiebreak on BOTH sides makes the agreement a +// property of the code instead of a coincidence a future planner is free +// to break. +func SearchProjects(ctx context.Context, db *sql.DB, projectPaths []string, query string, perProject int) (map[string][]Hit, error) { + if perProject <= 0 { + perProject = 20 + } + fts5Q := buildFTS5Query(query) + if fts5Q == "" || len(projectPaths) == 0 { + return nil, nil + } + + out := make(map[string][]Hit, len(projectPaths)) + for start := 0; start < len(projectPaths); start += searchProjectsBatch { + end := start + searchProjectsBatch + if end > len(projectPaths) { + end = len(projectPaths) + } + if err := searchProjectsBatchInto(ctx, db, projectPaths[start:end], fts5Q, perProject, out); err != nil { + return nil, err + } + } + return out, nil +} + +func searchProjectsBatchInto(ctx context.Context, db *sql.DB, projectPaths []string, + fts5Q string, perProject int, dst map[string][]Hit) error { + + args := make([]any, 0, len(projectPaths)+1) + args = append(args, fts5Q) + for _, pp := range projectPaths { + args = append(args, pp) + } + + rows, err := db.QueryContext(ctx, workspaceScanQuery(placeholders(len(projectPaths))), args...) + if err != nil { + return fmt.Errorf("chunks_fts workspace search: %w", err) + } + tops := make(map[string]*topHits, len(projectPaths)) + for rows.Next() { + var pp string + var r rankedRow + if err := rows.Scan(&pp, &r.rid, &r.bm); err != nil { + rows.Close() + return fmt.Errorf("scan chunks_fts ranking row: %w", err) + } + t := tops[pp] + if t == nil { + t = &topHits{n: perProject} + tops[pp] = t + } + t.offer(r) + } + err = rows.Err() + rows.Close() + if err != nil { + return fmt.Errorf("iterate chunks_fts: %w", err) + } + + ranked := make(map[string][]rankedRow, len(tops)) + var rids []int64 + for pp, t := range tops { + ordered := t.sorted() + ranked[pp] = ordered + for _, r := range ordered { + rids = append(rids, r.rid) + } + } + // Sorted because rids was built by ranging a map, so without this the IN + // lists — and therefore the batch boundaries — differ between two runs of + // the same query. The results do not (payload is keyed by rowid and each + // project is assembled in rank order), but a statement whose bound + // parameters come out of Go's map iteration cannot be compared plan-to-plan + // between runs, which is exactly what someone timing this will want to do. + // Ascending rowids also probe both B-trees in order rather than at random. + // Not measured as a speedup; the reason to do it is the determinism. + sort.Slice(rids, func(i, j int) bool { return rids[i] < rids[j] }) + + payload, err := fetchPayload(ctx, db, rids) + if err != nil { + return err + } + collectHits(ranked, payload, dst) + return nil +} + +// collectHits pairs each project's ranked rows with the payload fetched for +// them, and writes the projects that still have hits into dst. +// +// A row whose chunk vanished between the ranking scan and the payload fetch is +// dropped. Two statements cannot be atomic the way the one they replaced was, +// and a hit fewer is the right answer for a chunk that no longer exists; +// erroring would fail a whole workspace search because one file happened to be +// getting reindexed. A project that loses ALL of its survivors that way is left +// out of dst entirely, because this package's contract is that a project with +// no match is absent rather than present with an empty slice. +// +// What makes the split safe is not in this package: a rowid can only go +// MISSING, never come back pointing at a different chunk. chunks_meta.rowid is +// INTEGER PRIMARY KEY AUTOINCREMENT (internal/db/schema.go), so SQLite never +// re-issues a rowid after a delete. Drop the AUTOINCREMENT and a reindex could +// hand project B's chunk back under project A's score, with no error and +// nothing in failed_repos — at which point this needs cm.project_path in the +// payload SELECT and a check against the ranked row. +func collectHits(ranked map[string][]rankedRow, payload map[int64]Hit, dst map[string][]Hit) { + for pp, rows := range ranked { + hits := make([]Hit, 0, len(rows)) + for _, r := range rows { + h, ok := payload[r.rid] + if !ok { + continue + } + h.Score = -r.bm + hits = append(hits, h) + } + if len(hits) > 0 { + dst[pp] = hits + } + } +} + +// rankedRow is a matched chunk before its payload is fetched: the two columns +// the ranking needs and nothing else. +type rankedRow struct { + rid int64 + bm float64 +} + +// betterThan orders rows the way the per-project query's +// ORDER BY bm ASC, rowid ASC does. SQLite gives more-negative bm25 to better +// matches, so smaller wins; ties break on the lower rowid. The tiebreak is not +// cosmetic — in a trigram index over real code most hits share a score with +// another hit, and without it the surviving set would depend on the order the +// scan happened to visit rows in. +func (r rankedRow) betterThan(o rankedRow) bool { + if r.bm != o.bm { + return r.bm < o.bm + } + return r.rid < o.rid +} + +// topHits keeps the best n rows seen for one project. +// +// h is a max-heap on `betterThan`: h[0] is the WORST row kept, which is the +// one a new row has to beat. That makes the common case — a row that does not +// make the cut — a single comparison, which is the whole point of doing this +// here instead of in SQL. See workspaceScanQuery for why. +type topHits struct { + n int + h []rankedRow +} + +func (t *topHits) offer(r rankedRow) { + if t.n <= 0 { + return + } + if len(t.h) < t.n { + t.h = append(t.h, r) + t.up(len(t.h) - 1) + return + } + if t.h[0].betterThan(r) { + return + } + t.h[0] = r + t.down(0) +} + +// sorted returns the kept rows best-first, leaving the heap unusable. +func (t *topHits) sorted() []rankedRow { + out := t.h + sort.Slice(out, func(i, j int) bool { return out[i].betterThan(out[j]) }) + t.h = nil + return out +} + +func (t *topHits) up(i int) { + for i > 0 { + p := (i - 1) / 2 + if !t.h[p].betterThan(t.h[i]) { + return + } + t.h[p], t.h[i] = t.h[i], t.h[p] + i = p + } +} + +func (t *topHits) down(i int) { + for { + worst := i + for _, c := range [2]int{2*i + 1, 2*i + 2} { + if c < len(t.h) && t.h[worst].betterThan(t.h[c]) { + worst = c + } + } + if worst == i { + return + } + t.h[i], t.h[worst] = t.h[worst], t.h[i] + i = worst + } +} + +// fetchPayload reads the chunk columns for the rows that survived ranking. +func fetchPayload(ctx context.Context, db *sql.DB, rids []int64) (map[int64]Hit, error) { + out := make(map[int64]Hit, len(rids)) + for start := 0; start < len(rids); start += payloadFetchBatch { + end := start + payloadFetchBatch + if end > len(rids) { + end = len(rids) + } + batch := rids[start:end] + args := make([]any, len(batch)) + for i, rid := range batch { + args[i] = rid + } + rows, err := db.QueryContext(ctx, payloadQuery(placeholders(len(batch))), args...) + if err != nil { + return nil, fmt.Errorf("chunks_fts payload fetch: %w", err) + } + for rows.Next() { + var rid int64 + var h Hit + var chunkT, symName, language sql.NullString + if err := rows.Scan(&rid, &h.FilePath, &h.StartLine, &h.EndLine, + &chunkT, &symName, &language, &h.Content); err != nil { + rows.Close() + return nil, fmt.Errorf("scan chunks_fts payload row: %w", err) + } + h.ChunkType = chunkT.String + h.SymbolName = symName.String + h.Language = language.String + out[rid] = h + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, fmt.Errorf("iterate chunks_fts payload: %w", err) + } + } + return out, nil +} + +// workspaceScanQuery streams every matched row's project, rowid and BM25 +// score. It deliberately does no ordering and no trimming: the caller keeps a +// bounded per-project heap as the rows go past. +// +// The obvious form asks SQLite for the answer directly, with +// ROW_NUMBER() OVER (PARTITION BY project_path ORDER BY bm) and rn <= N. That +// is correct and it is what this shipped first, but a window function has to +// sort the ENTIRE match set to find N rows per project, and the match set here +// is the whole server's index: the trigram tokenizer makes a common short word +// match a quarter of the corpus, because "and" occurs inside command, handler, +// standard and random. On the load-test fixture a six-term query matched +// 623,913 rows to keep 2,300, and a six-term query with the word "test" in it +// matched 1,288,739. Sorting those to keep fifty per project is the single +// largest cost in the statement. +// +// A bounded heap looks at each row once and rejects most of them in one +// comparison. Measured on that fixture, window form vs heap: 144 ms vs 119 ms +// on a 21k match set, 2,466 vs 1,455 on 624k, 5,216 vs 3,014 on 1.29M. It wins +// at every size, and it wins by more as the match set grows. +// +// The result is IDENTICAL, not merely close — same rows, same order. That is +// what TestSearchProjects_MatchesPerProjectQueries checks, against the +// per-project statement as the oracle. +// +// Those timings came from a bench harness that is NOT in this repository: +// /loadtests/ is gitignored, corpus and tools alike. To recreate it, time this +// statement plus the Go-side heap against the same statement wrapped in the +// window form, over a corpus whose match set is orders of magnitude larger +// than the result. +func workspaceScanQuery(ph string) string { + return fmt.Sprintf(` + SELECT cm.project_path, cm.rowid, bm25(chunks_fts) + FROM chunks_fts cf + JOIN chunks_meta cm ON cm.rowid = cf.rowid + WHERE chunks_fts MATCH ? AND cm.project_path IN (%s)`, ph) +} + +// payloadQuery fetches the chunk columns for rows that already survived +// ranking. Keeping the payload out of the scan matters for the same reason the +// ranking is not done in SQL: carrying file_path and content through a +// 600k-row scan materialises them for every match to return a couple of +// thousand. Both halves of that lesson cost a shipped regression to learn. +func payloadQuery(ph string) string { + return fmt.Sprintf(` + SELECT cm.rowid, cm.file_path, cm.start_line, cm.end_line, + cm.chunk_type, cm.symbol_name, cm.language, cf.content + FROM chunks_meta cm + JOIN chunks_fts cf ON cf.rowid = cm.rowid + WHERE cm.rowid IN (%s)`, ph) +} + +// placeholders builds "?,?,?" for an IN list. n is always >= 1: SearchProjects +// returns early on an empty slice and the batching loop never produces an empty +// batch. There is deliberately no n == 0 branch — an empty list used to render +// as IN (NULL), which matches nothing and is indistinguishable from "nothing +// matched". A syntax error from IN () is the better failure: it is loud, and it +// happens at the call that is wrong. +func placeholders(n int) string { + return strings.TrimSuffix(strings.Repeat("?,", n), ",") +} + +// scanHit reads one row of the single-project ranking query. +// +// The workspace query no longer shares this: it scans (project, rowid, score) +// and fetches the payload separately, so the two paths now build a Hit in two +// different places. They must still produce byte-identical Hits — +// TestSearchProjects_MatchesPerProjectQueries compares them — and a mistake +// made symmetrically in both would pass that test, so keep the two column +// lists side by side when editing either. +func scanHit(rows *sql.Rows) (Hit, error) { + var ( + h Hit + chunkT sql.NullString + symName sql.NullString + language sql.NullString + bm float64 + ) + if err := rows.Scan(&h.FilePath, &h.StartLine, &h.EndLine, + &chunkT, &symName, &language, &h.Content, &bm); err != nil { + return Hit{}, fmt.Errorf("scan chunks_fts row: %w", err) + } + h.ChunkType = chunkT.String + h.SymbolName = symName.String + h.Language = language.String + // SQLite returns more-negative bm25 for better matches. Flip so + // callers can blend with cosine-style "higher is better" scores. + h.Score = -bm + return h, nil +} + // buildFTS5Query turns a free-text query into a safe FTS5 expression: // each whitespace-separated word becomes a double-quoted phrase, all // phrases are OR-joined. Single-character tokens are dropped (trigram diff --git a/server/internal/chunksfts/chunksfts_test.go b/server/internal/chunksfts/chunksfts_test.go index ab9e4de1..e55a1b42 100644 --- a/server/internal/chunksfts/chunksfts_test.go +++ b/server/internal/chunksfts/chunksfts_test.go @@ -3,6 +3,9 @@ package chunksfts import ( "context" "database/sql" + "fmt" + "math/rand" + "sort" "strings" "testing" @@ -270,3 +273,565 @@ func TestBuildFTS5Query(t *testing.T) { } } } + +// hitKey identifies a hit for comparison. Content is included because two +// chunks of the same file can share a line span only if something upstream +// is wrong, and if that ever happens the comparison should notice. +func hitKey(h Hit) string { + return fmt.Sprintf("%s:%d-%d|%s|%.6f", h.FilePath, h.StartLine, h.EndLine, h.SymbolName, h.Score) +} + +func keysOf(hits []Hit) []string { + out := make([]string, 0, len(hits)) + for _, h := range hits { + out = append(out, hitKey(h)) + } + return out +} + +// seedCorpus fills several projects with overlapping vocabulary, so BM25 has +// something to rank rather than a single obvious winner per project. +func seedCorpus(t *testing.T, d *sql.DB, projects []string) { + t.Helper() + bodies := []string{ + "func retryWithBackoff(ctx context.Context) error { return retry(ctx) }", + "// retry policy: exponential backoff with jitter, capped at one minute", + "func backoffDuration(attempt int) time.Duration { return base << attempt }", + "type RetryPolicy struct { MaxAttempts int; Backoff time.Duration }", + "// no mention of the interesting words at all, just filler content here", + "func retry(ctx context.Context) error { for { if err := do(); err == nil { return nil } } }", + } + for pi, p := range projects { + for bi, body := range bodies { + // Vary how many copies each project gets so BM25 scores differ + // across projects rather than tying everywhere. + copies := 1 + (pi+bi)%3 + chunks := make([]Chunk, 0, copies) + for c := 0; c < copies; c++ { + chunks = append(chunks, Chunk{ + Content: body, + FilePath: fmt.Sprintf("src/f%02d.go", bi), + StartLine: 1 + c*10, + EndLine: 5 + c*10, + SymbolName: fmt.Sprintf("S%02d", bi), + Language: "go", + }) + } + upsert(t, d, p, fmt.Sprintf("src/f%02d.go", bi), chunks) + } + // Deliberate BM25 ties: byte-identical chunks in different files + // score identically, so any limit below their count forces the + // engine to pick some of them arbitrarily. Without an explicit + // tiebreak the per-project query and the partitioned one are free + // to pick differently, and the equivalence this test asserts would + // hold only by luck. Ties are not exotic here — a trigram index + // over real code is full of near-duplicate boilerplate. + // file_path and symbol_name are indexed columns, so a tie needs all + // three to match: same file, same symbol, same content. Only the + // line span differs, and line numbers are not part of the index. + tied := make([]Chunk, 0, 6) + for i := 0; i < 6; i++ { + tied = append(tied, Chunk{ + Content: "func retryWithBackoff(ctx context.Context) error { return retry(ctx) }", + FilePath: "src/tied.go", + StartLine: 1 + i*10, + EndLine: 5 + i*10, + SymbolName: "Tie", + Language: "go", + }) + } + upsert(t, d, p, "src/tied.go", tied) + } +} + +// TestSearchProjects_MatchesPerProjectQueries is the equivalence test for the +// workspace-wide query: for every project, the partitioned result must be what +// the per-project query returns — same hits, same order, same scores. +// +// This is the whole safety argument for replacing N queries with one. The +// per-project BM25 signal feeds project candidacy in workspace search, so a +// partitioned result that merely contains the right rows in a different order +// would silently re-rank projects. +func TestSearchProjects_MatchesPerProjectQueries(t *testing.T) { + d := openTestDB(t) + ctx := context.Background() + projects := []string{"proj-a", "proj-b", "proj-c", "proj-d"} + seedCorpus(t, d, projects) + + for _, query := range []string{ + "retry backoff", + "retry", + "exponential backoff with jitter", + "nothing matches this string xyzzy", + } { + for _, limit := range []int{3, 50} { + batched, err := SearchProjects(ctx, d, projects, query, limit) + if err != nil { + t.Fatalf("SearchProjects(%q, %d): %v", query, limit, err) + } + for _, p := range projects { + want, err := SearchProject(ctx, d, p, query, limit) + if err != nil { + t.Fatalf("SearchProject(%q, %q): %v", p, query, err) + } + got := batched[p] + if len(want) == 0 { + if _, present := batched[p]; present { + t.Errorf("%q/%q limit=%d: project with no hits is present in the map", + query, p, limit) + } + continue + } + gk, wk := keysOf(got), keysOf(want) + if len(gk) != len(wk) { + t.Errorf("%q/%q limit=%d: got %d hits, per-project query returns %d", + query, p, limit, len(gk), len(wk)) + continue + } + for i := range wk { + if gk[i] != wk[i] { + t.Errorf("%q/%q limit=%d: rank %d differs\n got %s\n want %s", + query, p, limit, i, gk[i], wk[i]) + break + } + } + } + } + } +} + +// TestSearchProjects_DoesNotPrefixMatchProjectPaths guards the IN list. Project +// paths are namespaced strings that routinely share prefixes — "local:host:/x" +// and "local:host:/x/y" are different projects — so an implementation that +// filtered with LIKE, or that built the list by concatenation, would leak one +// project's chunks into another's slice. +func TestSearchProjects_DoesNotPrefixMatchProjectPaths(t *testing.T) { + d := openTestDB(t) + ctx := context.Background() + + upsert(t, d, "proj", "a.go", []Chunk{ + {Content: "func retryWithBackoff() {}", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"}, + }) + upsert(t, d, "proj-extended", "b.go", []Chunk{ + {Content: "func retryWithBackoff() {}", FilePath: "b.go", StartLine: 1, EndLine: 2, Language: "go"}, + }) + + got, err := SearchProjects(ctx, d, []string{"proj"}, "retry", 50) + if err != nil { + t.Fatalf("SearchProjects: %v", err) + } + if len(got) != 1 { + t.Fatalf("asked about one project, got slices for %d: %v", len(got), got) + } + for _, h := range got["proj"] { + if h.FilePath != "a.go" { + t.Errorf("hit from another project leaked in: %+v", h) + } + } + if _, present := got["proj-extended"]; present { + t.Error("a project that was not asked about appears in the result") + } +} + +// TestSearchProjects_SpansTheBatchBoundary checks the project IN-list batching. +// dst is filled across several statements and each project is written to +// exactly once — a batch that replaced the map instead of adding to it, or that +// dropped its last slice, would only show up above the batch size. +// +// It does NOT reach the rowid batching inside fetchPayload; one hit per project +// keeps that list at exactly searchProjectsBatch. See +// TestFetchPayload_SpansTheBatchBoundary. +func TestSearchProjects_SpansTheBatchBoundary(t *testing.T) { + d := openTestDB(t) + ctx := context.Background() + + const n = searchProjectsBatch + 7 + projects := make([]string, 0, n) + for i := 0; i < n; i++ { + p := fmt.Sprintf("p%04d", i) + projects = append(projects, p) + upsert(t, d, p, "a.go", []Chunk{ + {Content: "func retryWithBackoff() {}", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"}, + }) + } + + got, err := SearchProjects(ctx, d, projects, "retry", 50) + if err != nil { + t.Fatalf("SearchProjects: %v", err) + } + if len(got) != n { + t.Errorf("got hits for %d projects, want %d — a batch was lost or overwritten", len(got), n) + } + for _, p := range projects { + if len(got[p]) != 1 { + t.Errorf("%s: got %d hits, want 1", p, len(got[p])) + } + } +} + +// TestSearchProjects_EmptyInputs pins the two no-op paths, both of which must +// avoid touching the DB: nothing to match, and nobody to match it for. +func TestSearchProjects_EmptyInputs(t *testing.T) { + d := openTestDB(t) + ctx := context.Background() + seedCorpus(t, d, []string{"proj-a"}) + + if got, err := SearchProjects(ctx, d, []string{"proj-a"}, " x ", 50); err != nil || got != nil { + t.Errorf("all-tokens-too-short query: got %v, %v; want nil, nil", got, err) + } + if got, err := SearchProjects(ctx, d, nil, "retry", 50); err != nil || got != nil { + t.Errorf("no projects: got %v, %v; want nil, nil", got, err) + } +} + +// TestSearchProjects_ScanDoesNotSortTheMatchSet pins the reason the ranking +// moved out of SQL. The window form has to sort every matched row to find N +// per project; on the load-test fixture that is up to 1.29 million rows to +// keep 2,300. The scan query must stay a plain scan — no sorter of any kind. +func TestSearchProjects_ScanDoesNotSortTheMatchSet(t *testing.T) { + d := openTestDB(t) + ctx := context.Background() + seedCorpus(t, d, []string{"p1", "p2"}) + + plan := explain(t, ctx, d, workspaceScanQuery(placeholders(2)), + `"retry" OR "backoff"`, "p1", "p2") + if strings.Contains(plan, "TEMP B-TREE") { + t.Errorf("the ranking scan sorts the whole match set again:\n%s", plan) + } +} + +// TestExplainRejectsTheSortingForms is the mutation check for the test above, +// kept in the tree rather than run by hand: it builds forms that DO sort the +// match set and asserts the assertion above would reject each one. Without +// this, a change in how SQLite reports plans could turn the guard into a +// tautology that passes on everything, and nothing would say so. +// +// Two shapes, not one. The window form is what this PR deleted; a plain +// ORDER BY added back to the scan is the regression far more likely to +// actually happen, and a guard is worth exactly what it rejects. +func TestExplainRejectsTheSortingForms(t *testing.T) { + d := openTestDB(t) + ctx := context.Background() + seedCorpus(t, d, []string{"p1", "p2"}) + + window := ` + WITH hits AS ( + SELECT cm.project_path AS pp, cm.rowid AS rid, bm25(chunks_fts) AS bm + FROM chunks_fts cf + JOIN chunks_meta cm ON cm.rowid = cf.rowid + WHERE chunks_fts MATCH ? AND cm.project_path IN (?,?) + ), + ranked AS ( + SELECT pp, rid, bm, + ROW_NUMBER() OVER (PARTITION BY pp ORDER BY bm ASC, rid ASC) AS rn + FROM hits + ) + SELECT r.pp, cm.file_path, r.bm + FROM ranked r + JOIN chunks_meta cm ON cm.rowid = r.rid + WHERE r.rn <= ?` + + for _, tc := range []struct { + name string + query string + args []any + }{ + { + name: "the window form this replaced", + query: window, + args: []any{`"retry" OR "backoff"`, "p1", "p2", 3}, + }, + { + // Written out rather than derived from workspaceScanQuery: this + // subtest is a claim about how SQLite REPORTS a sort, not about + // production code, and appending to the real statement made it + // fail for the wrong reason whenever that statement was itself + // mutated to sort. + name: "an ORDER BY added back to the scan", + query: ` + SELECT cm.project_path, cm.rowid, bm25(chunks_fts) + FROM chunks_fts cf + JOIN chunks_meta cm ON cm.rowid = cf.rowid + WHERE chunks_fts MATCH ? AND cm.project_path IN (?,?) + ORDER BY bm25(chunks_fts)`, + args: []any{`"retry" OR "backoff"`, "p1", "p2"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + plan := explain(t, ctx, d, tc.query, tc.args...) + if !strings.Contains(plan, "TEMP B-TREE") { + t.Errorf("this form no longer reports a sorter, so the plan "+ + "assertion no longer distinguishes it:\n%s", plan) + } + }) + } +} + +// TestSearchProjects_FetchesPayloadByRowid guards the second half of the same +// lesson: file_path and content are fetched for the rows that survived, by +// rowid, and never carried through the scan. +// +// The assertion is on the WHOLE FTS5 idxStr, not a prefix of it. FTS5 packs its +// plan into one string: "0:=" is a bare rowid lookup, and a MATCH adds an "M" +// plus the matched column, so a payload fetch that ALSO matched reports +// "0:=M3" — which still contains "0:=" and does not contain "0:M". The first +// version of this test asserted on those two prefixes and therefore passed on +// exactly the merge it existed to catch. Found in review of #266, not by the +// suite, which is the whole argument for the companion test below. +func TestSearchProjects_FetchesPayloadByRowid(t *testing.T) { + d := openTestDB(t) + ctx := context.Background() + seedCorpus(t, d, []string{"p1", "p2"}) + + idxs := ftsIndexes(explain(t, ctx, d, payloadQuery(placeholders(2)), 1, 2)) + if len(idxs) == 0 { + t.Fatal("the payload fetch does not touch chunks_fts at all") + } + for _, idx := range idxs { + if idx != "0:=" { + t.Errorf(`chunks_fts is not a plain rowid lookup in the payload `+ + `fetch: idxStr %q ("=" is the rowid constraint; an "M" means a `+ + `MATCH crept back in)`, idx) + } + } +} + +// ftsIndexes returns every FTS5 idxStr in a query plan, whole. The planner +// prints it as "... VIRTUAL TABLE INDEX " at the end of the line. +func ftsIndexes(plan string) []string { + var out []string + for _, line := range strings.Split(plan, "\n") { + if _, idx, ok := strings.Cut(line, "VIRTUAL TABLE INDEX "); ok { + out = append(out, strings.TrimSpace(idx)) + } + } + return out +} + +// TestExplainRejectsThePayloadShapes is the mutation check for the test above, +// kept in the tree rather than run by hand: it builds the shapes that test +// exists to reject and asserts it would reject them. +// +// The MATCH case is not hypothetical. The prefix-matching version of the guard +// let it straight through, and nothing in the suite said so. +func TestExplainRejectsThePayloadShapes(t *testing.T) { + d := openTestDB(t) + ctx := context.Background() + seedCorpus(t, d, []string{"p1", "p2"}) + + for _, tc := range []struct { + name string + query string + args []any + }{ + { + name: "the two statements merged back together", + query: ` + SELECT cm.rowid, cm.file_path, cf.content + FROM chunks_meta cm + JOIN chunks_fts cf ON cf.rowid = cm.rowid + WHERE chunks_fts MATCH ? AND cm.rowid IN (?,?)`, + args: []any{`"retry" OR "backoff"`, 1, 2}, + }, + { + name: "a join FTS5 cannot serve by rowid", + query: ` + SELECT cm.rowid, cm.file_path, cf.content + FROM chunks_meta cm + JOIN chunks_fts cf ON cf.rowid + 0 = cm.rowid + WHERE cm.rowid IN (?,?)`, + args: []any{1, 2}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + plan := explain(t, ctx, d, tc.query, tc.args...) + for _, idx := range ftsIndexes(plan) { + if idx != "0:=" { + return + } + } + t.Errorf("this shape reports a plain rowid lookup, so the payload "+ + "guard no longer distinguishes it:\n%s", plan) + }) + } +} + +// TestFetchPayload_SpansTheBatchBoundary covers the rowid IN-list batching. +// +// TestSearchProjects_SpansTheBatchBoundary cannot reach it, and the reason is a +// coincidence of the two constants being equal: that test seeds one hit per +// project, so the rowid list is at most searchProjectsBatch long and the +// payload loop runs exactly once however many projects there are. Production is +// 43 projects x 50 hits = five batches, so without this the path that always +// runs in production would be the one nothing covers. +func TestFetchPayload_SpansTheBatchBoundary(t *testing.T) { + d := openTestDB(t) + ctx := context.Background() + + const n = payloadFetchBatch + 7 + chunks := make([]Chunk, 0, n) + for i := 0; i < n; i++ { + chunks = append(chunks, Chunk{ + Content: "func retryWithBackoff() {}", + FilePath: "a.go", + StartLine: 1 + i*10, EndLine: 5 + i*10, + Language: "go", + }) + } + upsert(t, d, "proj", "a.go", chunks) + + got, err := SearchProjects(ctx, d, []string{"proj"}, "retry", n) + if err != nil { + t.Fatalf("SearchProjects: %v", err) + } + if len(got["proj"]) != n { + t.Errorf("got %d hits, want %d — a payload batch was dropped", + len(got["proj"]), n) + } +} + +// TestCollectHits covers what splitting one statement into two actually +// changed: a chunk can disappear between the ranking scan and the payload +// fetch. Racing a real delete against a live query is not worth building, so +// the seam is tested directly — a payload map with rows deliberately left out +// is exactly the state that race produces. +func TestCollectHits(t *testing.T) { + rows := []rankedRow{{rid: 7, bm: -9}, {rid: 8, bm: -5}, {rid: 9, bm: -1}} + full := map[int64]Hit{ + 7: {FilePath: "a.go"}, 8: {FilePath: "b.go"}, 9: {FilePath: "c.go"}, + } + + t.Run("all present", func(t *testing.T) { + dst := map[string][]Hit{} + collectHits(map[string][]rankedRow{"p": rows}, full, dst) + got := dst["p"] + if len(got) != 3 { + t.Fatalf("got %d hits, want 3", len(got)) + } + for i, want := range []struct { + file string + score float64 + }{{"a.go", 9}, {"b.go", 5}, {"c.go", 1}} { + if got[i].FilePath != want.file || got[i].Score != want.score { + t.Errorf("rank %d: got %s/%v, want %s/%v", + i, got[i].FilePath, got[i].Score, want.file, want.score) + } + } + }) + + t.Run("one row vanished", func(t *testing.T) { + partial := map[int64]Hit{7: full[7], 9: full[9]} + dst := map[string][]Hit{} + collectHits(map[string][]rankedRow{"p": rows}, partial, dst) + got := dst["p"] + if len(got) != 2 { + t.Fatalf("got %d hits, want 2", len(got)) + } + if got[0].FilePath != "a.go" || got[1].FilePath != "c.go" { + t.Errorf("got %s,%s — the surviving rows lost their rank order", + got[0].FilePath, got[1].FilePath) + } + if got[0].Score != 9 || got[1].Score != 1 { + t.Errorf("got scores %v,%v — a dropped row shifted the scores", + got[0].Score, got[1].Score) + } + }) + + t.Run("every row vanished", func(t *testing.T) { + dst := map[string][]Hit{} + collectHits(map[string][]rankedRow{"p": rows}, map[int64]Hit{}, dst) + if _, present := dst["p"]; present { + t.Errorf("a project whose every survivor vanished is present with "+ + "%d hits; this package's contract is that it is absent", + len(dst["p"])) + } + }) +} + +// TestTopHits_MatchesAFullSort is the property test for the bounded heap that +// replaced SQLite's window function. +// +// The scores are drawn from a deliberately tiny set so that most rows tie: +// in a trigram index over real code most hits share a score with another hit, +// which makes the (score, rowid) tiebreak the part most likely to be wrong and +// least likely to be noticed. Rows are offered in a shuffled order, because an +// implementation that quietly depended on arrival order would still pass if +// they arrived sorted. +func TestTopHits_MatchesAFullSort(t *testing.T) { + for _, n := range []int{1, 3, 50} { + for seed := int64(1); seed <= 20; seed++ { + rng := rand.New(rand.NewSource(seed)) + rows := make([]rankedRow, 0, 500) + for i := 0; i < 500; i++ { + rows = append(rows, rankedRow{ + rid: int64(rng.Intn(1 << 20)), + bm: -float64(rng.Intn(8)), + }) + } + seen := map[int64]bool{} + uniq := rows[:0] + for _, r := range rows { + if !seen[r.rid] { + seen[r.rid] = true + uniq = append(uniq, r) + } + } + rows = uniq + + top := &topHits{n: n} + for _, r := range rows { + top.offer(r) + } + got := top.sorted() + + want := append([]rankedRow(nil), rows...) + sort.Slice(want, func(i, j int) bool { return want[i].betterThan(want[j]) }) + if len(want) > n { + want = want[:n] + } + if len(got) != len(want) { + t.Fatalf("n=%d seed=%d: kept %d rows, a full sort keeps %d", + n, seed, len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("n=%d seed=%d: rank %d is %+v, a full sort puts %+v there", + n, seed, i, got[i], want[i]) + } + } + } + } +} + +// TestTopHits_ZeroLimitKeepsNothing pins the guard in offer. perProject is +// clamped to a positive number by SearchProjects, so this is about the heap +// being safe on its own terms rather than about a reachable call. +func TestTopHits_ZeroLimitKeepsNothing(t *testing.T) { + top := &topHits{n: 0} + top.offer(rankedRow{rid: 1, bm: -9}) + if got := top.sorted(); len(got) != 0 { + t.Errorf("n=0 kept %d rows", len(got)) + } +} + +func explain(t *testing.T, ctx context.Context, d *sql.DB, query string, args ...any) string { + t.Helper() + rows, err := d.QueryContext(ctx, "EXPLAIN QUERY PLAN "+query, args...) + if err != nil { + t.Fatalf("explain: %v", err) + } + defer rows.Close() + var plan strings.Builder + for rows.Next() { + var a, b, c int + var detail string + if err := rows.Scan(&a, &b, &c, &detail); err != nil { + t.Fatalf("scan plan row: %v", err) + } + plan.WriteString(detail + "\n") + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate plan: %v", err) + } + return plan.String() +} diff --git a/server/internal/config/config.go b/server/internal/config/config.go index 4743a947..c66f630d 100644 --- a/server/internal/config/config.go +++ b/server/internal/config/config.go @@ -5,6 +5,7 @@ package config import ( "fmt" + "github.com/dvcdsys/code-index/server/internal/chunker" "net" "os" "path/filepath" @@ -54,7 +55,19 @@ type Config struct { // 0 (the default) leaves it off. Env: CIX_VECTOR_MMAP_SIZE. It buys // roughly 40% lower search latency and costs resident memory: every // connection maps the database file and mapped pages count in RSS. - VectorMMapSize int64 + VectorMMapSize int64 + // VectorScanQuantEnabled controls the compact int8 copy the vector store + // scans instead of the float32 originals. Env: CIX_VECTOR_SCAN_QUANT, + // default true. + // + // It exists because turning it on costs disk before it saves time: the + // copy is about a quarter of the float32 bytes, added to a store that may + // already be the largest thing on the volume, and it is built by a + // background pass over every existing vector. An operator who is short of + // disk, or who wants to isolate a search-quality question from the + // approximation, needs a way to say no. Off means every collection keeps + // using the exact float32 scan — correct, and as slow as it was before. + VectorScanQuantEnabled bool SQLitePath string MaxFileSize int ExcludedDirs []string @@ -309,6 +322,12 @@ func Load() (*Config, error) { } c.VectorMMapSize = int64(vecMMap) + scanQuant, err := getenvBool("CIX_VECTOR_SCAN_QUANT", true) + if err != nil { + return nil, err + } + c.VectorScanQuantEnabled = scanQuant + authOff, err := getenvBool("CIX_AUTH_DISABLED", false) if err != nil { return nil, err @@ -352,7 +371,7 @@ func Load() (*Config, error) { } c.ChunkMaxConcurrent = chunkConc - maxChunk, err := getenvInt("CIX_MAX_CHUNK_TOKENS", 1500) + maxChunk, err := getenvInt("CIX_MAX_CHUNK_TOKENS", chunker.DefaultMaxChunkTokens) if err != nil { return nil, err } diff --git a/server/internal/embeddings/provider/voyage/factory.go b/server/internal/embeddings/provider/voyage/factory.go index 64d12c57..926e4869 100644 --- a/server/internal/embeddings/provider/voyage/factory.go +++ b/server/internal/embeddings/provider/voyage/factory.go @@ -33,6 +33,13 @@ func (factory) SchemaJSON() []byte { Description: "int8 is dequantized to float32 on the server side.", }, {Name: "truncation", Label: "Truncate over-length input", Kind: "bool", Default: true}, + { + Name: "tokenizer_path", Label: "Tokenizer file", Kind: "string", + Description: "Absolute path to the model's tokenizer.json (huggingface.co/voyageai/). " + + "Set it and token counts become exact: batches pack to the real limit instead of a " + + "byte guess that overestimates ~2x, and over-long inputs split on token boundaries " + + "instead of byte windows. Empty falls back to the estimate.", + }, {Name: "api_key_env", Label: "API key env var", Kind: "secret-env", Required: true, Default: defaultAPIKeyEnv}, }, } diff --git a/server/internal/embeddings/provider/voyage/voyage.go b/server/internal/embeddings/provider/voyage/voyage.go index 04318f85..c9e73375 100644 --- a/server/internal/embeddings/provider/voyage/voyage.go +++ b/server/internal/embeddings/provider/voyage/voyage.go @@ -36,6 +36,7 @@ import ( "golang.org/x/time/rate" "github.com/dvcdsys/code-index/server/internal/embeddings/provider" + "github.com/dvcdsys/code-index/server/internal/tokenizer/bpecount" ) // voyageBatchTooLargeRegex matches Voyage's per-batch token-limit @@ -159,6 +160,15 @@ type Config struct { // all in-flight + recent requests). 0 = no throttling. RateLimitTPM int `json:"rate_limit_tpm,omitempty"` + // TokenizerPath points at the model's tokenizer.json (the file + // Voyage publishes at huggingface.co/voyageai/). When set and + // loadable, token counts become EXACT and the per-batch cap rises to + // exactTokensPerBatch — the 40K of headroom the byte heuristic needed + // is headroom against the heuristic, not against Voyage. When empty or + // unreadable the provider logs once and falls back to estimateTokens, + // so a missing file degrades throughput, never correctness. + TokenizerPath string `json:"tokenizer_path,omitempty"` + // MaxInputsPerRequest overrides defaultMaxBatchSize. 0 = use // the default (128, safe for voyage-code-*). Operators running // only voyage-3* may bump this to 1000 for fewer round-trips. @@ -189,7 +199,13 @@ func (c *Config) maxBatchSize() int { return defaultMaxBatchSize } -// maxTokensPerBatch returns the effective per-POST token cap. +// maxTokensPerBatch returns the cap implied by config alone — the operator's +// override, or the conservative byte-heuristic default. +// +// Callers on the hot path want (*Provider).maxTokensPerBatch instead, which +// also knows whether a tokenizer is loaded. Two same-named methods one on +// Config and one on Provider is how the batch log came to report 80K while +// packing used 115K, so this one is only for the Provider method to build on. func (c *Config) maxTokensPerBatch() int { if c.MaxTokensPerRequest > 0 { return c.MaxTokensPerRequest @@ -266,6 +282,10 @@ type Provider struct { // budget is a sliding minute and bursting saves nothing. reqLimiter *rate.Limiter + // counter is the model's real tokenizer, or nil when no tokenizer.json + // was configured or it failed to load. Safe for concurrent use. + counter *bpecount.Counter + // tokenLimiter caps tokens-per-minute when cfg.RateLimitTPM > 0. // Burst is set to maxTokensPerBatch so a single full-budget POST // can pass even when the bucket is otherwise empty (we'd just @@ -294,6 +314,19 @@ func New(cfg Config, secrets provider.SecretLookup, logger *slog.Logger) *Provid secrets: secrets, http: &http.Client{Timeout: 60 * time.Second}, } + if cfg.TokenizerPath != "" { + c, err := bpecount.Load(cfg.TokenizerPath) + if err != nil { + // Not fatal: the byte heuristic still works. Loud because the + // operator asked for exact counts and is not getting them. + logger.Warn("voyage: tokenizer load failed, falling back to byte estimate", + "path", cfg.TokenizerPath, "err", err) + } else { + p.counter = c + logger.Info("voyage: exact token counting enabled", "path", cfg.TokenizerPath) + } + } + // Convert RPM/TPM to per-second token-bucket rates. burst on the // request bucket is 1 (one request worth of "credit"); burst on // the token bucket equals one full POST so we don't deadlock a @@ -302,7 +335,7 @@ func New(cfg Config, secrets provider.SecretLookup, logger *slog.Logger) *Provid p.reqLimiter = rate.NewLimiter(rate.Limit(float64(cfg.RateLimitRPM)/60.0), 1) } if cfg.RateLimitTPM > 0 { - p.tokenLimiter = rate.NewLimiter(rate.Limit(float64(cfg.RateLimitTPM)/60.0), cfg.maxTokensPerBatch()) + p.tokenLimiter = rate.NewLimiter(rate.Limit(float64(cfg.RateLimitTPM)/60.0), p.maxTokensPerBatch()) } return p } @@ -402,6 +435,38 @@ func (p *Provider) EmbedDocuments(ctx context.Context, texts []string) ([][]floa // such chunk, but oversize chunks are rare on well-chunked // indexes — the indexer should already be cutting at function / // class boundaries. +// splitForInput cuts one input down to what the model can read. +// +// With a tokenizer, the question "does this fit" has an exact answer, so the +// byte cap is not consulted at all: an input under the model's context window +// goes through whole, however many bytes it is, and one over it is cut on real +// token boundaries. Without a tokenizer we are back to guessing, and the byte +// cap is the guess. +// +// This matters because the chunker now sizes in tokens. Its bound and the +// provider's byte cap are different units: at CIX_MAX_CHUNK_TOKENS=20000 — +// legal, well inside the 32K window — chunks of 40-80 KB are ordinary, and +// every one of them used to be byte-windowed here and have its window vectors +// averaged into a single vector representing neither half. The averaging path +// now only runs where it is genuinely needed: no tokenizer, no exact answer. +func (p *Provider) splitForInput(text string, maxBytes int) []string { + if p.counter == nil { + return splitOversizeInput(text, maxBytes) + } + limit := p.MaxInputTokens() + offsets, total := p.counter.SplitPoints(text, limit) + if total <= limit || len(offsets) == 0 { + return []string{text} + } + out := make([]string, 0, len(offsets)+1) + prev := 0 + for _, off := range offsets { + out = append(out, text[prev:off]) + prev = off + } + return append(out, text[prev:]) +} + func (p *Provider) embedAndAverage(ctx context.Context, texts []string, inputType string) ([][]float32, error) { maxIn := p.cfg.maxInputBytes() @@ -411,7 +476,7 @@ func (p *Provider) embedAndAverage(ctx context.Context, texts []string, inputTyp var expanded []string totalSplits := 0 for i, t := range texts { - windows := splitOversizeInput(t, maxIn) + windows := p.splitForInput(t, maxIn) spans[i] = span{start: len(expanded), length: len(windows)} expanded = append(expanded, windows...) if len(windows) > 1 { @@ -419,23 +484,35 @@ func (p *Provider) embedAndAverage(ctx context.Context, texts []string, inputTyp } } if totalSplits > 0 { - p.logger.Info("voyage: oversize inputs split into byte-windows", - "original_inputs", len(texts), - "total_windows", len(expanded), - "split_windows", totalSplits, - "max_input_bytes", maxIn, - ) + // Report the unit the split actually used: with a tokenizer the cut is + // on token boundaries against the model's context, and logging a byte + // cap there sends whoever reads this to the wrong knob. + if p.counter != nil { + p.logger.Info("voyage: oversize inputs split on token boundaries", + "original_inputs", len(texts), + "total_windows", len(expanded), + "split_windows", totalSplits, + "max_input_tokens", p.MaxInputTokens(), + ) + } else { + p.logger.Info("voyage: oversize inputs split into byte-windows", + "original_inputs", len(texts), + "total_windows", len(expanded), + "split_windows", totalSplits, + "max_input_bytes", maxIn, + ) + } } // Phase 2: batch + POST as before, on the expanded slice. - batches := planBatches(expanded, p.cfg.maxBatchSize(), p.cfg.maxTokensPerBatch()) + batches := planBatches(expanded, p.cfg.maxBatchSize(), p.maxTokensPerBatch(), p.CountTokens) if len(batches) > 1 { p.logger.Info("voyage: splitting batch", "model", p.cfg.Model, "total_inputs", len(expanded), "sub_batches", len(batches), "limit_inputs", p.cfg.maxBatchSize(), - "limit_tokens", p.cfg.maxTokensPerBatch(), + "limit_tokens", p.maxTokensPerBatch(), ) } allVecs := make([][]float32, 0, len(expanded)) @@ -561,7 +638,10 @@ func (p *Provider) embedWithAdaptiveSplit(ctx context.Context, texts []string, i // operator can override them via the admin form when their tier or // chosen model allows a higher cap (e.g. voyage-3-large at 1000 // inputs/POST instead of 128). -func planBatches(texts []string, maxInputs, maxTokens int) [][]string { +func planBatches(texts []string, maxInputs, maxTokens int, count func(string) int) [][]string { + if count == nil { + count = estimateTokens + } if len(texts) == 0 { return nil } @@ -569,7 +649,7 @@ func planBatches(texts []string, maxInputs, maxTokens int) [][]string { var current []string currentTokens := 0 for _, t := range texts { - est := estimateTokens(t) + est := count(t) // Close the current batch when adding this text would exceed // either limit (and the batch already has something to send). if len(current) > 0 && (len(current) >= maxInputs || currentTokens+est > maxTokens) { @@ -586,9 +666,12 @@ func planBatches(texts []string, maxInputs, maxTokens int) [][]string { return batches } -// estimateTokens returns a conservative upper bound on the token cost -// of one text, in Voyage's tokenizer. Uses byte-length divided by a -// chars-per-token heuristic; see bytesPerToken doc for rationale. +// estimateTokens is the FALLBACK used only when no tokenizer.json is +// loaded. Measured against Voyage's own usage.total_tokens on 20k real +// chunks it overestimates by 1.94x on average — which wastes round-trips +// — while still undercounting 0.5% of chunks, worst case -41%. That is +// the wrong error in both directions, and it is why loading the real +// tokenizer is worth the 7 MB: see Provider.countTokens. func estimateTokens(s string) int { return len(s) / bytesPerToken } @@ -784,3 +867,90 @@ func dequantize(raw json.RawMessage, dtype string) ([]float32, error) { func (p *Provider) apiKey() (string, bool) { return provider.ResolveAPIKey(p.secrets, p.cfg.APIKeyEnv) } + +// ---------- tokenizer.Budget ---------- +// +// Implemented on Provider so the chunker can be handed the live provider and +// stay ignorant of which model is active: only the provider knows whether +// tokens come from a real BPE table, from llama-server's /tokenize, or from a +// byte guess. + +// exactTokensPerBatch is the per-POST cap once counts are exact. +// +// The 80K default exists to survive the byte heuristic's ~43% undercount +// against Voyage's 120K hard limit. With the real tokenizer the count is the +// count — measured against usage.total_tokens it is never below what Voyage +// bills — so the headroom collapses to a margin for Voyage-side accounting +// drift rather than for our own error. +const exactTokensPerBatch = 115_000 + +// modelContextTokens is the per-input context window, per model. It is a table +// rather than a constant because the factory's own enum offers voyage-code-2, +// whose window is 16K — half of what the rest of the list takes. Treating that +// as 32K would let the chunker build inputs the model cannot read, and with +// truncation enabled Voyage would silently drop the tail. +// +// Unknown models fall back to the conservative 16K: undershooting costs an +// unnecessary split, overshooting costs silent data loss. +var modelContextTokens = map[string]int{ + "voyage-code-3": 32_000, + "voyage-3-large": 32_000, + "voyage-3": 32_000, + "voyage-3-lite": 32_000, + "voyage-code-2": 16_000, +} + +const fallbackContextTokens = 16_000 + +// maxTokensPerBatch is the provider-level cap: an explicit operator override +// wins, then the exact-counting cap, then the conservative byte-heuristic one. +func (p *Provider) maxTokensPerBatch() int { + if p.cfg.MaxTokensPerRequest > 0 { + return p.cfg.maxTokensPerBatch() + } + if p.counter != nil { + return exactTokensPerBatch + } + return p.cfg.maxTokensPerBatch() +} + +// MaxInputTokens reports the model's context window for a single input. +func (p *Provider) MaxInputTokens() int { + if n, ok := modelContextTokens[p.cfg.Model]; ok { + return n + } + return fallbackContextTokens +} + +// ExactCounts reports whether CountTokens/SplitPoints are exact rather than +// estimated. False means no tokenizer.json was loaded. +func (p *Provider) ExactCounts() bool { return p.counter != nil } + +// CountTokens returns the token cost of s. Allocation-free on the exact path; +// this is the hot one — it runs for every chunk that gets embedded. +func (p *Provider) CountTokens(s string) int { + if p.counter != nil { + return p.counter.Count(s) + } + return estimateTokens(s) +} + +// SplitPoints returns byte offsets at which s must be cut so no piece exceeds +// budget tokens, and s's total token count. +// +// Exact when a tokenizer is loaded: cuts land on pre-token boundaries, where +// BPE merges never reach across, so the pieces provably add up to the whole. +// Without a tokenizer it degrades to rune-aligned byte windows — the old +// behaviour, kept only so a caller that ignores ExactCounts still gets +// something it can send. Check ExactCounts before trusting these. +func (p *Provider) SplitPoints(s string, budget int) ([]int, int) { + if p.counter == nil { + // No tokenizer: there are no token boundaries to report. Returning + // byte windows here would be the old behaviour wearing the new + // interface's clothes, and callers check ExactCounts() precisely so + // they can avoid it. splitForInput still byte-windows internally + // where that is genuinely all we have. + return nil, estimateTokens(s) + } + return p.counter.SplitPoints(s, budget) +} diff --git a/server/internal/embeddings/provider/voyage/voyage_test.go b/server/internal/embeddings/provider/voyage/voyage_test.go index 542e0a3c..4eb4dadf 100644 --- a/server/internal/embeddings/provider/voyage/voyage_test.go +++ b/server/internal/embeddings/provider/voyage/voyage_test.go @@ -5,9 +5,12 @@ import ( "encoding/base64" "encoding/json" "fmt" + "github.com/dvcdsys/code-index/server/internal/tokenizer" + "github.com/dvcdsys/code-index/server/internal/tokenizer/bpecount" "io" "net/http" "net/http/httptest" + "os" "strings" "sync/atomic" "testing" @@ -187,7 +190,7 @@ func TestPlanBatches_SplitsByTokenBudget(t *testing.T) { small := "tiny" texts := []string{big, small, small, small, small, small} - batches := planBatches(texts, defaultMaxBatchSize, defaultMaxTokensPerBatch) + batches := planBatches(texts, defaultMaxBatchSize, defaultMaxTokensPerBatch, nil) if len(batches) < 2 { t.Fatalf("expected at least 2 batches, got %d", len(batches)) } @@ -213,7 +216,7 @@ func TestPlanBatches_RespectsCountCap(t *testing.T) { for i := range texts { texts[i] = "chunk" } - batches := planBatches(texts, defaultMaxBatchSize, defaultMaxTokensPerBatch) + batches := planBatches(texts, defaultMaxBatchSize, defaultMaxTokensPerBatch, nil) if len(batches) != 2 { t.Fatalf("expected 2 batches (128 + 72), got %d", len(batches)) } @@ -652,3 +655,78 @@ func TestInt8Dequantize_Base64(t *testing.T) { t.Errorf("base64 int8 dequantized values out of range: %v", v) } } + +// TestProviderSatisfiesBudget pins the provider to the interface the chunker +// consumes. A compile-time assertion rather than a runtime test: the whole +// point of the interface is that the chunker never imports this package. +func TestProviderSatisfiesBudget(t *testing.T) { + var _ tokenizer.Budget = (*Provider)(nil) +} + +// TestFallbackWithoutTokenizer covers the degraded path: no tokenizer.json +// means estimates, the conservative batch cap, and ExactCounts()==false so a +// caller can widen its margins instead of trusting the number. +func TestFallbackWithoutTokenizer(t *testing.T) { + p := &Provider{cfg: Config{Model: "voyage-code-3"}} + if p.ExactCounts() { + t.Error("ExactCounts must be false without a tokenizer") + } + if got := p.maxTokensPerBatch(); got != defaultMaxTokensPerBatch { + t.Errorf("batch cap = %d, want the conservative %d", got, defaultMaxTokensPerBatch) + } + if got, want := p.CountTokens("hello world"), len("hello world")/bytesPerToken; got != want { + t.Errorf("CountTokens = %d, want the byte estimate %d", got, want) + } +} + +// TestOperatorOverrideWinsOverExactCap — an explicit MaxTokensPerRequest is +// the operator's call and must not be silently raised by exact counting. +func TestOperatorOverrideWinsOverExactCap(t *testing.T) { + p := &Provider{cfg: Config{Model: "voyage-code-3", MaxTokensPerRequest: 42_000}} + if got := p.maxTokensPerBatch(); got != 42_000 { + t.Errorf("batch cap = %d, want the operator's 42000", got) + } +} + +// TestSplitForInputUsesTokenBoundaries exercises the provider-level split with +// a tokenizer loaded — the branch that keeps a large-but-legal chunk out of the +// byte-window-and-average path. It needs the real tokenizer.json, so it skips +// on a clean checkout like the other fixture-backed tests. +func TestSplitForInputUsesTokenBoundaries(t *testing.T) { + const tokPath = "../../../../../loadtests/bench/voyage-code-3.tokenizer.json" + if _, err := os.Stat(tokPath); err != nil { + t.Skip("tokenizer.json not present") + } + c, err := bpecount.Load(tokPath) + if err != nil { + t.Fatalf("load tokenizer: %v", err) + } + p := &Provider{cfg: Config{Model: "voyage-code-3"}, counter: c} + + // Comfortably over the old 30 KB byte cap, comfortably under the model's + // 32K-token window: byte-windowing would split and average this, token + // counting must pass it through whole. + big := strings.Repeat("func handler(w http.ResponseWriter) { defer r.Body.Close() }\n", 700) + if n := p.CountTokens(big); n >= p.MaxInputTokens() { + t.Fatalf("fixture is %d tokens, needs to be under %d", n, p.MaxInputTokens()) + } + if got := p.splitForInput(big, 30_000); len(got) != 1 { + t.Errorf("input of %d bytes / %d tokens split into %d windows; a token-sized "+ + "input must pass through whole", len(big), p.CountTokens(big), len(got)) + } + + // Past the window: must split, and every piece must fit. + huge := strings.Repeat("x := compute(alpha, beta, gamma) // annotate the result\n", 40_000) + pieces := p.splitForInput(huge, 30_000) + if len(pieces) < 2 { + t.Fatalf("input of %d tokens was not split", p.CountTokens(huge)) + } + for i, piece := range pieces { + if n := p.CountTokens(piece); n > p.MaxInputTokens() { + t.Errorf("piece %d is %d tokens, over the %d-token window", i, n, p.MaxInputTokens()) + } + } + if strings.Join(pieces, "") != huge { + t.Error("pieces do not reconstruct the input") + } +} diff --git a/server/internal/embeddings/service.go b/server/internal/embeddings/service.go index cbb12c70..b1b18c3d 100644 --- a/server/internal/embeddings/service.go +++ b/server/internal/embeddings/service.go @@ -20,6 +20,7 @@ import ( // provider purely by kind string — these imports are the wiring. _ "github.com/dvcdsys/code-index/server/internal/embeddings/provider/openai" _ "github.com/dvcdsys/code-index/server/internal/embeddings/provider/voyage" + "github.com/dvcdsys/code-index/server/internal/tokenizer" ) // Service is the public embeddings API used by handlers and the indexer. @@ -457,6 +458,31 @@ func (s *Service) Status() provider.Status { return st } +// TokenBudget returns the active provider as a token budget when it can +// count tokens, and nil otherwise. The chunker uses it to size chunks in the +// model's own unit; nil keeps it on the byte heuristic. +// +// Snapshotted under the read lock like CurrentKind, because a provider swap +// mid-file would otherwise mix two models' limits inside one chunk set. +func (s *Service) TokenBudget() tokenizer.Budget { + // A typed-nil *Service still satisfies the capability interface the + // indexer asserts on, so the guard is not decoration: without it the + // first indexed file panics inside RLock. + if s == nil || s.disabled { + return nil + } + s.mu.RLock() + defer s.mu.RUnlock() + if s.current == nil { + return nil + } + b, ok := s.current.(tokenizer.Budget) + if !ok { + return nil + } + return b +} + // CurrentKind reports the kind of the active provider, or "" when // disabled / not yet built. Used by /status and admin endpoints. func (s *Service) CurrentKind() string { diff --git a/server/internal/httpapi/openapi/openapi.gen.go b/server/internal/httpapi/openapi/openapi.gen.go index 639d9e72..c8d8a047 100644 --- a/server/internal/httpapi/openapi/openapi.gen.go +++ b/server/internal/httpapi/openapi/openapi.gen.go @@ -1528,7 +1528,12 @@ type DiskUsage struct { FsTotalBytes *int64 `json:"fs_total_bytes,omitempty"` Id DiskUsageId `json:"id"` Label string `json:"label"` - Path string `json:"path"` + + // Partial True when used_bytes undercounts because some entries inside the + // tree were unreadable and skipped. Absent means the sum is + // complete. + Partial *bool `json:"partial,omitempty"` + Path string `json:"path"` // UsedBytes Size of the tree. Absent when it could not be walked, which keeps // "unreadable" distinguishable from "empty". The SQLite entry @@ -3267,6 +3272,30 @@ type WorkspaceSearchResponse struct { // no chunks returned but at least one repo errored out during // the fan-out (see `failed_repos`). Status WorkspaceSearchResponseStatus `json:"status"` + + // Timings Where this query spent its time, in milliseconds. Returned only + // when the request passes `timings=true` AND the query actually ran + // a search — a workspace with no queryable project reports nothing + // rather than a block of zeroes that would read as "instant". + // + // The fan-out phases report a sum AND a max, and both are needed: the + // sum is how much work the query did across every project, the max is + // how long it waited for the slowest one. With perfect parallelism the + // wall time is the max; with none it is the sum; in practice it is + // between them, and one number alone cannot say which. + // + // `projects_scanned` versus `projects_returned` is the ratio that says + // how much of the work was discarded: the fan-out runs dense and BM25 + // over every project in the workspace and then thresholds the answer + // down to the relevant ones. `projects_in_panel` is a separate, + // smaller question — how many of those the caller was actually shown, + // after the `top_projects` cap. + // + // The named phases do not sum to `wall_ms`. The remainder is the + // workspace visibility check, assembling the projects panel, the + // round-robin interleave and writing the response — all in memory and, + // on the load-test fixture, ~19 ms of ~9,900 ms. + Timings *WorkspaceSearchTimings `json:"timings,omitempty"` } // WorkspaceSearchResponseStatus `ok` — results follow. `empty` — workspace queried fine but @@ -3280,6 +3309,84 @@ type WorkspaceSearchStaleFTSRepo struct { ProjectPath string `json:"project_path"` } +// WorkspaceSearchTimings Where this query spent its time, in milliseconds. Returned only +// when the request passes `timings=true` AND the query actually ran +// a search — a workspace with no queryable project reports nothing +// rather than a block of zeroes that would read as "instant". +// +// The fan-out phases report a sum AND a max, and both are needed: the +// sum is how much work the query did across every project, the max is +// how long it waited for the slowest one. With perfect parallelism the +// wall time is the max; with none it is the sum; in practice it is +// between them, and one number alone cannot say which. +// +// `projects_scanned` versus `projects_returned` is the ratio that says +// how much of the work was discarded: the fan-out runs dense and BM25 +// over every project in the workspace and then thresholds the answer +// down to the relevant ones. `projects_in_panel` is a separate, +// smaller question — how many of those the caller was actually shown, +// after the `top_projects` cap. +// +// The named phases do not sum to `wall_ms`. The remainder is the +// workspace visibility check, assembling the projects panel, the +// round-robin interleave and writing the response — all in memory and, +// on the load-test fixture, ~19 ms of ~9,900 ms. +type WorkspaceSearchTimings struct { + // Bm25Ms The workspace's BM25 search. One FTS5 statement covering every + // project, partitioned per project by a window function — not a + // sum over projects, which is why it has no matching `_max` + // field. `MATCH` is evaluated over the whole server's index + // whatever the scope, so asking once per project repeated the + // same global work N times and the N queries contended over one + // index on top of that. + Bm25Ms *int `json:"bm25_ms,omitempty"` + + // DenseMaxMs The slowest single project's dense search. May belong to a + // project whose query failed; the fan-out logs a warning of its + // own for those. + DenseMaxMs *int `json:"dense_max_ms,omitempty"` + + // DenseSumMs Vector-store search summed across projects, including hydration + // of each project's winning rows, and including projects whose + // query failed — the time was spent either way, and omitting it + // would put the sums permanently below the wall time they explain. + DenseSumMs *int `json:"dense_sum_ms,omitempty"` + + // EmbedMs Round-trip to the embedding provider for the query text. + EmbedMs *int `json:"embed_ms,omitempty"` + + // FanoutMs Wall time of the parallel per-project phase. + FanoutMs *int `json:"fanout_ms,omitempty"` + + // FuseMs Normalisation, candidacy blending and thresholding. + FuseMs *int `json:"fuse_ms,omitempty"` + + // ProjectsInPanel Projects present in the response's `projects` array, i.e. + // `min(projects_returned, top_projects)`. + ProjectsInPanel *int `json:"projects_in_panel,omitempty"` + + // ProjectsReturned Projects that survived the relevance threshold — NOT the number + // the caller was shown. Capping this at `top_projects` would peg + // the scanned:returned ratio to a request parameter instead of + // measuring how much of the fan-out's work was discarded. + ProjectsReturned *int `json:"projects_returned,omitempty"` + + // ProjectsScanned Projects the fan-out searched. + ProjectsScanned *int `json:"projects_scanned,omitempty"` + + // ResolveMs Loading the workspace's project memberships and applying the + // per-user access filter. Separate from the rest because it is the + // one pre-fan-out step that grows with how many projects the + // caller can see, rather than with the workspace. + ResolveMs *int `json:"resolve_ms,omitempty"` + + // StaleFtsMs The pre-fan-out probe for repos with no BM25 mirror. + StaleFtsMs *int `json:"stale_fts_ms,omitempty"` + + // WallMs The whole handler, from its first line to its last. + WallMs *int `json:"wall_ms,omitempty"` +} + // ProjectHash defines model for ProjectHash. type ProjectHash = string @@ -3411,6 +3518,15 @@ type WorkspaceSearchParams struct { // recall (e.g. "authentication and authorization" across a // mixed-domain workspace). MinScore *float32 `form:"min_score,omitempty" json:"min_score,omitempty"` + + // Timings Attach a per-phase breakdown of where the query spent its + // time (see WorkspaceSearchTimings). Diagnostic, not API + // surface: it exists so a slow workspace query can be taken + // apart, and it is off unless asked for. The server logs the + // same breakdown by itself whenever a query is slow, so + // catching a regression does not depend on someone having + // passed this flag at the right moment. + Timings *bool `form:"timings,omitempty" json:"timings,omitempty"` } // SetAutoVacuumModeJSONRequestBody defines body for SetAutoVacuumMode for application/json ContentType. @@ -7449,6 +7565,19 @@ func (siw *ServerInterfaceWrapper) WorkspaceSearch(w http.ResponseWriter, r *htt return } + // ------------- Optional query parameter "timings" ------------- + + err = runtime.BindQueryParameterWithOptions("form", true, false, "timings", r.URL.Query(), ¶ms.Timings, runtime.BindQueryParameterOptions{Type: "boolean", Format: ""}) + if err != nil { + var requiredError *runtime.RequiredParameterError + if errors.As(err, &requiredError) { + siw.ErrorHandlerFunc(w, r, &RequiredParamError{ParamName: "timings"}) + } else { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "timings", Err: err}) + } + return + } + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { siw.Handler.WorkspaceSearch(w, r, id, params) })) @@ -8042,694 +8171,720 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl // const string: with thousands of chunks the chained `+` fold is several // times slower for the Go compiler than parsing a slice literal. var swaggerSpec = []string{ - "7P3dkts4tieKvwr+mvmHM92S0nZV93Sno+JMOm1X5W5/5GTa3b2jWSNCJCShkgLYAJiyusITc7UfYMeO", - "OM8xz3Du5yH2k5xYawEgKZH6SLtq9zlxrqqcIglgYWFhff7Wz4NML0uthHJ2cP7zoOSGL4UTBv91bfRP", - "InM/cLuAf+bCZkaWTmo1OB+8lsY69vR3bCE+sWzBjWV6xtLbHy6eniy0dZOSu8VpOma3QiQqlcoJo3hx", - "VtJH7Rg+e83dIh0najAcSPgovDMYDhRfivpfRvytkkbkg3NnKjEc2GwhlhxmJD7xZVnAo7+d/pf8WfYH", - "8ZR/M/v9k2+fDYbwNgw5OB/897/y0ezJ6A8//vz0d5//82A4cOsSXrLOSDUffP78GQaxpVZW4MJf8PxG", - "/K0S1sG/Mq2cUPi/vCwLmXEgwdlPFujwc2M6/9mI2eB88J/OaqKe0a/27JUx2tBQbTpeqXteyJwZGpCd", - "LKW1Us3ZTIoit0NWqTulV4rdSZUP2ZTnLNNqJueng8/DwaVWs0Jmv8I8b4TVlckE44URPF8z8UlaZ9mJ", - "GM/HTCy5LJjjd0LhvF5rM5V5LtQvP7GLyi2EcvBVAQSqHCt4dmeZWwgWeIcZXQiY2JXKxSdhPip+z2XB", - "p8A9v/wW5+ITbKkV5l5mgint/CZWwNc4LVvNZjKTQrlbpw2f/wrzeqcdE0pX8wWbGSGYLXkmmNNsoYsc", - "yQdf45kTwHPlmvFCq7mVuYAfE6WNnEvFizFLX3LHp9yKW8edGAeqT3Jp7ybTtRM2ZVzlLIVxmn9NVMaN", - "WeNgqlpOhbEgDpAiJDBo9r84LT6qBVd5IXLcJGGYoCeHQKXXulL5r3jEgD9mOObnYfyr/VV5tj7uWaYr", - "5YB9pcWZrfBAacXcQtpArhOlPUszqYg9iKCG5aIUKhcqk8Kyf/+f/8YWvCyFsvBgyY2TvBg5EH24ok/O", - "nnoW+Kh45RbayL+LX4H6b73c1YZJL5Mvrq/YnVjTXEqjM2Htr0P+t7yYabMU9b0w1fka5hauhyjZ6J6A", - "Of5Zmzs8w/alxHn+KjzrpxFkm2eSWrxtcUqml0utijXjKlFCZWaNHxvdiTWbamB9LovKCFYacS+I9ebS", - "LarpxOk7YJyZ0ctErSRc30MgCm8z0ktRIncprUal0XmVwQAt/rpciOwOxY6fVqHnFmWUEdZx45AHPwdt", - "A9WCi8zJe/FqORV5LtX82uh7mQsUTqXRpTBOkv5Ai0eK57mEsXlx3XiC9Jg2Ia+FsdKCqC39d8N5mhZ6", - "Oma3C14Kds8NnKLpGtWB58ChiboTa8u4Eezd+w/MOg1Eh3OGRBbqfnTPDQOdCp8idcurQHoKyhgwj8y3", - "dbw0LHF89fLkNGUzqebClEYqN2R476f3es3n4pz+M8p0LkbfnD998uzb81mhuQPl7i132UJYlopAuclS", - "56JIgTPOrOOusq1JBb1sOIBFoqKnquXg/K8DXRR8yQfDgS6F4nIwHNDAgx+3lbqm4vhX+hKu8seOxV/k", - "+ffS3YhSN/S+9p5ODVcZ6sFLqd4INXeLwfnTjjl7Vq1MsU3QhXOlPT87o2fGmV6e6ZUS5syIUrOPN2/G", - "XVQodVFMUIG+58XEikyr3G5//H1JnMZKYUb4QXiRZRxkr4Dz4F9lJ0/O9FI6YLZ//5d/DScgFzNeFe60", - "MQcYdC5MmARsnVBRsrSHf4U/MP8cs2uVjZkXD5atxHSh9R3u/HePci+fHiXqxP/C/vL+Jrx8+pxptxBm", - "Ja2IahwcbGmZEbBpImffPnvW4pqp1oXgeHGgmJh0cXSkkczBXOHhuHwv3Q/VlF1ffGAntWTVhpVG3nMH", - "Myi1Pe3cnubSaESk4+B8sOSq4sVgGPk3/oFXTg+Gg0CH/fzb4Kph4MU+Tja6Kt/CYTO93FxZYTyBdo8b", - "Huwcq5R/FOsO8WcE6OITjgPDPQb/N8i5EyMnl6KLiJ1zGQ4Kbt2ksrs/pqrCa0UkWHd8RZbwlSNeqPhB", - "L5DF2rEAPN6TfnIPB6URM/lpm1VfSlsWfD1CKU4PAcvCcZhVRQGKiTe+0kx+mvCn02fZN/m3Kdxub7Sa", - "B9XeaWZEpucKDpNUrACzbcjsQpuo/rsFd0w60MYV3N7wgrLOVJnDAaOm3y2mjbjXd6K5vMZh9D9+wQZu", - "sKQEQd6mq9+ASMxhkwfr+fUz8SU9vs3LvJSTO2LyXfqRPwqfhwPYm/BGe0M/LAQrCy5RCcHtu+dFJcbs", - "8eMb4SqjRM7EJ565Ys20ysT48WMGtqDAnbEiq4wo1nizg3D0qhZb8TXtsTNS3MPDrOBOmM692iBlWF1j", - "2v00eiOtu/Fukl5C4f9LJ5b2cJL58bgxnP6tHS8azBRvoe7Z20F4pXPuldN/4llVLXuFYRDcQUorrQR6", - "pDIjlkK1v9xDSfxG1/gvtHbWGV7eoqLTT0AlRG4n0/B4B/+YSrDVQqB5xYD1LXN450rLxLJ063HHbbgx", - "z81RuqZ8ueBqLq65tStt8l6yZZUxQrlJ6R88QDdSYtV6fNMCU3JZLdnv0Z/IMyeMHbN3mlVlKQybgkUM", - "S2wM8vt9+7I1yY1JdK4fKHfJnZhrs74RFi/zzdXnohAgYNA69kuH2Q/On3SpT2DTHPd0ZcThhwmn/L5y", - "mV6KriNFd8+uL9yIrOByGZZ9lZPsxj+KnNw1LQEulfvdt7QbO1Zi72RZkmD9Kgvx36sJuWmOjtB0RxuX", - "ccdomxhcNMxymYN4XHEUnIUGa4ZZPhPjPevouoHaDLA5s40d3yZlL+OFxW9xnL9UOm5f7m35PSYQCnr/", - "dO/wvcedK16srbR9ekxGnCOP4NpOnltKdUUvP93c/k3535hRa/wdi+s+zA+Ye5eQ6ODXvDLIixP64p5j", - "L5W0iyM1569wRh03x+nrGxvR+EB7Ee31b891/66hXkaqQi9nBuW7LQp+qJZcjWZGCpUXa1bwqShA610p", - "76FkObeLqeYmH7MPDa06UaiXwa06F0oYUAy9jTxC5zd5ibo0NlS5dt6Bm9cxTL1/4d+j1fcBzNlfcPX7", - "5jwc2EyX4dorjchIV+5yY13NFRjURFHvWFB6xXJh5L0A850XjD6HbjxveT+yifrL6P1F5RajW/o1ROTY", - "QvAcrv81yzj5Fr5/9YGdgQLEVtItyNtsq7IspMgZGv9DZjWqSKP4dxyULaRy5CyLN0CiwNipCgfT/qMo", - "HRr+U57drbjJLQVBnJzKQro1jaiLHN8rJMgEMp+sk0XBrFBwx/iYZhAkWwTdVnnvKFa2y2S4vvjQoqv3", - "nVq802BaF69uR99fvmVTMdNGJKokn6JU8+fkgpUUFUOTsuVYxhUI+GjGDZzGRLnW2GSqPIy/w/J28LnR", - "VdnL4S2a/NxvfH/Fg+dD371TihHu7j1DLp/JQti1dWLJ4Ek2FeS3n0vrhBE5O5kKuOkty8nUp5B5p49p", - "ybOFVKLTp3UtzMj/zj5+vHrJIstPKbB2+eaKneBh+x9n40x+Oqu/djpmf14IlajSCCsUWfs+RA/c8ub9", - "5cUbFHgS+CwXysEhAOMVrE++FBhwyBNV6IwX5z/Xn/58/nOk0mc4juhs50tB1NCK5XI2E6CdJ8q/Zs/I", - "rMm1CGGEopC5GLP3S0nnUnyiuCB55HocEmEWKPa2Kabt+AdtHUz/5DQ4VWSI0gZagqHtdwZPzHjvPVhz", - "RT9nfbQ73HIYRm9dwvSXLoeZkk7yYoc59V7R9c3CIxRlFSsUjGxZWQeGlpqDQGAzzOco9FyqcaKAiXm+", - "lIrZBTfCkvjQlRvp2WjKVb4lCn7fpZroomVY4xcHQ3Qq7jepw9K3Vuo/3E/jGAg7VKT0OIlnRogRbAVr", - "PNB5Pr+qCGoF0zsU8crpyT26NLoMIJ6zQt4Lul3poqfPoUAaMoViHn/1zm8rHNwRFi7NREk8+Jk2BmQA", - "WFFqHW4cI+bc5IWwmOyz0Cu4eubasYUIgaUdThTyMnVs/HAwLXR2J/JJbcu0l/XnxTokI2Akj9yUqHcy", - "I+cLh0oGnFgOCs5oVuAfs0IrwbRJFJ5u9pOeDpls5FrAAb+jEKJiQGRvOPocF1MpJdV8nKh3oByi80U6", - "GJ7FIOEB/mdhnVyiO7I3ePMqPEKZFnBuh0x8yooqB5lEQRAak71EXSqPO5youMVW/p30U86WgluMv7qF", - "0dV8UVaOUUgWlSPaZq4SpU0uDJzrJZ8r6apcsHkFeq7hbiEM6AaKcbgVltLWOsBOO6YQD7ZGNlJEHviF", - "QqIonncc98E1/JlUHlAEeQF3F5B+WjnKfVEaTw0c9SPHjjyEJ7Uo3s8G53/dbUm+RR1LcZWJ9/Htzz8O", - "u/QK4kfMA9AW/fTAxHHQIZMzOK/jLa78PBwANWq/yZHrwpeBu/ZbsUEt6rNUYVL13m5EnNt7l7L/6/9k", - "aRw7xRO+4taBHNOOuBLEJhzaEByRILrh8QdsX3OKpTCZT5xY8k/00tMnTzo/QblLg4aE32DhDSmt9ZLx", - "pkRDv+y5T7oqUbtaGemcUH1ZV3iKp9otKBOPcdewQY9c9r0wuc8ijKH2O7LX9XIpVC7g1q3MHOjRJb/9", - "B3rl93sFVguweSaY+IR2RZBr/t0heuIolKHA4ON2JG13xJUXfZS9BQHoOSEdrXiRMiBdxs2YveGGdB3Q", - "ZdEStjUXTQuxBPaKF+BCZHelBiLlDK7cJXcSDFCKwICooAdxyxaCl2BMuYVU80RRzA04Kd4ZNNYDdmdD", - "a/BJsQ0R26RH86C2TvyWVOw6jt38P2xpHDWzbO36tuzuPg5dF2KnHiRmqPtpdAh2aMsqnxRSia5QkadQ", - "rywKmSUdoV81r3z65faPvaP1BoBLjvGH3t+tnCvuKiP2O3C9Me1TWer1+XkNa4I0lrGbsL0K8kOpJ5fS", - "tZIgnj7Z635cL6e6OFZ79m/tW15ftM2gx/Zwd+8GL+6KWh5xmMMsdgUwX0rzSjmz7tmjAyNKPVu5Q7jQ", - "h3tmJDKnDUaE4TPb5pU03Y6RglLc8vAFdgJm/MiIgjt5L55THJN9x4zWrtsVIpQ7ylH/wQhBBOzaNFOp", - "LET+d4VbYVQwQSpF8aOMl6XIDwi4AinqSTdH7CatvftovfjZkHaYb9+dYzGzExS9/Rpzh3y0E2S6o16S", - "rXQ8+7dCOpA22cJozMtDp8xgOJjPq1mnohA9Mh2CskfYYB7OAde9M0KM2cUU/VjRVNMV6vOOTQVb8eJO", - "5EO2Wshswe6EKG2ikkGlwNqDOy8ZgEEIPF1Ju8DYOhpZyQD5MhmQ//X2v72RzvMEmJZgpQnSBEDlQANs", - "ZBfLoHrYnmv/kDAjEWwYjqbngi7e2UpMvVIz3RFGfHhOZbP25vDM1ktMY73FNxm37J9u378j5xo+NvWU", - "Qz8LJTOTohvzYHmWidLZkAMrLUt/pgfP2V9/hltxSAGOIRXJJCpQcRiSGocMljts+nE+//g5HbMfuMkz", - "nYuc3QieuUTBNCyTGMZAr9dzJt0jC0qrtj4pMPogndYF+Qe6UmqtyIxwE6Huuwz+Vl4uutfigoEdLY6U", - "GYE+V17YIfn4eaJmBZ8zJygWsloItNIFzxaot1LCQ7FmVjjKvQ4Bg3GiPtraLRwDQA2dG/7uM8wpH5sr", - "JUyiKKLALL8XG6GNnVnjmxx5ixR5pe63ZXF3vq7ntzYtD2J+uJu2mT+Q+PDro/tU7Zt+Pc5Bk63pcmB4", - "r8k9IQvw8uovkz+9/+eL719NLq6vJn989c9p9w1qhdt/3d0z+D5yJUUGTkCqKa1GKApPN1jrgMwj0l5h", - "8E6ahDqfTUet8y7x3ZqLf67ry69lIS7rIoitVP3wQ4emUdsXbWK94dYx+KmOA588HU05nC68Dqy8Fz2J", - "1Ls166b5sZEPKhwlPodH0G2pqqJgcsYqlfvfx4d4JNGT2LM4qux84OpIpYCXO2TeB/iRvkyaVHTZFIIC", - "Qvau57OH6Gm6cmXVVM9A0nnPNVjEZzgymij2AIZtGlgNerXMrOZym7McRsbq40gf8uzOR5kK6yY202QV", - "Rr0BSyoGHU6nw3mqI1SG1RkHy0OYO1Z07JWBTfo1FlQP2Uca+vz2UV1U6m5Cb3TlHx14kjtsdwFGyWQh", - "j7AH3+E7P8jOpJ8jdq59EPuM4z73QVceTgeXBtKEmQ2btOzbhWu+LjTPd4rMjRrKD69Hv2dOfHJj9kIq", - "btYUA2d20dTCbTWl4pPOy8l/fbLorDq//eFi9Oy3VHSey7mwKENS/1La+cWd7N97aA7xdXdbzzW1W2vx", - "n+wj943g/am2QuU7LqEh25LOzYC5wlIX2IlWmuPTvsupK6jYsNF9+L4huVF95AflEQGD7rpxdi+F4n0H", - "LqZDGu0gPsjhbi/jP8B9vUOw7vTzwdJuBTfZopezth12z/Y67P5WCdNR33BbTWnCjAR8zvicS2UdS+OM", - "0/GRCUQ01r7FfS0v3wYv/IpevtfaZOLW6bJ/MRlXmSiK3ToQV4xjmSqTWPuaCWsptYVZYa3UCvUjrC5n", - "XOVY8kOfHbPXvLD+O0pjKAMfjpkxJ3Dkf9LT0d8qUYlEZaA3VaVPfTNcoVlvhWDpT3pqJ/C7ETmWJHUW", - "6jWf2l7VZdARS6HAWjoL4U8M60+wxPE3NDv6B3wO47qJWgkjfPJ3HShnOG8UJWhww0utqTUdphRY3Mcx", - "PlFsu+4gbtbGKrs239eedlAA8xd+E6on2VI4nnPHcQlc1Y6Ik7l0IyRLfhpiouNEvfLJqU/Pn8ZUSTqd", - "QMYAxcKMXj1nmMBV/23B70WilGZ+cvAQ0aojG8XPr+OOEnOerRkvJCeHRtqslmTffccS/EIySMedHFKX", - "3W5rCg8oM2wX53bX/YlgiR5WJmgXB5YIik9ugqW8vOMKvJhaXVShUiFyJ4YNxSfHcs+3HGtkxyzmoyQq", - "VNxKTCTEetIxexuyPyLvez0A/hfm7YWCqdSGc/KoykqQ6T2q2tPfjUBLu/3h4mmjfNHzF14GQ1bRTc8+", - "3ryxX1L6fL2n4tnTa7vYOVEnl1d/mbx89fri45sPk+v3b95Mrt59eHXzp4s3p2N2Uaz42rKs4EswJ6sS", - "dB3UewqtjX/57dW7zRd3pQUdU1P9Z/THwNvka1mACKFFggDKq0IYNhNUX18zDSZtJSrQjeQ2L1BW4N3g", - "dBApdCqVViNK9/N1zol6W7kKo92YhASKGEmQ1vn9/33H6lruPiHf3PKOoi+PDhBxoWISJV4mGVdayYwX", - "iUoGnWXz/5VERDJgxDY9KaHNmvC9bF2V+dGiZbMM/MtrvluEa561YWc5+LAtizdmtFES21jhjhupvywW", - "RgqpzBOlXW8JQgitMEoS8FpK83UQAZbNQPfolAEbD+/SflrMCarLI3j5Ebt497LhrEyUrTJQjGZVgYnw", - "cR7wDF60yOpUmtDH1nPpUOvYpyGEy/0BOkW9heT+7qDx24tLRj+26og1yD+tGO05+w394V7yREUEtrOf", - "gZk+n/kxRlLN9Pjx4+7jEybSCWtxXU0LmRVr2OyMwmbX728/wJWDGTRkIRKVQSp7tAW6vnKNeqbXcKxw", - "VcnozBTrQ2qYA1EbO9Ke7hYVexh+UU0vsp66woswZw+HhJxyffGBUkGFoLgg5lXrVcxsggekTVR0o2KS", - "9ZDNdFHoFfknxb0wa6bNHMNc1kqg3r3kVOByps3c+nzsGK95ZBnPc7rwZoVeYV0PBs2owp+zW1GIzMU6", - "EMoeLbWVGHMvZXYnTEjJp+RBbXApuQFNXiqnGWe2FJmcySxRMD2w5ARHHcKIYo1piBQC4LOZLCQmINoR", - "n8+NmGM65b0U3SrjPXfc9Cthei47Usj8BuCv7ARJjdEPbZB6tqjm3eGO4DFsfy7BLPRk4K0B2iy8VZ6z", - "ZKDN3P+kzZwraWl17RRnTGMfwrP7ZTktyj/Vz4DdZsBFc/fuJfEIbRFlcF9ffBhvkdmrOJNahe4qVCn1", - "Ixu0IUaPPt8ID5ZGjGayKChO669bJRU628mqkLZdmY48ZhknfSTqoIW0rud+3lfkgwAF3cEw0gVCqdHm", - "iwu3LHp5zcO7dGVVbDpd4vjDTcrWn2mM1r/HH0Kt1z84akp/ylqjGrC5Dy+EdSMxm2njfLUd7je7vnlK", - "jApMwh3WH2DWJJbPhXIl+zxRCFwB4kVwCzqhLiv4EzFYswDQFw36KsBwzyQqGrl15RoqfsfV43XlYYTw", - "Ja29pU3t2erduBkEaXawh6rJQl8AneFH3eWRwgDV12HTndU2rzuLbNgrTATzDkVSGjGzZ3zEOejl4OP1", - "+x0s0VzOkVo2kPgq380gc3hoIvM2jxzHwfU3eqdxwCSO4FLknS/gTz/eXv4kmKsO+yTPj+TRI0rvjixr", - "Gx6PtjWMg+NYw3o9eyixexeX+MyR2+hJ/AWbGYbdtZs/CF64nZ787sKCW0xyKtYkIlJCE0wxR6xSC/zo", - "ujssSI9ulTzEt/brdP4LXctByN8XYi53ZFhXRdEKvKAFPOz3AK1kKSwVezS8twxmIbyqD8p8tD68v78n", - "02HnlHt3oVJ98B+kiaJ/Ih7BjgTBn/fcDoO3vCR9EUOL5Af6l39lIfCrZ3WG28hrvz7M6u+MREVNNJBo", - "wa0ve5wKocjzKXJ2og1LYRtQA0rRYVBya0V+2pnRtxnWIWJsLr2XHS4xJHBgfGePNlo/2zvca1kIuzO/", - "/7i4WMgHwJyRTx6i5bdPtsVCzSTHBPoiNWlm+5bVS8RFpe7sJKsdV/tLFe2EMkwPf94H1kQ+eUg8cGPM", - "4eak+0bZQRMl7WJHsTMhFcFhOkqLOHgvQyIUzTuXNtP3wVd3TKCURtu7zq+7+ZHM+1/YvjPgsCB1D74u", - "tofdYoBeAlwbPTfC2lf3nQk475VgCJkcMF7evcRka+uM4EsmPObrdM1S9M+doSQ8w/mk3h3XNMyEyi1L", - "L5BRz1kTPfrTSOU/Wa1ScnylOGpK6duJAgYwcikVdz65+54byZXzuK4hzZsbEW28nHGLlt89V67LazTl", - "LlvEEtTtvSEa7vqtyRjbzyA6sYdpOqASQoQtCJyAOQ4eCSnW0MC49T8JALn+d0418vQbRh2Hg4Xgxk0F", - "mg+0ZP8UPdClXs54Ww9rFobAp3GX+4vk2uLvCJG3/ehSWHt0ptUOpcLZB9pntDt7z1GokNhwZ/tfWUlX", - "HvE4pVWMQhaF5+iAl+P9uMjYz+kULSQoBjLjxWjGi2LKs7v4Fqqs4dV0g8LpMFH+b0jrdEjNE9pcnHYd", - "kmMlYIBHjOrAhjLWKDKnVD4CsfEa1JApsRLWkV/7uY+PfjNmb4SzjLOPV4myC73ygBTarLjJ2VJjxXNe", - "oWnPMQTtzX0dWgf0k+5YZCVR8NK2kRdqftLVtBB9CbXHXGQPuEsaG3xA+d6C25bNCZsi72HNw5130I7j", - "9Xnf6ei/aEv/xD69cfuwtS7Rjdp/mRe+xF/pmK2EavsZi2xQ1V0wxthXiHKTUp96RC/RWYXfI23SszQq", - "zelZSuiM6Vnqc4ro/YJbNzIVonu4yiOUpT7DqFI2bQcAYMKIcUJzaG3FsJUCRMMNcDdguC8yLv9JTzs8", - "Hs6JZekOQBmMc/wi7/DD/IB5VYqAjrx3iF3e7cOTdJb80+Rw4pR11vPhFW43fEVVbf5t4kUsVqPmJxYk", - "WwqjpWN204A3YNKrXDHa8pzlWj1yjFtbLQUjEO6qt21DSAM5biMOQHw8pIRkQxf2aXoNLm8fCH8IftwR", - "ozvA64qPDGttOu7txlZv0Gavx/6f9HS39+wnPT3cYoYz+gUuMxxrl7/sjVR3+0DqQv5Id34W6DQ+RyuN", - "qSUpdkWoXSQhlbABO5goI6wu7gXiDmK3qJCwgzhxygrjSOs/WQUgronMh1jRGRNaTjGhEL8b3DSISDal", - "vC3c3e8e+Xn43KIl/xRt0N+184h/d2gyDRKjk6J6LtUbnd3tlq0bkVn/S6PsUqqA1cJsIRG9aSVVrlfd", - "lU3R77yROKlXwowyTIXHR57HQjzUHTFqvS4FS2U5wQe6vZziUykNqPhdYMmvL7/55ps/EKhO8JnpIheI", - "JoMLYwiVpCvnQaVsoR3CotlxX87gthjvwDK/peZSV9cUFtbZHZOW3Yk15q50l3HUmeqbbJzxkuCcnEEM", - "5vjRnmKyzoSAVJZpANfHFipX1wybT2nleDGyKyFKKlsThp0suVrTxngtQSuRKOqZdTpu7ErrkydX10N6", - "6zR+CpMMVGy0taFhlKBf+G/tVxq8bMS3GoKQSNdihp0nYLccBMIeLgjrY/UF4pCG3CkP9S5f+xHBnYPR", - "7HvQCnciy/tZ9lEWA0h76PnRdtBnI/JUD9jAFos+o03ZdrjaUIh7Am+I7Akq/XCw4kbt9FDs9AsE02ZP", - "qwpgYJpAeKf+7p6lv2+Csh0BpkgZMoTjhCmFlGJbKVYIfu9dWxF7T6oxhRNSCjYkipelwMaoCjNhCG8X", - "5AFhCIZ6OOECRh8NcHF9FTGmeKIaKGHYsigOGFAK0dnHHU0RbwlDt/OQKjwsX1tm9bEgjXs1QMTkIPdU", - "l89sf/4zfSAKgY0S908l1U/ZBsZHpsv1MCTCBzfNlJtOqLf9EzjcikCHUlfTLZNT5qjhsoBZrmAnqMOh", - "yIctmD4wXMbsosR2iIi6wBOFYa6piDpD2N0A0mhLrmwT/9HfdAgFFwErS2EIuCLcNnVXO8SPQ9QMWxbS", - "MZ4ZbS1zK52EnoaskDMBh96Sp4ngcJ2AEUF/WfBiBh+oLKWEE4Y0JhZyx3KZe6zepUDY7DH7EAqhQ758", - "kwweC8d7yRB1nahnHTBqqAM/HGZiS8h13DSHQOvvZQJQfHbB7O9nuE3wFX+2a+Q1DAYE5Ll9snRnvnXj", - "x3h6SEll1onyIJiAgyAQe75SO9OihN9QuEojSm7Q+SIty43HBAQWmhssUAP74TlL4dSHx5DxYZHUGItU", - "6+csRVE4cXpiV7xMmVYE3h56jXJTV7x5z2MLU5UaVJuqdCLHcTi2n5S6IoHf5HQ4tMBN8bBFibzilk1R", - "kjtcBiVpZlpha0cVgvU4izr31iPaMiuXZbGGK8GIUSzM2XA2RaKhcY10oa4e9erB9F3xsgw/4RLpHyHi", - "ENxRjVXvcR5uwPwjKs1MgnEAuivpmmN2FSCl8RyTT9xUBMfPQ1wp4ypRThQF42BL2EWDDqhLc6BTITy1", - "UEIWYubYdE1iH2TWhvCxlbmX96InubTt79iqvIqAUaHqccEtuaUv2N+FQTBYwVZYxA6UZhz4Y1rNE9XC", - "yLUsGTQ/Ea6BZPDwGqu+MLkHyKGz1an/iB3Nuiq3mCyFW+iucgoRUn3rFOCAMuQ0UHrGM8GSQaHnunLJ", - "gJ14v+spQi0v4CqTjp34/lw+sb1uXPbIRkI7jVcUGJh6dtpmeP9RMGV8m7IuDq0z09qr+JMUqxH9SLKP", - "FwUmgCCaKnPaRyna66S0VZQxyQDLrWCK+JlkEBLnV9It0CT25WUMZcsIjM8QyEANJVGYnYqNaOkb9jmh", - "qltfMosirJAIHCok1g4wny21kKVNFPZ5O4n3JH6EXqDOAdRC49UHdkbfPz3i2uxN0vsyM2TY4q64Qd0s", - "utRm/bH7OvtewxHEcsglPjdmC8HLCcI10yUc4FeXgsPNMasKj3AdS3wTRYrQuUeazRzCCGgjMCZlQRmR", - "2FbPR8xhiKB6ORP6yCeqqYQWmudYt5iLT2Nm19bPBvvC2PAvuDUWcr5Apst4ZUMhhl/UQhe5RV9K7pU8", - "n8NF1SL+hloGDer9bVe8b5MiD0B2xk/A5fKFXwBm/7JP+LzvhzcyUtVyMs8e+qI2unIeiGVPkEDwu0nc", - "6Q5NTM4Xo5XPZbfYNQOtPGQiPLJz4UwFfD9mFyGcBTfmG6mqT6QfLHn2/hZvVOrjjPj70gob2DO4SxH6", - "rhD+VoK50Qco481WU+ukq5zHrovTPhCTcDjYsdBLr1BurbKJw4gNLBv2Lt6gXoYmKptrdoJrJUBGeHYq", - "FlLl2IjjkWWO27uJVDN9ileIR6pLBuqMJ4NhsLWdERxEcuiPjpF4IAlc4AevNR7mo5losxXI5sHsOCbb", - "Z6/7IDSnFZl8k2k75avORdEDHtul1f3wmup1rl76fk2N8vGMZ3BTRuDWBvIc1d9gT+/uCqzuyt9Y8o6x", - "Ay/mxvN5NesDBGuj83ydjUJlKqB81V/tJWc3riBSZ9ILe3t59ZfJ999/fD25vLj84dXk5dUNGRRgL1g4", - "GSIPmgNe7ljGErH2WPw6+w5UiZpGHsiju2UQzPZwP22DV/ZVE/gvDxur7iJXDZF1LJTXbriufzh0rXox", - "YXJd5Liui383iWH0kveUx183wJY87KzXYqLRCYoEaQsyltE/sonKdFFQa4Qxe/fxzZto4mDfAbgQDmxf", - "4id4xJnb7wjJtHJcKmF2rbsRSovPsxM9c0Ix8bcKAVjr8GO37HlQnkKjH9dePws8RHHMzq5fcOe2oVWG", - "pAdSCX/9UMRy0UrYcSOq6pXcZvOtRJ3UvbdYKUxsWhWHs3RbhhZ1HoINs7GAUXoMZaxytGuV9XZy2Fg9", - "9nJQZLtTe8FYgue7RcHX2JTnczDOaz5sjuT77UcUgMPYcvsLu1FNm6fDo59IhL9YcjcihwPdZ6jwU+qm", - "ORMBq9U/6f0bOSJ4GSbdKer92L/Dt/QohEMf1bSSRT5mV4repJ5SqMz5GEBOPRHbJmgyIGuYwdKSQaKQ", - "dlSTSxgdzsj5HLs/kwdrrbKAaY0oQx5PueDzmFCHGDI16gFZk+ThQD3tDAmPhpTF+PtaZX3ABzt638Ur", - "ffMU+ErxUGv+yAaG7S7no6j/BFgJsRy6wgNxX/ABfzxmEvGiEeRka9vxui24dYk6MeLUj+KFo1bMUBU9", - "d1jxjD6f3MiZNwRhKO/CSFQDOxnki6VvoAPoo7pTeqWSAdvwDeG3DuTtAOt2ZLo+pkYF6n2Jp/uofoO8", - "LeBqzDAK5hNtmiLurEBYqPExM/nqrfz2joxq7aRRu7eBdon9Rxca1N8A4C2MxUN+ggRBrwKSocbkImLg", - "OUsUjlA0fGM13E3wfWrDXv3lw6ubdxdvamyuE7fQVkQ88IB7ARMQ5jTIAuxtBQKDoH0Cxgl1rw04HSiO", - "EBeEI14DucwIaehAXt2BAUXAjk8JrzNbYCx0hpBQJ/W1Tdly1Mn+482bxkkek2oOTDM4H/z3v/LR7Mno", - "Dz/+/PR3n/9zD7A1NrI7EGDlNjwOr2Irgx6xdkMJT3nQwKLqFS9vFxPESRzVQiVKCi9aKLuP0OTRE8Xn", - "wLAzfRhuM03zq6piwGkHkwyf7a6EidVLjXxYz/87kwO+uBi7CYxUX05bOmZTqjZ4JRCgkVLYV8O9LWA3", - "pcQOxX93Uk042wfbaw2QoK+BURnH35Vks3luOnqUYCuMiT+0R95eS/5pQlU0x2Pvbo28+bld6wkHYMND", - "4rc51kbsdgpSBVxdhXTI00d9moxVeyRhmgMNN9a0MenNgXaRrFoueZdbqaUcfi215h/nhqEUhuZWHHRY", - "b/H53i5EtTDtqKssJ8HpdkzXo9g1qk8+/ONxap8Yj2K5Kb7bbL2TjbeJuLWPOzg99hLu8aMejzTRTIru", - "3PP6gcMcU60Pbr2+Bzxic5nd/s34zaMvqA367fMtNgbqmu2N4NbKuXoPt25vuucezf2dWAWkrxBDQSxd", - "AlYYMg/HiUBW+5t+71cAbii150LxYm1lx03D/S99DJFxJ+bHnX8/5iW92SkG8opyLnzN6+6D3c4jPxBs", - "Vygw9488HnR4O3vFHhnVW3EMo3bErd5pNcKq2dD50w4Zun54I8KB6nqrf1hAuX8gCk9zmzeo0yJwe2v6", - "KdJijB2MF5mgAxmAgMws4gX2INB6J0YrTYK8iaWBbczuyPr1k1l3AznvxIC6xSD9iFdugYgeDa/ikDoB", - "UB3FYs14HKcJeOxBP7DPaqywko6KrJ57Rxm3GsPqFPlv9S/CzucUnS8EFabnIpMU5scUp7LgmejHrsMy", - "LXkvuvNYFeLnasPkstQ24vcZEZjgeXByz6RZslzyQs8Z8K9l4pMzvJumdRtVw5fNGG1phO+W01Wu9jL+", - "TumenFCOPb4Wu3IYBTRO5JgOwbhzRk4r0p8cEcoKhqeA0txi9Ugrw2IhitzngOU6q5YETJAoSud43sq2", - "sgL76VqpMl87g59Y6ntKIiXLe8hWC21FomZau9JIFfrqwjmGKZOP1WlKw5XUztuO2R9FGUERSEQmCl0f", - "VmPn9wL4peZv9NyyXAv6+NQIfkdZ3K3w8jBRVBaTcZXLPDhsjFjqe16E8dD3ip+AFy+ur5gR9xKheRJ1", - "6f3zeBHBWMFNJd1hQevh4NOo3vBRcNsPLpqb2qBra4eIvHpGbrPvNe73c3YH1IKVrKQR1KXayakspFuT", - "06mTXpQAYwnM4Tg61NWeR9xpV4hFA1w46UFtpX5TxKfU8QAMgSVHUGsR6oSwwgPbyOCT6XjvxRLl/yZM", - "p+XLsiAm8Ps4DK2oSM6MrMzFmF0WlLEUj17mokBC17wVbnxoHpcnS1/HCvzGpNVAa1uS9Lfl/MqR9tDO", - "sg2a19jE1ojD7RuqLW0DibaXecB9eNXug2nKBVeTOmpqEdsV/xjammKUaBJDb2iUoCt7gjFSsYzP+ALO", - "SiEMp4+Yd3m/mrt3ePe74cAXUXfeoxg1jfFCDAPGVTHqlxmNrWFD49HGB1NCf50j2rd+PT6BldWMsicn", - "w5Ov1xRY8k++4fm2A78shWFTlAtaMXyq7ngXgP7rQJZUQc8pinGisCuR05S3js+uFhobxlKf9TF7AZ9G", - "uAznc5vwKSOdSBTWPNqFNo4uljqHHL36DK/NAjvWxy9SEezunkf9FOppNYc1OJiG+AAVu9GG/gEvtzvS", - "d+ieHO93zP0O0tG/w/jMCdPYkGPb6lN3/IeufMt1UX9q2CLpRqf+jSV3c/RMGKEy0S0S2qkzNRyIf6lT", - "xvR2bPNKAmUZhd49EbW9icfa3Ym7rz9m+G7I0Ujr3JmUnRgxs4y0SQ8LTHC0Q1SADJYPfHHzzD0tL78w", - "F+iQ3pCNPnCtDKHGOHv6d0VWeGCb/u3mXr/99bvxNxbxtdp0tY/Ir9il60agA/2VwkZS+Q6kVd8EZFOg", - "yWzBDH2EzQp+ryvTtEJXMqRl+EJSiZZRA9IyVJFi55z0/4CnviP0ypPGd3yzqYCPI/KJXfA0JJgLmr5U", - "81MPyLaSVrC0UQaakmVDYHBaidFPevrIonYwyoUTBsHdsPJQ+hx5TH9KFBahnlB5A2jaHiwCpABqttzF", - "DHhQkxE5aoSzHNLLxYgybLBdXsmNk4gCJYsKbBFuhd2oE8F61XYRa5cQPL75xnaM01MOxLovvp18DUCf", - "G2GFi4Xy+6vYN6PTPn+KIHARZQYBcCMGwXPfzzyvd3t8PFJDrH6gujbksSMaLhDzB9SD8AFvkYUORiLM", - "v90y4GsgIvQS/qMV5tpX7ffSXonVpAkNsAW8iP5MFh6pGyGgYU8cDToMGsHYDMJ39aCULsy38FkkPm9u", - "ylUeNb4giH+/b72tefYsGXc0Vt1sWBzS3h0TZbJ39KEOIfww9y85h/ZmKzcqhz4PB+TUmKA2se/VP+Gz", - "t/Cof38Te7ztmPUTGnrSbAzWQ2JQCS4aqIpb9x3qDF3Bifcl/1sl2NXL52xWOZB598JYMEe94wITR0ps", - "eEb14LEOvvIt/KVlMt8fuGjMonMVJKUvtZrJeZ8eCuZVppUvLO4oEKEsSnbijBAjKx2c/RW3y1PsJ8NV", - "Jkbx/WzNMl4OWS4yXZVFqD6oMzAbT47ZK54t4kd8NdX/+N0f2Fv5YsyesO+YEZleLqnW/uSb0/1unThQ", - "X85hoz5CG8b7kh2x6LdO0u9PcSRA0AnBfNbx2A0SGm3tCCsi8PERPu5ridDLVupGi076DOYE4+NoHZ0S", - "RbSiOkj8dauHbidNioIv+aSNvbq7hTC9QaUBhi8nSzndXhQ+NPLaykJb5OJl6UZUZpLxEsztt/IFOxnR", - "30aGL/0ygtMfLIl6izHBMOwgZdTRN8mdT9VQRoDiRG5vnMMjy6rS4+P+nn0vX8ReOHNMB725vWVwEIqN", - "LPT379/a0yEbPWXfsUqhoi3yFjlHu6jjPh1FTTWZl9Wk4GuP3t8mJk4CtpUeYCdvhePF2eXHlxenQ6TY", - "5fXHmPfYP4ZbgErTMQB8ohCOtXaNV06PqInxfjYCOVEfr8Y53k+Bxh7vtQuaIuum8R4oc3jr7QKmDxpG", - "PkV1437QHvvHfY1NMDGWys60kXOpqDIv9NmqveUZV6GKjbNk8PJFMmBniUoGr9Q9/C9LBo3JY91xUZDm", - "4DQTIPfueVGJMfujWFvSnjwgSI2ujH4+e87SDamWDlnaZsJ0yMbjHnjBdm5eV2uCuqJ1ElLqmNGrmGuN", - "/i4nVN39GtVUqmlU92fNIwznVComZjPPVA9LXg6Tnq67Jq2ZtLYKzn+Y4fXHD+imd+2GqT6fs9FG4bhS", - "/c3rZOvwd57u7eO46/R0COgdd8uw+9bultnxyOxVDm7aJ/QwPeGg6/eoa/Owy+vgC+sQ2X2ovD5I5h4p", - "NfclRP4/m/v2Mt1HPORdrsciwMvrksT8mN0KDNOi1EQMC+HOjMCQPtWl3AtjZI4KlQcWoQAvYuCzNBkk", - "g5Sd+G5U9PlTEGjpk5SdqGopjMzi351O1OWbVxc37W+foATH6ucZLwobEWKEumdnTXX11IcXMJBKa7kT", - "ovQgEQHFh+6APhDwjiN3AB7W9hHcj9K740juH7HriB761h4d86188Zw9aZZE1VuxZwMaSmanknfwDBuy", - "4tB3NmXH4e81ZMn+l3bKln2vd8WZbj3abu/BpEyERusHjOtUyhfUbTewzExPL8hGv+xDOhuFmeUfuL3r", - "qt0G5qvK/uQnj3u6lNaimw1ssabXlluW65UCRQiUHifG7DUvKE+lKMCMgKU4PmWCGvI/x5ZODPt140fI", - "fHHc3lmWYdKNULqaL7wwsncS4ZwIESRilhAi0VSwlTZW9NXrYVbRvOqsj8RpNlYEM0Cns4enYdJZzO5w", - "I6lA4Al1L41WS6FcorypNGRyLMZM6anO15jNky20DcV3AYW5d3qmMyPMcZWDsjyT98Jr1TURS0NK2TDm", - "9OBOPLJUiZbgPfF3rcSYpf8157JYp2jyzYxENG4sjfLumQf2I93Bg5sQ5d0I5ktZFPLQbiP4hqnUFxX1", - "4Uf60fe3ANAI9yyy+QbUVg01SFj50ibK6/oh0ICAOlzlBZwhlYewhI/dSkewYGpNaWDIc4lacnMncub9", - "6owjaJpxVdnAbqphwFrAbD563w4P1Fjk2N/tYOiz7l6ztxS3rCsIhhSK4XTDoP+n5IYvhROG0FsqRwmH", - "RqBJliiPEokOCbksC4xa2Hj+ehgStAdsQtANMYHKxUysUAgNY4l/M++ITdd+ksZSXSp27hCIpSaWAYEi", - "xpqCMAhX4Qa21IHpvNsdpXA7+kq1m5LokY3sIy0Tn0RWgYHZnQzppCt2d/vsswfRCLQcMw1bEqsGsYFZ", - "YXJpLRl58BTt97SG1sQ4Qy/xagnSktDD+jpqbnlbAnQpx7diyZWT2a3gJuvvBebrpLq69xY8u8PUCqyW", - "NbpkPiRKSZfE2yFUw9WalUbM5Ce4FRBkyPitOaZc+WFVzlth7adPttsIIlwlIywQpmfs9dWbVx6FjZ0g", - "CgaqqaeUiItyY78bS6pJhB/Z1DbxRZZpK5VgVi5lwY106zHDTCG434Mg9R7GkyfjZ0DsRBVyvnBsVmjt", - "jyWlC3GgKs8ce/eG/a0S2CwoosKcklmTKLBBnA6n9DnGoFj6ZPztb1Ia1RmZOZbpXIwoTs8sMgkc/IwX", - "cmpiyualzsUNV3dYXT/6b7/fyEHtBVqJreW2Yn5ORJ5CGwYdP780YwGx1sfmMNBLh5ytvqg/fmGCvrCu", - "/NA/86IYZRg/xSdRVVTZekgAgpR09hQTz5e88BnnLS9Yb7+iYzMoXstCIP6fTwv7hXIohhsk6SYuwTx+", - "lU7TDylT6Sm6kXbSG9bCuyrUfYemQxk3Zh2BeHyiRvddRZqYEOqoidZvUaD+QIUPXqj4AS905eu2SqFb", - "NSqtNbTItWOXdxdBe0oeUVfpeecLWgvEMXfl/dwuuBEftD8xPVdraLK9P3ssPtk5lsxFxs1t1NM3q5In", - "M7wudkGiYJYKy0XpFozAwtlSY/2FnlGSuhepe4J/u82Yfjdt2RXZfhIAqFi2kFg9RBo82BrYIO6EdHN2", - "Vjtf9s8Rs3C6rbAY4O4nGR7kqXArIZS3CIFE1B3T0k6chUA7IWfYkq9UwPnp6a9MuUPtDM1ohjTgmFvY", - "zNEyCavvQUvwsZbYzu4I+UyzCkQbNpipkxORBXegCGOAZBJiz5PQIKkTI7JYj4JJ12yohqA7+3eZl3Li", - "8yBIi8VCi8H54P5pl6Sc8uxOqA4efEE/NNGCCPMpneu0G0Bsb1bABcWJSqPvZY7hNzUXhqqTQO0B4S6M", - "h83/oZrPpZq/5pnwkft8mCilVyy99h8YX708OU2pHjHV6Nw7b+ll2B9Sl0Jxee7EJzeKUxx9M7JLXmCQ", - "716v+Vyc039GqP19c/70ybNvz1GLS8eJ+mipG2w7POk0lfwYwficS2UdxRwb0HLpNkJT6o+Hz0ERBOE9", - "onqCAP+1m76BgvtJfCdVfh6IA4slaqQYZPQrT7f7gdeZJKCJy0w0rVvmr/MZvxNsJj+5Ci3jGj+VcVS+", - "b8OrwXLHPMiDFzdZcoXZ4l787cYsi1HIuHREG8IOXaMgQFGcJuqk7kGFSnYgzymh0820JkBa2CHLOJsb", - "IdQZpomC+FXwqVx7JHRqOU/VbaUwSw4XL72CD8VEwkSd/PDhwzWh8YdZgt1+L0DURzcNGCjkM7rBpiU+", - "oxMpTIDi3MXsVhBxlA0eUeT99EtdFKd9rkTQp61rCoqNplr4OwvwqsEe88+zkwDsjXmI9OPZferNkWGi", - "6Eg+Gf92/BSo+q4qikZyCKayNkHWYK4REM4eiKaEB2ZCENO7y3Fb2Rj+rvKVm5ZRZxhYkFTst0+esCVM", - "ILh7PW+Fl9CtQfcQnAL04BpuFxvO0Qalm3A1XTV3RswD3lR49JCrHPdlUpkOKfu9dD9U07B3WK2DWB6g", - "haftjU/91uA6K4JjOwxdCWnZ5J8Duh7qux2oRpPYrHTfOfeDjrD7R5RPsBVUuchrjnycJirQQSvmKUNm", - "frHGnrwedc5nZ6gg8oQBJkBnurRM+xBjHU+AybQn4strVxLlxIkVjqWXV3+Z/OnVze3V+3eTyx9eXf5x", - "8urdxYs3r15+hyiCTW8EngGp5r1H1o82wdH2p27iw5fwrNePe5usBxVga1fbysTGgRt2xM0b0EidGk+n", - "6rSSLltEhTxc7b22QxazLHd1Hd0aZrOdDEn8wXBA9+FgOKC7cH+etG8n4efRuaQGhs2RdShhmscV7BxW", - "ieN9q37+zbqcnaU1tBryfnQXWu0GIf4lFtw/Wg8phgPQ0pSb9P5u5Vxx0GS+iJBbpU0HkHaPHxqG+WK3", - "77O91Uxf2xHYWtrXKmfa4sVfsaLpg7CuQ0z1LS2XS6G6lava+xAfYtzWUBIh+BQUWQwCJuqW+i89qW/C", - "+EQhEAADrZP4yYL/XRbrjVLYzr3Xdz0JAi30nrtushjRB/jUPMKbuBqOOtaFR3yFUyFszH48SA/ZcZz/", - "3hWTlH/HvEnqO3JCwPCgjz9nTzwWQcS+Ot3dEjXWVEn0M7eh3fdEt+ChPlr2yoFOyPwbUeqREQVH0yfW", - "p1Mk5UzEDjtUdVZqZrTuCcNtz6ZSShQvpOqEr8N6gqI3rk+22x69PNTgoZGGnzsjZZAcw1Mcuk4z/XjV", - "7UHqvVyaZnIE3yx0lc8KbjARYG56lNJ+zXYbDpLGGDZIUq//xz2E7QYRw5UfA2LV2qt96Erx6/2Tu+SO", - "F3reib5JduuRUwtK6J6p1Z/fMbeeOpudeSYLD33cgXDFlyIfOfw0K6tpITMWnmYIHp9Tv1CQE6e9DSSa", - "PIYvYUhHZnd9JZYP5UysAJ9Y4XZlYWVaKYKvwcfJbsW6xpNQ3kHoDafdR6qdzb4nT2BPcncM3jeOClKs", - "sSvNZe3b+Tpt7oH7/w+wfx3O3/g21iNEjtzYSR8jdzrmuSYq4PHRE88pEzcZJIO6djRgJB0s9vuCKjus", - "dHKZwG1TO7cwRI9r0o6that9gSLfEfDbHUnZHjqhLUsGoDwktG/JAOdS78rzaMbvru1V2oldsZoOfIyH", - "cwMJnG5HzjUJI8Q6+Xjzhp0oTf1f0UlRcLs4JV2wkPfdS9mKrcSACRqwwFgUUSkIIsgHVnZHUw5KxOu/", - "JUNkpeak/gMPOncfHMsOxB+fLTBZHpo02K0Ab+5Oj4trknme3EODHgWaxNn30oEud7tWWa8GWOqimGAy", - "3j0vmnGt7eIq1PXIq8pzoTLUev0b7OTJWTgJ//4v/7qRa4MZ9apYN1sFIpNh/xDqVYg1895flbITXljd", - "zO1LVJgkkzOWrsR0ofVdupHL3+nvasFrxPE66kuxOYuIjRQ8zOJUECJb6MvRGBxWWlZ2McJWIipRJ1iW", - "F9yvQ5zd5uSYVsH9fvq8seR//5d/Df0S2UxQOg4YcYqFlT9n6ZKrihc0ckCy0IrlYskRRynYZuFs+qnC", - "RUnjkBpZ8QOK95vEOpDJuk9V6JCwtyUmfaopL3d2v4GHGJ+GmIGuHDbB/vf/+W/Md5LhjnkKJKqxNaHb", - "LQW8A9RGpP32nvVddK20grDKvbT6AFdq74kkpaW77W9AgfB6l/c9X198aHda8ZxbWeF98dokKhxPkHER", - "74E0uZMwJvLVu49v3pz6jGGthI2GXqK49drsmH377FntNJANuEYqiORhjqRhHBJ9+byLbotqegDZ+pJz", - "V6wssE35JwfUGrM/8QIRI/MYZPW0hGULlZl1iT+6RBlhXagcYIW8EwyzcqRWz5t7gb1rKbfdCOyr7KsZ", - "Cdn0L6P3F5VbjG7psYXguTAUGsSZP7JAROxlBDYCfKauzdjEodjrQyNi7GDEnak0+5Lvg867b0Y9g3uM", - "l97hH9xHpH/Ij3aH9z/qLz01thiLjYgiGcckmkKrOZVhL4RyMsOykxsxwyvL17H5AtcAaE1YI3TvCVS3", - "4dO90VSd8WLiT/TkuDku+Zo6zWH62EYDNBQGeHOchRuEsj59ixzyryIrRjjwyPLMOr5OFC8KvRJ57KeM", - "2QoedewT9SL/gVukE7eOOuCwecVNb0jT6C71/51YMaOLOtsPQcGbdGbdZPZpB57OKbyWtm9GfGzg2y3/", - "eAT7Rkj1BkNtpLlot2hWWoVKSLyZUAxjaYWvFC8ED23IgtGVKKqVrCUAu+aEfMuVx4sL1ZDasLQxfOor", - "7hIl3ZilcFTTiLNe97RE6ngFOu+qYvzlZIBvb/3l2ab9Z6KO7abhoQl3Ac9rzF6GpBPYfOub46KGEA8z", - "u5e8xhN6f4OIuXdi3ce/jXEeXiIUIabi6/SXw9Nm/4OkBjuZAs+TA4h9++Sb0yhH9AzEBSZEjEJbsvjR", - "HiFjYNkqipkxu+gRM8yIOTc5NvFC1Uha7Lw3TtRLMj0w9wUD48/j8QrbTjxXl0vBy3AkE0U91JzReZV5", - "ZARq0Xfip3SKBw+0xBViSjRBmvtYBE7hhA50C8yqXxgeKKu+QhMnScF3YjYcvme+u9ozNRwBPXz4Y49A", - "2J2Y3JtMTBQ53G0MQ/1ZukXso7XTb0zf3hW+a3/v/OcBL4r3s8H5Xw/p3j/sSegMKdF9sNqX8GfgdpTm", - "mBOehyx4G1vvx/YZ+zM778T6sMGMuNd3Ig+i0GIbDx9aPHhE9MUhBFsnrMlbjSliGbUz96n9QS4AK1vH", - "lyU7uXl9+c033/wBLH20cOSsFmQLUD3QI13o+RybCGxU0hwhlTe7SHRu0hYht7nlx8/DwRb4WVevO2rl", - "TjBnIwKKRw60Q49eYPSSESwaKhRKs6uz99v1236mES67P1+iCbq9txVJ6CXwxUDTbajv+rPDnpl3HcCO", - "lKQOCBSR3fWA6LwBxRFdWg3W+vjhcshuXl8yYjCyoBs1tZRpCG89HCSnEVfoD2WWwkidyyzYpjhRaUNO", - "WU9XiuDm7lgp/saWwgLzDcOZWTaOHA5BfpHgOlAhqfIBGDyqX+z/mfwyu2D6St3XkucobNDhwONvwhH5", - "UrBQP+0rNdM7UvMrpyd1Eua+5MOQRBpzV4s1azkSPVxS7cryPgtsG0G8ATIZvY8T/xTi0rIgPQX7DbqJ", - "FjxPFKoT50hfePJ0zFAdRA1n2OoUTJZDmAYjz2HRq+H4oSdWZKYrlvjD24tLRj+O2QeYF8PmI8pKF2ra", - "jXbckeuTkgsELaAzFhEG7Ax1vAb2/XjzBv143DoBKp32BHtkAzmp78s81F+DrMF06mCE4TZdXv1lcv3x", - "xZurywm2sLOsUmBcEs6dKIXK2RrhhCnCRhBkh7gNm0vYouBwi5V28OR7HLOjH1f8+7ZrrNwIBwWa5KKQ", - "mPj98eYN6d0IQxG8ZYnqiBu1KtgJvQoIlQyUViIZ9OTo18hwW4LQCJbS5FPQsIXFu2/MUiJyitTnGKli", - "PjfC03+cqLSOs6QRLCHw9Qj2bmNPT6SaGU7NLSojEuXN4+DyDG17Ucd/znjYap+jq4RA7CCWwnJTqixW", - "OrzsMemkZXW5OQIweIqjFfoIrXRP+2CXBwlHww1aAaQh0na/QPMssBNUzHPRjci0ymQh3pM7vbvwyBcC", - "+al5K6A2DmCkO1mWBMHfHwDsD4t6y6FHZ+lunxoyJv0ED1lkb1+GRib8tr7kF9mdmkqr7fzNRygON1j6", - "9qSri6Wnd+fAuywov3f7Y42RJnV9cb3dNQvs2PjGPgSj/yu5gna1N3tthBjBd1otFbyw8m40lFm+BVe/", - "u2Wjg8mbq5cjDAhoQg1uNzY9ENLko5Lwbu0Fgcc63z+0I3rwa6DKED4bW2KHQgPqrUPmG0e7KVG58EWV", - "7E8ytklDuQlDD8HqmArq8crVutmxHNE5EoVA2jlz2hcvodvmwNKbr+PD8Jk/7QZH/S6L/e0jW605H+Sj", - "eED3zvp4HNGxc5enIn7wusbh32yfFXx5Begaucez5jULETQ0oVVgmxw0HoglFrIk2Ce0oSiuRbUcImd+", - "TCzRl97Vjd0IwohSzXSiTkjtHsYEXvzfVsfv020411wLqx65RMEFzLhPSCBMB2dk2eXcPr5r7LHtC7ov", - "qH3dYDd36Ss3Ld9igi8o3D+oY/nmgG8js3yNTr57dIS9rX539/Hd1CkO2jfyeSPsemeVkWvDr3Y38/mq", - "fXf2EqkHUOeGr7bBdGKtPhxBAlGh5AlQa2HRpPViQR3aTh4OHewHG+oQxuydRgT9cPqnWlu6PnhZFlLk", - "7ISDUXUvdWVjl0K2rAon6XfK5F6jio6rw0UM2QqbXBTCIdY6mo+5xl4pwtexUlFGohCZB9t/BWQfvMhL", - "9Pi7EcXgM6PVehkanuyH4fmqPY42+O+Qlke0lXXrowNY9TWqaDc+OadTvvSzTuh2uVURgYSNjWJnsbSb", - "usiAMZaSfxPdm5SZiWX1cNR15dIhEy4bsytcB8ZOC4pNYSoKX7U9Wdg+FpOkFC8Y5fBZlmuwpwrB754z", - "qqZs+FoKPScOSpvHPq3nCtcTDnKIDb+xV54uB5D/WmCLugfSvw9M0GM+tA4ZPTtmFwHvT3s3I1cs4Amk", - "iVoK7mt+wosLDtcrYu9jG9rQnI/QM/FqatupJa0J+LDQwQHnSzZ3WYO7abrLJbdB0/qq3rDpls9+24sf", - "JrgKyVVOl6N3yGUv3j77LcM3bOw/GCOeVs5VomYFWjvklac2uY8sg6FOUFkptfdtfQfC0wkD0uSWau7z", - "bnB6u9Arlgw8iUvNfIl+niitWCGdMNjX7Q60+HthCl4mA3ZvxywZlHDArAfMakju4H7ZL8Ryoaw4jkyb", - "94SsyRVF9Jh90HNybaPumNa7kZLP0a00fg2LJgsbgKAFxm+cZmlL2KeHrif2zOySUYt2QiF1DMiFkfdN", - "KPomKz6yiSLEQjFHSB8CMEnIkjiD/fqvGLweYC5dMmj85bQPW7JaThayq5z/ku5PP5MG9xEuLUGB5iFY", - "EA57ougyzniJ9/OS54SfqLw5Ny/0lBfhdq6bW3Zmoe+WQa1N6fD4rqdG5qFDc7YGvvjrk+HTH6NL7n//", - "r9G0EApTG2ENqFYkainVaMk/MQUbXMi/i5xOI6wHWTTwCTv53//ruyfj355SlrCfz8iIQtxjb5o53P6G", - "w0pB+QDLJBl80GXMQkgGiSq5wl4RxtkYz2wgfO9js92yKzRTbdOqse/DpmxqH8EDBF6/hVBDgR9nHzTV", - "2A4bgUS4b8vbWT4YsQUbN1BofU1pIbHTK1d0zyYqr8wmfps/XZlGqNhmH12tWC7tHaGr+CRNL5hqV0oE", - "DOXzuRHACPnzIE092KhvPB8DHndKr0LGK6iK1Bod5FmAEtnAYT2CoA1lq4Oq/t7cTVY894TEG3wz9XKn", - "lWMrYQTc13iMQKglau3jFJjNi23wMX0nXuy0rJydNLPpuHNiWYKmjJMmOksTgSKxVyIvS8EN0767+Zpc", - "5IlK6bb+LugVwdkmZ1ENLzXhb/N8/XCCNtWnLorugEmpjz/KBvKDbV4xzF/U1lsWuDWgam4QvjaHpA3h", - "VBQ/Dl1hQAbYXUTtRvhhic2zcnkv86oWxDARtpDzBTAzyejiS6jTb+VTl+yZswdwG3AUC5lbjTA4iuOl", - "pLN7ktIa4JvpKWKyo8V8jnzxyIiaITGzDkVcorwwmPoMfovYyGzBi1k4zAu6QKRvk+ttvESBKOCl9d4k", - "Xsy1kW6xxFhfZcSI7ogZVyNduaDWw5AC9Fhhx+yDkXNM4W0WUiDUltOY2TUDFoevv/5wmyhqHU98jAxP", - "nFwzAfL0gls2BQvZfxOUtiqCcimxYrRZD9/VW9i61x9u+5i+F2EcC1b+579F9FcCxx+zFAlLv9WrIbM4", - "ZzPQ7KaVS5TSZDgEGHGEaYqQvCnh545Z6puGTry5V0fCApcHyQ+7ztFCsw2DHS8DkTPYNroRSEKHrTyx", - "QrC0eQWlGx1JsdwFFzVAiI7mbA6O/beAZ/w9esBd3NqdYy26XVrE9tiImJRVoN3fAqt4Iwc2x1xUNMCG", - "i5fytjCtlQqrUnhQG/l3zK86Zy/wbZZUT558k11e/WVycX01+eOrf8Y/iBR9DDDU4NwPVKtCC+fKwefP", - "2JJkpjs0wQ8frjFLIZjYaSY/efystDZZECmPjmPOxRKh3ahV50oazANfUlvy6dqJkSU8eJ4Zbe0GoJil", - "8oy0gTeUJopyrqVi6Rkv5dn90zPa8JQ5bKXbkNWFhwlI2xBGKcZPEsVj+qcdkXbAHcZMfLvRgqvc4uz/", - "039iF3VqsdQKl7TSCFVfFKLAMgLMPAilbCAM+TKElNwaq8WLc3hxxB4/fmH0CnNYz2rb8fHjc5YSdKZf", - "GXz1DFPtUjK6MMGT/SZRrE5txnZciGn3g3MlVullWt9J2qCQ6JaS4ux/wTxquMwY5qksOSysQOQ7xIwG", - "5U05XMHIB769QmfH7DakKhpdFPCJmTaYRPv0W5bztW00n+U2gtKNaeGXb67YGbt9+Udc7S7u9Ql5nnNh", - "z/y9BSdgxS2M7FuRwc3fJlwpR3dibVPf4w2T+MG+G2GVD9XpgKk+FfCZkBdZ3+gFgb+BvOLYTq32uGSF", - "BPGOjOGLcUPLd0RGJ14IcuD0nKXfv/rAzhaCF26RDv0/c51Z9JjhvxBvqpTjNV8W8ZEmE0y1dtYZXo48", - "t8OrfbwCW0QlCog1dvHxww+Tl1e3hDFG3a7tnSx9WSa51iJQYGxff5KLe1HoknBvga0oW2PFDQKiSeuz", - "M0+RFH/eTIZyHGwxZNtY80B5274HhAtEsonCib54//7D7Yebi+vJxcu3V+8mr95eXL1J2W9Y56/XF7e3", - "f35/8zKljlRwUdfJfVSwcjLTJiN/lz/T8dRo5Z9Ekp2O2QUrxJxnaz8XLzdTNB+wXQGWhLGcO47JNmBS", - "LD02DyhLzEo1B209Fep+FPcrDcm2zVxb7icYhEuIr/E8x84scGUmKvw1XWhLl0hKJq0NjTQpb0d4PY+q", - "INi0EbiTKlEfb94EX4fFu18Va0xcCZa2PxI1Ezt+Jxhn6c8w5ueUfbx5Awa278pBg0nS2x4/Jio+/R1b", - "iE9AZYoup7c/XDw9iRM/TR8/HifqkrpqwNaTDyn4fM8iyuEP3C6uYamBNrfYXBQZzvsg4Yc274e3z2jG", - "Z1TmgEA+KVtopSvfwSqlbMXUl++dgwKLFkj45ZxhCIOk/Nmnkcp/snBjWASki6WUZK9jJ5VEKbEqpAKN", - "1be4Yr4hKtDhCqZy7VsMv7oXyqWMFAA79IcjUelCcOOmgrsUTqFy/iw+fRLKs8fsfZEH0eOdR0LlTGlG", - "E08ULQmNwLS5CFzAKZsLUtGJyz23jv7p9v27phsYSf4KNDgL/7gITvT4DGaJ19cbtiyyC16Kc5b+nPj6", - "+2RwzpIBiXHv4icxngw+w8a2JGJgJWpp+QkWI7WK7qVK0XNrds+NBIushvgr1okKMWkYnfz2NPp4PPaj", - "xdYh54NaY4FjOWgg+gzun2KKBgniwfngm/GT8TeDRmuGKGjh5J4FOYDl0V2pki8xFUPNEaG5Lq61CyPV", - "HePe1UxthPFqLvlcWDbXIG1QMM+MoOYPmImBZamVRxwuODaCNtIJS12AasGEzAFmTKGtS9QSzD34kUIF", - "kn6zkjxrUiG7wq1dcDMX5G7UFnQnFNkwN2nB6vLXQkBIXegVW1ZYH+STqgsUiitM1MS6IJ8JTY1fVtq4", - "RaJyTUVwPohBxd8IRJKoy4Xg5TkYIHNBafiEipwGSkyQRikSw7M7tmwjrFubcWWHibI+ugFKDp+JUAWO", - "Q6SYjnnPs6pa+jZNXsNfh4VEQtKKnBXFjAYIJdEgNeh+BSal5WFmfyHvBW6HdKHMyYhZQU4KwQvQA+F8", - "44USsQLISOeWVeXc8Dy4y6mcSSD8YExKrMM7OLuMq0ThwZkKYK1K3VE6EIa/jJhWssifg5zNDIEtF+Eb", - "QFPPc/iV5qmCr0Urz3upGl264eJZIjCC77tB5hXim4pNTax1Ws6MyAoulynpDOm00JjaTwc3pVRMNar7", - "3KJWTg5Df70RlrzBji9Kr8g4RYwF38Kb4F2UYD/pKTn+GIG0DyNEdb2UeC4X/F4kymi9jPZNpsv1mN0Q", - "JDd6362DXdYzKlX1DeXADHYBwZrckFKrq3xwPvheuJd+5bcROt7LUZAYz5482cgr2BTcWE6NfoR9Xob2", - "QGjHdQcVI38T5srn4eDbJ0/7vh6ne/YRqytBWaeG0N8++Wb/S6+1mco8F5hc/ttD3rgRlOFqP6oaCwZN", - "5Wq5RMw5dG0ZFwSolX8XQ5I6OSPXByr3NePwHLGAT0gnRMAuuBL43NaleT/CED08i5n8JDLQB1B1SPur", - "+oARLlghXOuM7T5R3tYAm9Qf2oZoWYF8WPI71JMPOVys1Baz/7CBmk9vhrmcMwk3ApeoPVqGEl2Y0ZKX", - "NE0UXqR7IEI0LwqdYb6YNviFPOSA4ZkTn5zhdA0NQaxY7ADNHfvN0/Fv//8R/IQO5gjVBCoXLTTPUQI8", - "fnxh70IZNNWy5KIthrG/BeFxSxUPLNwejx/DVsNM7ArsivTZkyfpmKEJzBXjmSPfLGr+mbZYr0oXDw5+", - "sfFbS2pib0Qh8QYjCEVsmDgVGa/CLSutDz5gUiZfB54Ey9J/Oq5Jz+Cu0WVVID3D8sbsFpXI9NmTZ6CW", - "GtGQ8eRBCI32CBF0Dw94vk/P0S3KcwKWX0mV6xWGafHm8Y0zSPEILnAEkCfmW/CyFMpSby9kKrorsQ0e", - "9RfHtgukUiIbdcm/W+EuKqf/hGfnLaG7eXfAC01tR76K7KsHCSX9n9u+ODB6P/+CwvctYpMorjLxPlCg", - "SwZ/6GdsMixw9h5bcMzeeY8tdUhVgtidnvSyJBIc4ZdkXlBg/fNw8OzJs199fRcNDvKg+CTvYGgCsKVz", - "Mf4V751vn/zhqxECTaPOlfud9DVrqGssRJGTOzBIDVRMyDwF9cT6lv2YhTJfOL9z3z57dghdfG8IuiG/", - "6HqFl//L/pevlK1mM5mBEXrrtOHzzav5shZ6gc0f2ShCUA4++Br2ko0wz7pQO27J40Ru3TnmD0d2pOYq", - "cGYsk8ulyCV3wjv5GMreMbtGdybZpsua4aN/2YMYoiXvXV6+ORQtCc7z3AhBoItDbwP5R7BoyCzpqOaS", - "F3qOtXmJsnzdQjCQoJAXhcjRWfyYvQ5RXa3mrATlt+Flk5Y9fhzl/OPHZKjkeqUCfsMwUYxNwQglwyuP", - "EVyeE1wkXMhwJbJ3YkXuMNt4Dm934FNylP5EEM1Et98++Sb1TU7SG+HMenQxc8Kkz2ttHH4NDUFz9Ft7", - "ELWSI7rVqwDpEAvg4Y1YoN6cYHAJSAPfQVwai7msIWsdzNolyBnrdMmmuBENKCdPx7wi+YWE8ZFcsBLu", - "fcVEqM3k7Om3o5yvYxV7IWcCxhrDrnzYcHfCLniXJ5mMjx+j3Mt16WLoEDQ+4guJ7muqFK3qo1i3SqVy", - "ORQeVYkjvvpUiswxo6v5AhQctpSqcpiiwX7Pvn9BXsoVN0t2e/uyZcUMWVlUOCKV2INaFLoCw4qes1RY", - "J5dYVuE9Til8L21ZFSl8oVkDEFrph5djqCe0/PK6J0ccPVjM0Ou0VHkAd4G0yzG7XfHYoBupFIKuvjgw", - "dApHVDTE2/HcGRt96CL36Dq6FMr3vU1CvpQucqYV2ea4aStewjRKYbC9FCqtU63dMBzFoCgmSpMm1Ly5", - "fdV+qXUBBy7MAv0QrNEL3jt2SE6D1jWiJrwiMAkzvM7iFJ+kw6sD/TnexSuMb2YqnWXXVy/ZU9B30fEX", - "iFzqQmZravdZwkysNp6ldXEv8sautPVBtuTOgeJcK7UiSsxMl7DNnD22ipd2od3jcxjaO24yvfRwdNSN", - "OrAamwoUINL6aqw6GhRPAK3Ahj1H09rvrZUFBe5yaTPspoVpSOTpCuaMUOTX94kcYGRRv/8WOXG/myvT", - "U0pIJ9dnzX6JAiPLaHQxM8TBqQ8ksIf11oLRcN4oHQ0BrDzoU+04R08OmUN0yukCwB1MlEcgZZYr0bDv", - "soJbK2dS5HiWcZShx7+YrhnGXQnFcUh+u5h2gTo5WgN+NMwP8tmdsE1+SlgE3pL1iXpRX5LwRw8GXYbg", - "WG7IN9jIF8HtJRk3Zq+l97/SL3g1g8qjETPL3xrOcGV5zPeKCNT47xmytF6pISZxRc4i53YhS4tF3w3E", - "BrfSYaPR4zYL7lAyuxLVcagKTv59XTk0XvD0e4Ymp2PtKIjOoGyvGoOs6P9gE1X7HRpgYKjuBNxBDifY", - "9yKSzl9lFEfhzIqSG+5EdCwO/RSA/1Hgg9X3sdfoa3goKG5yabhdoPfTrc+j0gKD/aQro1C9oD7w6AQi", - "Z+JUBAolKnx6yEJPeu70koLEw4Z2kOnlVCoedpT2Dj8ZTRXqp0DGOPY4E5Yiipj/x5a8RDszEIrySDJ9", - "j1mktDNjdqFYo+E8sSI1nwMpgUB27Se8hA/xgTwGqbXBuLUgzZwkNIW4OZaCYwklNqGw1GBfhdqEYIM3", - "tjjcSyFU3DDrg+1GafbVEsRVu4rdG228KEbajEJ2jj/JXpMxYmSqOoWCSEv98CUGJWHjmvzOfXS1NCKT", - "VqAvN4rAel/DHUxjBlUJWybn6DrQK19sOw3Xcpx0ojKU1eiNpQzmeKkOWf3bVETmiZmhjUsoUcErQf66", - "xmuom6Lu03qRfBYdfgZ/hoOysu1m/fUt4YZY8Wbw/2fxdlm8GyrmeDMi0FQ30R1GEUZvKf8j2Lw3XmFq", - "aYgNx1nDCfxg29d/q9/2vamUZen1zcX3by9YI9oTQl2htG6p70XLL+0VZMrfrwX20GcXhvLyP1+8oRos", - "8vWx27XKFkYrXdlhDAOhyM8oiCdd0AUU0yanpi2kFWAH92Ce0Cyko5n5OJtKlPiUFZUF0YKPa5+xu9BF", - "TeUQaov3NlUQew83HVKv6kYJWi8C9X/Qn73CFEVKmAS5t8fsSuHyg2c6UegaJ0oG6GlnKkXHixTIAsW4", - "8ob7fFgXCtHUE4WSezMsXHfDLDReHTNd5P6qgs9hiqe/bKTDGCIG4gKvheAstTIihz2ZlDFkGO5dRMCp", - "8YkTBGaWRjSii06DMJaqyVCsoWsgSZ6z9Nsnf0gTFfsn+LyGZgQm5E0sBWaLw1bIqFBZVme/R4LkYmY4", - "FehQ2hH65rOmVJVFATYVnr7GrJHyy9Lg/lBYVmcYe/XmGG4+qJ/1quA70c3tNEZbvEccA++NCxbUc6lE", - "1zV0Q994bYS4Brb+hbzdfpiGq/uXdG3H0RBnpeMeuAlyTippF//vu+c2ffbAnXvOxNeNdiIk67bIbgfk", - "26H4o2+augV2q5dTZ2bLjfejgqYulDNrMltbLWvrPtMkDMDUI9ykRBHFo+hElMAghIW6Z/fcWMoO5Dnh", - "YWZGYBotL+wwUWVR2ZhcQv6B+Bqc1rqPs8/dqzX1MV0CdbmmN+KlD3XFRH+ceW50maOBSmJfmBH+fabN", - "chjnnwwCFtWf3v/zxfevQupgMJ0tv5dqngwSNeVKIcQLmFOYEygtW0pM+OuSJ29kR3NB+0smEWyNhp3I", - "Og4F/B32rGPX7a8mAFrnBGfUmM42R+OmfpWjcUYAl3tPiC/rtITHEudhRSiCO0F++g3DNLxwPqaFnp5G", - "9kL/52Zz+0aP/HHtLEcnPfmN4qhR86CvM/h4VKqFuh/dc8PeXbx9desTieC2DWlTsdJO+xBHSEC8F2bK", - "nVz2JL1Qv/ktZvolObdvyP5EmGK92Rz+18+F6WkBaqNfOzaP9oWUIGZevX3x6uXLq3ff37YbR59unIjv", - "fcZntrneuo9/ZMk9h2LYnfVygT4hnxzrkwl8CYdvz+ENt2rqncXEhcPAn4hy79MS7IpTSjqx/K2ngIbP", - "naA70oY6vcp7BKktEUJZUZZqE4O/cd4wv7FSTlfZwmu71GEaPdmoA/syNCLQjuP2nBxfsU75kY3oCxO4", - "6ajtdQr2L0ZwsFCvvuvEJ5eoOj0NLs5YEU7lbI08FPxuok5S+OZ38EdEELZafYejjGjqoXxqI/eiu4X2", - "L6ST7mnYfVA6xrP/CGFAE2fco9g+p14zgXOkRWakM/5k/xl/wfO45P+IS9CvBj27uZzNBIXnjj7vB12C", - "P8Pl9fnMhZ4snY6JF2SdcXRkr0Z8xdf1LGJoaFtAZLworG9vzE7gvBDaD04ElUKMi8TZPAcjcqWoL0e2", - "kIVv4kIt3U+9jLl1GkNpY9a8ovPQeLjReVnl6IbQd9SOgH20ocFAmHOtRHpFD6YM419//JDsUhvCjUxb", - "YAUVb5FZSwXfJMxOphihXw99cOrjzZthUBeDxnvaDNCABO0SBZ1Nqqm0kko6LCLeS9gvj3hAEEehm3n7", - "5A4bp/AhnfN/fLgI2tXdf6PfnV/myJYikzMsTKwVoBPMRcSKAoF4EbDQZtfTWKP5q6aR7e4m3ullxnga", - "gzNI8NMiD2LkH1NcgerzrKtNQWMhoa1bAyS7Kn1NTVCMTmAaiQoJKUOmhMM4ZqU8zG8hEIQKL8e2lAy9", - "w6gubctM8IwS/GxeVYFDd6zUxGSaUaGzO3uQrSDVyPcF8G9iPRHMqFKu1Tm1odM5ag5HlQRCGoyI24X2", - "OcTw3atr8lSeXF0PqYb0lJVcoq6BI/no+1TUFEUZW6fZoxX/7bM/jBn66X3yxMjnNlCajyhmo4XghcU0", - "VZRIhcR7h5yw/zd777fcNo7tjb4KSjeRuiXZySSzz2dXX7htd9p7nMSf7UxPnc0pExIhCW0K4AZA25qu", - "rjpXU3Vud+2q7wnOA8wz7Pt5iHmSU1hrASRlSnZiy0565qo7FkmQwMLC+vv7WYZsF+STYK0Tx4DcQCrA", - "CawnZ1sd4mP/vscwpRvciXGUBoJjWxYEVTqu8fM5vjy8BrTqcidQdvC9HiK3/gXEmvP9WE5c1f2IwfoA", - "dwV9udjKuvzYdMhSr2pTVsjxpQ1lBCDmOyyVRaDBitBIRycMZkwrx/OBvRaiCDfs+hsuQK5Tb1037qvL", - "PF0Pog3ltGP4C3ou4TiGdh18IchJQy40ciaHLUV+DnrbNCR1sXtffJioo0zMC+1FcQcvQNvkUixi5r+i", - "ocOsVyxRfLX9uj3AbEW1ATYWXq4P8kkW/OsW+fACERAnutrEkmesDuo9ZbT4M8pqm9WtIAG8fZd98iYD", - "F271ufATzy8tdj6/ffvxh4v9vf0fDy8Ojk7TRjFrMwg7nE7LCfUUfrQiS9Ro0TRWX9jakQevADvQe8k6", - "kAEi8OKMXwnmdKL8PmU//oDoHkcHYC/NuMoCmAr0KEYujDEfz0SEe8H2rMpUbrL9QlO1GACrpzc8mVRF", - "6TBvBg3NVqw6Bt7h7G2ypt+PsCr+ue+/Esv0c5iAZ9T9XjzoTbDWZowvFzJ9nyqYJiQlVstmqGquqpBL", - "8EqivbDD3mo2E7xgSJUA3EfEjGSFg64toHgk16XIuQOEd3FTaIsZYcgk5wsEF5zo0kD7JZ8isx3wDoFZ", - "ArpaGngmWiKQLCHI6xB6MWuImWJ2OECcwW/SYlFRsMka5Z9Y/hRIj6AnEONCNYgzqCYa61pHp8A54S6W", - "30ItJxwi4kqC3QUBWJ2oFIcd+jsuoBHrAtK6KXOG+1MdFkD+JVb85ZqaR4HbL59D+69WAJ0/gGpIaKbF", - "b7yAbxxG1qbYho2IeLgBxQ2w7hIX5UH4WDg0A1PuEqFVmH7EbsHGKf93WJxE0T15o/kNNF1A5ADwAetn", - "hOeRnByKB6FBmTgMwYSM+Rpp2UjMJBSi+1mAenKHsIFV5jZUnku3ImwdsnFI7bXRnGp9oDW9miVe8LW2", - "aNZkAjQRfM7n66Mtrni++ItYUwYTeDao8BZKYnFsKsgIr0S+ZezPqAozEHAQYQShPb3fOHBRl0DKGwrt", - "ptyM/FcBjQLt9EQRdnKoXKLsCQ/2ZQNpBuPRVUENllfeFELZwKkkHe0St6ykgnqDmHKi4oYbsv1Qh+4w", - "mMYC4hoqo2iy2zFXtRJx6itLYa6ttICqDKUygDlFR3d7F2K1UoAzkoaCba9GAw6St72TDiZUJRD11Gbo", - "mltAz1VJJ7bcQxFrVa29Q6pXQuIbq+qvbpGBY9cAeqq1dh8KlmHdNVV6Yr19DOiT6VG1XwTIdBI+4lUX", - "HPQ7caWQngHIW8WuZRG4f5o6Zg8fcVqJT2fzlRt7tJCt3iv9FkEkv0Jd84OE3Xl7R36+noHlXa1lANoj", - "VgDnGC6p7Xg4D2sI+GErUZkV6hFoVkY0CLCda2iSgCoYomDaIIBjkDRyQLEcIUK005E85saQe4l1kqi7", - "JkTzVZXQeY2AQZ9EzUJdPI5bYUY2+OxohkU2xIKvqk05UeEDWWAz9S/Nq/ZWWwjldqmYOWwkmI2IkhLZ", - "bGqskgDCyS2y7ReNjdx0wxNsCLGMs7f72JACsG6wtdPbBlRmdGHJWOXXfDFkP+prNuEmUamxNlyGJZAQ", - "pQr26SCCy+34k+VYqvKmVsc41ZWC9H/9cEZnB9mewZKE8SMFi3/BXf+4OR9/OEMgVm8ozbm59D5cXbgh", - "0Ob4onom1gxaPRdYzC1yK6ASEhZnPkzUoVeYtR+xcP4SdkZrObUX/7jzNhRmoEGepV07fOCKijb4Odaz", - "sS7BKVC3qN93cPgQIqXtPX3F2/o79rWa5BLhLZ+8l7ihm1FTNhVlXZprSvOTtXUJdEUDjNbfK7zuFXx+", - "5V8B7w2R/u7B9xDk+8df/wvyav6/Roz1fO63ekbA5H67jTk4u1X0veICDKYH9loyzlKcnZTNeYHA9Dn0", - "EwJCLaBYvrCBVbCVRqBWjpZ0Dr5POmyLJZ1DdeX/L1EJWLv0jkmHFd5sVeLGQTcN2Eg1APTbzg7OwT5O", - "3ybNkMZALdvtEAEWr8TSujxPSOXUe7RixSt9Zp3MKXbWkBTiEy+o1ct60RuyH7xAWKYpDR6zzKR6KaxN", - "GR59JYzxp3b3O4ioxYCal96tuuz6UwVk4q2AapkDLSx7/+E8No9iEAElOxyqlRgGbNFahxqZ/bAH6cat", - "8DBsBeU1w4GdfDxvE8CTskUANxDLro/xEXjjnvqsuVP88bWypxb6R4iHn/HbGySI5qcr9IASsDrs+KER", - "63bcNhiB0GOfYbLG25kQNApPJZoBJ+fUlMfZ2Gjl+MgbrQZb/ClC5jfMhSkVBsXGCFpJ0J3+IOBzb+8Z", - "G7HbAEo/NDhOJHqY2LaAFZPYgesdWuF1MmwkTDCFoSPQnDZQLA3Og7iO/j1+5AsIdHHvVhiEbcukBdAJ", - "xFOrjiN4dSgLpAYQjhMWGRVwSGQFHvh1MokS6koarSDIl4kJL3PXryAS/NRyVaHU0bxTz0miApKwVzka", - "gCFn3DIN7ZhSOb0qlH8WF/6B23CJ0JHbT6DOCC+RnXN7eSdpIz77z63lIsuqv2IAh5u+PgcbkgwBNMSb", - "PuBKcoqpofPq8JTxm+bzN/7WL4rPxa8rQdb2CFAAj0rvIeVi4pbamPZUfVfB/oS9aqG/yNtVGG4HHgzY", - "pL/bZj+IkSk50j/AHp9pJfxX3/B5kVMMrmo5R/ST169epc3eWyoiBDxg7HPw30Vd0QHZAIf3WsIO2Rk0", - "B3jPmJs5teRye5momrMbS7jDFFdROsifRPxU0exJznPon0atdClEgVG+qAGhqbkwAvdnU8tAyxRgg/xF", - "KzFk+6grExVw3i1COu14FZtrFybaWuTaoI6zUB5CbTPUKgqAC/7fyPjPKA0ygdBBzp3oEw4sdgn6bRNi", - "GuMcitzTsV+9i7IIzODaCqyLgUiBULqczqiBC8eE6cmpcgWJvChax1nOC6cLxm0uBCRntrd3trdpocL9", - "3pf2v3FAEr2lyPAUD1rkdnHdUk8RqOLINtxnUNydZqNhwPAbAunarYI8YsddXZD3iMV291GXz2NTLSnr", - "9p4t1FCQSVT6Ojh/9kmd9dd33/Feux90qbLnd9YJvIvak5ZU/edp9qZ/sCaiWpX3V8XKWOjf/d22ZUQr", - "2eszJwzygTYQiBKFJb80MyFjAyXBtUCb3//WLZmswyom+2r7VT3KuAspQ1arYAofhJBg3tDUI9CZhMpH", - "ugIDCPDN8Z9BjRiurERQiyM1wC74GrT5iOiZKC011yG9HHwsMDYnXOb4WYfGnFWoP9CBDbYmoCgMMiOv", - "hCLLteLI6qZjeRNZcxCnO1DPId1Cb0X5kX+FM5yETaIs0Eh7dKC2G1bkc8Zrntqw2mzbaa0ZyNv4o7wC", - "qQosQAFiizOlB7q4FcioPHyozg7I58Hd/9zdXBEUkZd2K7ZEEnIWmHg2dxA0Bmpr78CNGopKn6krgyY8", - "WEOkPT51/oE8Y+W0eyv9o91wf6gf4K6KWH/Nc5dE8TwnrpG7A3atB1K9IBRgg1kqlQQiqkAhgnV4dsYN", - "livp0g30ZDDiKqM2YyWu4S3AFc/5dCoylnrtfIEOS3wUkbKAO+XV+0hQRqtOXSLdEmlJa+rGCO6EX4JN", - "pW3iAJ+Uu3n5qCLYmrSBF8t+a7mYhmwfqSsA46wk67N0yNYvMvsVZR5y/re7Mbgd86yCGvB3vbDtEJ1D", - "dhqw8DSVVKAxBORPAJTvTy6iu7klsJghigJ7VzkxXp59uUZ8W/qL32ut7tOPJbNPdf4Kb0+2ZSRqazbX", - "DmhFyc5g7asXYwRQtbfaB96g8qkGeKbE8SrlQzH835pv2eYeGp3XhSWDk+0BeggbTAbhKAQfcRP7oB0y", - "Gg53UKfwXoPxTFuhmBPzQhtuFhVhGEfgj5DFgx0NlX710xkidNge0F150Pf6xLpKYaVESdTrE6Dz3FML", - "2nBzLPMQDjrc/Yh9JtU4LwnqFvnogmrFmIeZChf0tbiNrQzlt0B2t7K/xMv4SViNDbaY1Mf5wrZzeC2c", - "/d/0rj5F+QoiE8X9EzY0cS2u9Uz2CvkHf80dcdGU5zmGdHE7auCqC6W3mIMA3Jylo2hP1eubutqwUlnh", - "eo2qXUBV9x9JDVohwgrExpVuAZ7ITluTM8/zlf3Mm4I1gXm7y+X6g1g8t8c1X1Q4NNjmleM/5ATXsiFF", - "QWRWO2D1up1vvoE2BCdu3DffsHRS5vnFpVikNczYsag4NCKZXRPqyc6glo+oBzmhVQN3E2HYJ53QlBpI", - "XRIMFC50iY6ZFYQCAJU1SSeQYQ7ZWcWairhVeDvKH3IPFkZM5E262m3Dxd6o44ZDPJPrhoNHR61djscP", - "9eMe7GRZWwYfi0S6XXRbdOCdnhWgDXsFQxVe/igORT3XihyqPVVrGKBruFok6lIAIdmVvqQUXiHMnKsA", - "kgjir68J74b2A8IBh1JSzFySCXDBXRrBKMpMOuYMl4CCDCjN5kpkfeQOr5ECE0kvsNxy5y0lVwuxw8ao", - "xadfb79stzT8G0SB34TFd7cziS/xtTiTp0EQ7i+VbczBd1ZKpr8kHSgfvoi3Jp0dBsohrfo8G1S+1O15", - "S+diCaNFMNUi54pjM9jYCKEafZ6sm3SooAcoFzBRAe5pkWsqvW2jAf6mBv6nskhtlXR6Q/YeyZgr3u5I", - "y7yiKPL78MWbD10vDbXueI+XUuS4QR3f2fmPP9fF5KcIjFhfCKwHh+Ah1ODHpWXdAjisG8dz6WYtkoS+", - "TNNTaz27/yiMnAAnK6Xnqphpn5UF4YJplipxXf8J2+gT1RojTUNSz++CYAuiBxQIWqAPUtpEYbjFVXzn", - "0BcROiCWviOUWHopvgQmXZ7LK9EbslgsCcBhlX3TLDloIw9vPeNh2A17Vs1BHtq8H/2g8qHxjUcKPtRJ", - "Z5Y8lrvlF/zy1VL7QYW61T4G/dMz4Qb7IEA7rEal/x0mTGWGudLdyLu/m6gzPhdn0onvzpyRY7fLTrib", - "fbeVNgGnQD4Lvsg1z6hcfJXUY3gFcJ9B8/pzb2lvh96XEIuoJJv0bEVpQxuGWBXa6vFgjjYjm/DsZ/L0", - "aezVOvYYqOwZfDxSC8M7VCLQFjxCtUM6phvEoM+WpKDXWWeq/PrUm2rFwXF4Q6EsAEth31axgImGou6l", - "z733uZHrqS7dfdvpzJUwAyAiCQMafU3du9aZcuzwSqA0CzE4qI1Pa3s0ZVeSr97Bu+wdvxnsTcV32+mK", - "beBf+T46MkgBlNt/poJsqLrD0NFLeo7e+e55nt8PkBaUD3cOQSIowxOQ6ehjPpyG026YqCMIOfrjvF1D", - "3WpewUrsiNCsEwWcSZPSwB8Uv5JTwsoPXfvtmmuFlfZuo22z78RaxLXa6fMYqx2eV7NTRYZPv3PBQ1j3", - "zmVHY0lpNQgNmg2TKYTG+t7nFdYNwE7sE7hGmnPrLqwQyvuLfVb7tyzIKqv9reRRIkDSoErdFtqxUk34", - "XOaSG2IZRNylVNoLknU67byzGtQBvCay+JABB3W4cIisxIZZnIWZ2WTtCY5xV2zuLLIHfnZ8riEwe42d", - "GoHT61bR/SWnJV7Rlp6NE/psrvpjaNmHud9eLRO92nxRTX/XyqliSP5EgAqZuJJjsf5gnEo3MKLQdvWx", - "eKSsQD5TKs2DTjHWRR/wu0IAsVmvzzjWsvvdMZXuAh6bKBOYlYSCQkpoSQSUCLgirbOk/qxHEZNsPAPi", - "N6BOxLKXTNzQLf46Hc1k3L4G2mgJXBYAFICeCrj50vC54QvSQG9FxTSAX1idREon6lqbS4DuQT8rl+oS", - "GQKdZhHLs3bRlfS2cxio+gGTi9XAcsIyYcn3T1Tq9KXXYNgoE6SVOF7lFXL8FdrusvRajGZaXwKyc5oo", - "aoxBD3bOVcnzNE4FyFDEV+deYgZ2pl2i4mNKk6fs2+qxVoyNqAJxMfaBwT/sHsGKCrqDScXeSvdjOQpI", - "XKyb8tLpFDlqgENFukCxM2+t5dzLsrfSnYpCb4qTOw7wTNFmGn1NuPmEBJZCzuxbRFEJO+YheubL6dL+", - "Ke6IA8rY30K189/8bcCNCqIF0i8hXMctEgWqjMO1tKlqCi5ss1tKblaOBrDT7jZS5sLxjDsOckvsrk5D", - "45R/AJKu8rnoMzvWhbD9GvHvMFEnIUWEIWjMdb8//OPhadUuA3XQQCCKjJ27MdEDz0pUzDNBn1yAsZS2", - "kVnyG78BftP4zlVGyVu46BznYoNmSW2cu0wTuOhhicPHEUHIINJik/id7J1b1o0ysZyHborW6jQiVpDD", - "IRqXFsWpSbnPRjpbNBgMhBqbRYEMBRh93js8G7zdfweeJfROKZ5vofbGkrhAakASNROJGstiJowfdsUR", - "0fjCmMWpy2GiAoyKVM08tlf9dsjO/HYIjAsAXl4DTsbpTJR35wDzaCKMCeSfOXfQwAkoK7vs5PQlrkJg", - "bfBC6J0F2G+JCqwfkNNVi9WJzJoMbjSbWRvn+Q6Z+KUrdxhK9j/HaXKGqGSQPa22MuvSdhLZgHvL17p1", - "u3nVGXJnevUk5EMBsAgp+pBWikYH+PdGtr6yGEPeCex67B0UwKAwFoEiChRHVvXgUFd5NLuHwapkP/wB", - "uaU+vGcHh8eH54fs7PCcvf94fAztnKE0C+x0W9GowghGXOlAJlg6Sg8bMUDrZCvYgYmaADwRvCpQH9KE", - "Q2q26hNCgCAeXh8h04LZvbomd3kTb7409xNrnx5HYGON7q3zZ/1xs4kaxTuxQyo5xnOH0u16ghlWKveL", - "Bxzc2U/UpRCRXR/QCyTWM/o3xXMJWE/qxs8tSL1E0cx0oR+utML0YgFOLi8FmtERRQHGwinF8wPpqYyY", - "GAGwShHi5E+DD3ulmw3O8LJ4RGIYfsi+r3G4ywwEOLZ898lSFDd4HkcnFPjwMLoZdy4S6RMFOiITA43+", - "YCSRJDxmhyu6D+qG9DoMY2eWIbu+EVOh/K5p30NYEbz5g/DWOM+UYLnXQWi0A5Hq3rZ6GsJHgLjjmRZZ", - "77fYsPtq8/2D+5FQG/gwgl5zOm5tLA/CFXssbXoKK0yxW9BQXjd9km5de/hvEWT//fg0/K6FaqsXUCpl", - "tdcQ9ATMu1IZgZlyJS2xm4c7Ea6VFOTtmhZQDzwQzxrwYEQRWZR5lkGED6q6W4M7mfFHdABjpdBWosL7", - "Uda2kONLZA6oeeR+x5RWTMocFRSo4y2K/GVaWPXCRfa++I2I4I5Ecmd7744HhdHEfKTNNJQs8qIQ3LAS", - "wMe2/A9bv0C8/lccoBfBYv0kVfsWN+0tcvzdpeQQDZJlRlhLV6J6Hi2YzFa5z6BA9sLiPypQTF2k7oUV", - "g7qOXuY2Vky/A6jitUPez9AUSs2bMDK8+h685T6IMnsIuO/3Fd3Oui8x1/It2x4O38Ni9p7ODiMluFl1", - "FqNSxE6zpLueTaE+chCkctmr1b2SVo6Qjjtq0qc3T9er5ZjLuAcIY6Er7Uw6ts+0yQAkabRgcw1MrmNw", - "4xJVlN5cRD4L1kJn0VS0TrNCF2VOp5A3OAtNFBeguH7y+jKlyYXgf8Dca8T4KObHJxOZS/QJB4ni06kR", - "U7BhAJ2rsoVJN9a4AWDg2ncmygrhDww4kfoIxz3ScB4A7BeeQn9Bx28u5iNv/PrXTVTjfa1okC/MpLMs", - "DR1VdU2dIgUetR+Gc0Ublrao9RTLPJR/jT5DmFtwUHU1XRdA2ROs48ifMwADPyNQ4Vvn0VyCosdH0xnk", - "FoUc8xzGbDmKNnzGsI+FF5Q329skjljHR1Hi7htE802UnrCX29u9ITvmZuqnsCYNzM5AIRgBDQgE9IaV", - "Kw5oPicyd8IguDFIIONsDozQIY0V+IfWnXmnsLPu6Jv5UCArHpTSDqSyAkBGroBZEfcww9eBHvIyzy/A", - "91vRAvOfawuW+itHDyKGHXBOo+dHnK7BN0WZ9lLcrwQbJQth8nhuNRv5tXWru3Tovk970dOQ5ru+pQSs", - "cLtMThVoV6iSu5aBfWbN+PDerc1ClIjXZrqJnqGmBRO17yeYL5D6e4DtYkguP8dwiZasfBAe07/slGez", - "U5aThFJ8sXbKFoVL7RaCLa+xUgDIEhMwUNwY4JkJEh+z75nIJRzwH0+P8eQADE1vHBAANMEiIgwoMklM", - "jJ7vMI68FHOu+NTLRqmUyPvNnoeBP973j/50cfLx++Oj/YuPp8esK4diuES4NOdZfM3RIlFSTQzHAsnS", - "CIIIuhLGQr72ZtFnUk0NVjc77uSYHZ30wO5QWiF86N7Sm/lhPpycH314v3e8gzpz6cVQcfbD3Fgs34jU", - "mVwt6FHLTjSCD2BojvErLTNsOVIYFE86StOdSQfzz4XRo1zMqwYUWhtoVgLmTrAOYRpW1A3+hG/5AcVg", - "g8Gw5kBrIa9vSRUJ6WMUksZBmtLsbS7av8uj1/Yqzn5rQKbaUSbw7ayj5BlArj3EWhrlNN9h2UoV2Xhh", - "l1+NOyKHpAK3IPLcCgZ7IzLGk9D6P4KQkED1oSk4UU3R7Q1ZRdA4ZKelsiwA2I6hMEljlwzsZggLVY8n", - "yvjdZjVBxeAuHcPCoPZOkMhSRHJin0AW45iraQ/iJUyXzqutZ64KOBWDUP8E7cS3BCcY77V095KUfDw9", - "vlOkjS6L1a7rHlIietcN5Reu34Vw3LTMuUHnCli8Q9ofr4GDZJGokcg1GL6su1wp/cKypAMIUv5nuAuA", - "/IGPcQyUZejK9lZWleDbbzKw70e4q5IELnq0ElewMLx/PZiGz4sWBP6hXvDRWoDgL9ts6YEf4bmKDuDr", - "Vi7D+J8ACQsXgfGamKyCrIgic2vTfxIOFgZk7EwWVOyDicaq4jRAwYWiATBkojZYk2ePsvobBb/6hCXq", - "r4SUXDFL20+0qb6SOX8LWC6fNOFr40p/rJ50dLACkvoBmGStKfMNqu7aCM+VJl8lZYEFZPplS9tz6Hqc", - "msfQ9VukxNdCFcESvaMLNy0KOM5d5hVe9bQQPw9TRIgJhJOIJQBfvFJqNSf3sqy2ThtslagGeWjTPgkL", - "zzKRsa6MTm7vN41jtpdlAWcT4o+Ppiu2fvEPPVrfJncKtabLknLPlcJC1WdaqyWP279JmEeiMf2iN+6t", - "FA+gTx8dAAcUfM2KcXBRPzu0/LMerT9F/t1f0B7fXsok2QA1czuHRDlY/5ZIH9DpdwJTr393pIhtyy/1", - "28e6la265325nMtmro264Do7L7e3+505v5Fz/85v4F9S4b9e9m9nkTaJlvfvenTXUfrvevTFdLw0OzBt", - "aO1kW8zPGiVs6xutqlVvqq3YfrVOIk/CRRtcABrjrkU4iZ2iD1qI7btvOqI+nZAta0WKNxVHV1FNUktv", - "2/qg00lsi9tc2InGeKbAU/jCu3spH3p+bTZFel7vogpFCzNuWQq8Vxe05BcB4xjx+RPVHXOldPhIIskK", - "8tEbMgoWcyOYuBHzAuoXKp9psx+1F2vfCYxPWpbOtHUX/uRLIxk3tAnYh1QvP3DfncagPrYd3LObNPx1", - "65cZt7NftwD2aGCdLu6HGe3vehzU6B+5yQZ8FJPF40YHbSELkUslQj1VaE1IVBdTW9jHk/XClw/Z61ev", - "qrxmWEUZWNigDd7/X6IiEgAOdSU53LJ/fAQxyRm/EkzpBopOfB2nE+Vni3VDM8X+8dELKEdjY67GIt/a", - "dyYf7FPx1bUmGlzbZyPtZmwkrBuIyUQbt5Moxl4O2QkaKFuB3agBMPDtLfAASz3p0vr7GcNiML9d0LCu", - "NYUg+RMlTOI34P6LNzvIlwIEwVwM/Z9fYaKZ4GykGkQGOpiwAHvSxQ7GHgAewLfnIuvTc5HwslSjXI8J", - "hwRwvYE0CZ6zNRJTqRDMYJLLgkgD4e6wfHSWB2ZeHpircviFEu64JQdjPacaxPGsVJd2yy7mI51TD/OH", - "c2a0f0F8WHeOFFEYXMY1xG9gkZyvR+yjNdhOu1DjLaJ8SpS+EubaSIJdakXz/8HvrzOniyN/yybZnuJI", - "62yGH+J2Dzw6T9lr9pwtlbUv58rLdJ00bFnLfLY6DYgkdxbxBm0XEvShl2fIXm+/Xq3GEtX1J6zSFUgJ", - "M/q6hxu2CYaBCCGAOpWRDqiBgkJilFsnIi4IlQycBUbtf/z1v1jIra+oBSF7pY6BsbnOKKy1uy3TjZn4", - "2hongSgd8lvxKxoQDvcWyv6mDu926pOza4kUezUpfWFBP/oPmOmMZdII6GuMx1GQ5oJPxY5X3YNYyIJ4", - "9SSCRWlnsRoqFIQEpr2hl1A4+hqlDC946fQLxCAPeMoOkGt1VQDhXwKbfhnrIprrUi3WVigvo0IVxDQ/", - "2TsnHC+GoNI7fqku/KN6Q3Y0IecHNwf0C9t+vdJswvM8nmL+IYXOc6SfyJjlC+v3p1QsVdqJdAgTQ5ek", - "Ec4AMqNZ5OI2bCJwCQB6HcEUrryKYKwLd1+EP3mFoFVm0z7TVGXcw2lcmsNgqr/AVyBcHgQN0VVlDyzz", - "bqQ+1Ipl/szMVi1NfGw/frieTODwRjmCqbjmJCqVTNSMJZZVhSwWauJAbyKXOM4ysH3r0uYR//wObbq6", - "5bSp2M4WarzpztMwznOReNx+j1UVTiHJBns9cKXDmL+xKuQjBU2e8KEXqNQeU/FHmvnACX+r/ZqwlL1m", - "sCiCn2eNbGHt9OYcvLaWewQbtuQINECIuultrIe01zYFgEINihwMpi1QedgZr2PEKR4V/VgzC54QKVhA", - "asC2eurejy7jCbeWRRCzHabKPE+BeUPPpeshcrrj4xnqGXx7Pzqab6ETi3FLtXJLXmgEf4mgF5n2PofS", - "DiMIu+sVFOvetvRCT0/EMbNI4M7tQNodxlEvhjhFLSqnI+AZtPAniupVsWd/yk2Wex9PT2jJ/KFX75DS", - "eWYrnIMKB6OstZSBi7e8VKMF6GvEyQmvbWp1idrAywgnWNeIgfUOODLTVNYErHd1CzbWuvrL9O6rzp8E", - "SSAO9HxQAqsM5lgl8UiG89fM6H2f+lmsS25T0NwxzjI5gVo1t7ZB/34aG2mTbivq9jCx19zM37giPfdo", - "uvyDyhfs+MP+3nEFo9lUTYUQpgdOJXBmc2vlVIkMOzlj7C7ezA0xuIDKGS0AVXKqiJwA2H5ev3rVXvaN", - "z6Y5+EA8U5shWMOhYIxn2sZrkgVhGwe38LdNsIZLgT0rQP0D9TCN5MGqzPr9th55M08dBj+sgGUB06cZ", - "/f1ZjwIC6Sp82rsC336forkSA8BtsfDgzNWi4b0hOxBZWQjssi4s1OUW4FklqurTUAR/HqGTqujaz3oE", - "Bst7bebQD1JF+v2nZWIsMwFhKiPmQjmesysLrbXNPpJEdevXIJ5QwNUV2YWdcYKgHWuTCQRmckaI4YGc", - "TBIFYLsis7v47EABPYD7+6zgxkmeD7znXkIX81hfCbPoJ0obJgKJ/MC7szk1rvSC+eifSNzTTiOPhF/M", - "Ms+95Ymzugw1gn/NjJwQSKYtoNMR7avQC0OY/JbV7OXGF3sDaWa0QqeX4vwK+qZhkr+lZApHvnTjbKhH", - "8Q9WzOYYRG/RsnBnlYxdex79VJFqhBduvGbIV6CxjJAsiSKAWKDWrN6cVdkOZkqFOLw4nSwG3r2VCKsS", - "+qEmMhfe+i+EZYWR2izlALaMmNgt6CYXF37zCtujDnwdVhunJi4Frs7q3mX/Ru3FHBOeWxGLNkZa+7lu", - "Ldp49YhnFUwNaZNsXQ6ALo0AvcQ0SP4C5aC8wGaw+Xv/HPmBd9Rq5r2MEHFcnZb85NMF2xruU1pyhldu", - "umb2KLtfO5LM7PK5QoFIp7+mMlqg1au6oVbFW+K3PbRMb6M2+grqpXzBDv90fnj6vmGnE1Pnsq0+5wvA", - "h8AP9vvd/y/02/CI87XVNLACEewK2xxElz78XG+yBwFGoiEeWup7hjPwT1PkC9/bKv8PqPlt13dbv0xR", - "16yt+/2obE1wfjB6fv9uLrr3y6j7RWrJMJ3/+Ot/4zRib+qXqk/6n1NgTMv62ZW/y+JCUcGBVBN9LwAs", - "DLfmiwHAcwB/Zogsfjw9jtipP77b2ycMxUQt51NXVhKBJo0KFPVnoiojPAUVil7DWBbcAW/yLRCC6GYN", - "/OTVfK1QYYS2RS4nYrwY5wLRZHV4UIz0zrjKckiAkvbdfg0o5teaZeBxjZH90/aBPhF8sFJamBVAngLa", - "C2nEDuvyHtEoczcDQzhlAenQCKvzK0QeUYsqAs+hvhMBObqjXsMawDI4AFiOSTW2D0WBCKmcKMBUdvCq", - "fD6S09JPF0AUgUHNUkD/WhKIlMAggZdMq4k0cxxLqDFS93mHQ0D8rVaeUxcjmCZuE5V0GodYvzbFkFgI", - "aZiks77KgWohjryIbh5qwA+zzjqjy9hYa5NJxd3DQYA2m007lJFRNUgP+HYxF9JnH05XCFeiGuGM+k4E", - "hp7mihJkZ2+YqIO60I0WbDwTCAS6Tuqo3vRx/Iqfalrp2wDn6jURJtdDsNgKByfek1V2tCpj/9C1vdxQ", - "mzvwZ9iQHRhdNH0DAIOVzjLyuvvMu9198M4Zet39RAGdUgip2CE7EAi4I68EE0qX0xlCBXlDRJgAixe5", - "oiCjF1lkQZFUME3SrW4Rr1eW37NJHKRtpLPFF20RPriWODaZx7yGyqCmX4YiWZExKEm6O8K6uvl85fxv", - "P2Fp/VPWhz1wVd4Kx2pEQUhTBdv8PkqibdzqkjBTP/oHrinzqu/3WBhIwD0IOzozUl0iqrwXFIjmof5N", - "VFfcQPnhRcGd/07bZ3N+cwFBOCv/Inq7tMlr+3gkGEf8s0RZmSMLRSYGgVgpGGl35Xo3mt/9nAaSf+WE", - "HifY98BddeIFvaovDzL9makjODC3JgG762FbcAXyV+X0kPRQLzicqrG2st6tMJUOLR1dIsS1SlQMDAWv", - "Z8THlw2vZym/631WvyHhwVmoQpQGg+gWqyT1EpnSnI9nUok+3Ek/Uv0I1NLX0rvb/ytRFJpKM+G4zFPm", - "BFYHVs/E+DzVyMIXU1+OkIbpa0Vvs7BOzJnTOrdD9mEuHUsh35FupUJlaeMp1zOd47N2K4zSREF3hv/o", - "l4MRt8BiPc5L678SkmaqhE7mIftQugL9nTEvCqx9wW/0GusvYgsuh2ZPy7qpM6UCPtodhkkiSADNpOut", - "SG9nIRDipWozWuwHgDDj2TMpMD/8Pj2orUnBL3MYCFTLPfb897WvebJ8xVN0pJ23JnW9bOE+DPu0sdt6", - "wK/ibKKiHpDoai2EY/yKy9xr1GG19cQNAESHej1E3wTYQM0yDbThgmehFe+5et54FhRfF7exNrg7DaRk", - "Kbt5O7T5wkaF+MnavdastGklz1lqSuRTDRGsOS+8CidIvnwxoCIjEjpyqxLVTfEHym6mvZBURUhtMNb8", - "K5Zg6Gcid7yewN6h5kynIYvaaMASgUUjWHhD5rURoL5TOXmbKoOOp+9FQAl9fC1WDVDTY5vUW/UB7+aq", - "1oVQv/UeXtwblJdHINza2Y3ku3TsRjeGwjlwuyMg/5au38q0mWgzB96BL1dJ76ma+0PrL23MqlMnZayD", - "qJvnT6ZK7xnRAiEX5qOKp8SSFv5QCGo4aX5wTbGGn+6jWLHrclOaFV6WItp+fwnlJFhjgM4yZEHzpr8k", - "ndjDmnTQVvsVCK95osKSXnPLLiW0ubIUyjzgCuXddP8brjMW8ewfH8E+sNSqKxXyhg6g9KYs/IEtuMmh", - "l9wBn+EUS9QlHObokFwDDjlwNSTKlIphO633wIGOQJvoQiNDod8wLwczXRp2fn68Ui/v46xvWlniMOu0", - "JV6RY+eUoaaTryZGg2+P0hW6pZfUQCMx+XlbBAy9Te2QM6Eyb3mMwDHWEzSvCr7INc8sQ+BdJJ0I7Fsq", - "minDRL1D1Br2ZpsM0gI0f55D/uqbb86cEXzuH6DEVDsk/vjmmx1mhcpYitzCO6wuaDcDlXlhSyEKZMRY", - "yCuiQ/XG3iAT4F2JjFl4uH/r9IgK1gCe/fBKKJcypFHw1hEwkl8BqLFAk7GP4WrO0pngxo0EdylVk73c", - "ZrY3ZD9RMwnmsZBIEYqlwAVtfXN4614bCU6icjHl4wWzUk1zMfj3sw/v6aW9v2PDHkkrChU+CT2LsDaJ", - "CqhFduW2hkfdVa6Xts+1jS2dSD3gZ1Zk8TtoElvnOcwpVIxCFdwOS2/NS62WDiezyl/gXLbCN93SQP1O", - "2/uv5AfZkN1Ji/Ys/vNtqQG11DotfiZvuJcbeA1cVvi/qMTeH4Aw0l5q2SpebwFHamen80vSgR+Tzk7S", - "wUiu48b5Q7OfdFAtwG9m8BL+BLlv/4c5l2o41fBHuBGLOTs7L/tJByQcgsJJZ+fV9q+Juj0QlHTSQK1P", - "xZpP/8RXrQ/ArNM9n9BPOnD9xdz/+83r9nfKtBKf9UJR6cCFzsIfX22/+v1g+/Xg1b+dv/y3nVdvdra3", - "/++ks3wrzlUcGbTuBYcdBLbLq+049AU1wiadnd+9/rd4ccR+uADiGf/rtv8+PN3uL4MNNbAm6ctJ6zEU", - "NJQ81qWKWQhW8JouR4FMFHyy9a4+FbeTL6uBvFcqKFZff4L0otvwpebFQ2GP94cmAGP24ZThPqr9bSv6", - "T3NpoRPgmZyHTbfcgvPBgr8JHuXbk4/MykyMuWGj0i6I+8r/b5+lp8KZxWDPn5VpPKWJ4I3iy7acToX1", - "MnPNpWNdaoenACzeAtqx9qzmx9wC4Pt1qaauHM2lW7aiLOvO+Q17s/35hp+SdvZ4ll+rxQBDbPSk9CM8", - "71GJb3B3zCZiCH29OqNUl0pfqy9HYzww3LAPS7KUYX5QxIGwQleVGf6EZOX1MA64djuxAG8us4H3xQs6", - "/ggbJC1m3Iq0z1I8ZTNpobFEZFvxwN2CA9df0zyg036iUgFNVlkNroN7Fyn4Wqj2AJls+dUS1YAYwchx", - "xQAaMcRKFUqm8FsAigN4HtMly4BeFN9g6V0B8qQW00sUgRLNpAVmZSwZ3IGoCs42GC4yy0XS+TVd6b6c", - "BRTXzeqDYLbcga6Ja0ueMDh+/gOerB9mqYfhSlTypicNnh5TKjgoc26BKg2hbf2f23fIw4pF1uwvK7gZ", - "zzYVqTjEji8CtvNipuAjoZmDF4XRN3LOnWBKcCOsGyghp7ORLg3DF4vsckuoRVdiDJBYOs/F2A82ZAh/", - "AvHoRPnXGSCQLKZ707lUF3asDex2/+029WaqdCKHSsbCiIm8GXw4HUTq0ESBEu71WUplMf6eUc7Hl3iP", - "5fOqybNHez/nalryqb/2H//PfwNanWJzYaZgADvtfbQBRGxiX0vGDPd+kn/RkbAOn8ngdSEmU3v7CuwO", - "wAgHkcb3H3/9r5C4JyudpdvDVynrYmOnEbm44mos2CTXENbmhCgYidNj8Y7RBeN+Frg/srgrDc8H4cNg", - "KaUgLMPrmbYC3xp1Dr62t/X/Y3v46k2fbQ9/9+bPPXxZcePVgPSvlsIbU20BRHEcogqN9JVgP74/+wlf", - "dOlGYEzzW8vfDVWH+DmA7phuD19/i92LfgnH9IFjnYkBVjiSXEEtVC5HBgLL/vp9nYlTri5BZAf/+//q", - "wbyD1F44ORcXc4v9qn6rY330S+iGnfOcFTkft3ZlntFineE221BrTWOQZzLbll9ijZ5uyD8UiuKtFEy2", - "X3774hfrjR3GKt2aQ0bKErLN3rT0nj2cPnUXLVHdmi/FyCuzwt3pcy3b5WAJ+f0BrluMBFAoBzw9P2Ab", - "Hvtqby2ISBc/pkf7uHZa0h/WWpN4zVYmvJsGdNab8tRwGxzUBtrM3q9GeKZ9X3+BNUwuAQmhPvW/wW3e", - "LAfWA6cH1RdD2RmeQhBE/yzZfeTMUpvUhpTEpurYnvWcqr/APeSVUmtu9tsXVz8z0PdWI89/iJYNhICb", - "S4TWK21zjeJgiYI6xZ2WejcDAC44OB0yE8rJiYQ61UuhholKSa5ShN/1/wsVUvmCiXnh0GlJhcouoG7t", - "u+8QmAP+RTY+8ZXCjClZFMJZBm+BxQAk3QEUA2QKoNF45r2BRKHhs0vRcsvsDO6b6DzX16wsMCwa7SSc", - "YIQAx1odLKqNAK7tpigKfVyUTUEw0QDPtL9r468D1Iiz8Nvf1VDBHb6XUsWwNz5vW1On2GaPoDMaZEMO", - "Ezz9ed2lxivc4yAK0/5bl9ezupvuLSZvKrEuhme24snU+1ThDQP8cle32xldufl2oDBSW3Yj/PTV1FWF", - "BIe+EuZKimvWdbrwBxL0lI6RioF6TCFQbXub6ItbIwLOiI31/BxD3zNkPOZzkUnuBBPKGSmo9QdPZm0W", - "gANwu/unVvYP7T/sru4fxCqwg1xeij50HebiSuT9RCnAxyqNhbAottZk0iBKLgCLgVnTizB1iNyldego", - "amDMhdYf1rVCVD09AfCsN2SHypkFw0Lk2GKTqHV9NLsIaTCcSpfWp8aGaOYq3vcAVeOXcjNng3/0ExWn", - "H4QPP0YEiDZFEK8B0jC46F/tNS3tNdg8w9b3ziTqruYZ1tY780iMazVB78b92vP64JHbX2p0eXcBs2D1", - "ZMXlDgH1GbeNonfGnePjGdR2XCuRAfpzLtVlQD+ss0QwhP/2tzvtHZlrlnQqzIWkw8YzWRCLD/BtQp9L", - "LjEN+nM5L0I6tHotXDV4PuRDDsEv8zuiDUZTvXCA9IKatP55XhzAYRNY5SQdFIHHhvnAf3EleQU1ESHK", - "/WSxkYDMEH4+RFFD232dKR/wYkZCKEIjv1OjVTP0BIZHHOxYWreaeBk+5SsyQWCbVcKPwuznjcOZW+vi", - "2KzhcUe1Qth7lKwL5DE77ErAmd2vAG5qsBsMkEz7Xt696OL+yXM+5wN6UIjmA2pngFjqpnDfRa55JrK0", - "16euWKYniWohgMQkZrym1sUWcCZjScfPerSKZWjzVQE4wtoCISQio2KAzxTjpSQEzPNWnGniOus22lSW", - "zNqRaCprZKm5Wzqgphcsq0IYhgVChdFXEkrax7kus0nOjegzNTVAanM+E4minoZ45ZgbMEKh9xvfl3qo", - "sXPL237GiQxwl8qqpIdtsaQz1nPEetWqHWrJ77hz+qANLjYOsc8dz/V0RelH+Fy65uGrjRiUyCkUptOG", - "Jk8ZFr+22GFl25Z7ayQV9wuxZt0LIFak/c0AHCeM+8IyPvUuFDxmUV//LAgAMDsURliotyWFh6qhD+eT", - "Nw62oo5Bi0dEhCdSImOuEuXNH57nWyVgU/hTMnb2fTxi3TlXfCoy0EgIQylcoJM40ONLr53knE8BsZcq", - "oByjhxLSs/8LPQd5lzJuZyPNTQY2g00UoS3RbfBfwBzSyrIuGn1QSQIAEOuF8/sw+RuXURhpsepYPRFm", - "EHcmLSWJ0SMI7J6XkEHjsaA7woifKqpbv4Q7f92iVQD3udUBPtDXCtuP4FziTlhoRfaqpCG6EXs8SFG0", - "ioeJAhi/SgeBbUf34eVzgU6mUJDbTlR3/+hPF+cf378/PL74/uj9xbu993tvDw+g67VXQ4uoQfH9r/Zi", - "NvjA+ip27gPjVZvc1VBeoTGm2rX+AX7XruyBeUYpPQobdTdOWmBteQyZfSIv8fs2scki4ls0Nzf7FmFb", - "hA3EiLp+Cc1r+Sq+WvPXlW/vIVsalfvqHX0qBtk9N3XfmxE5HwcgGHrHRI11sYDKPOfdMf9TIOmbOGGu", - "ucFCEVOqKGJ0QCGkZ6KWlMGa3b4aX+pfmzrCTP1rSz/elibrqHVH43Sv2cd0CNKe+rxdjW7i/cB3/a6C", - "fhragHgv6+JBuuXH3Zpp6/wOCL7EWCuFpWOBOowpiIHEfldv/BEZmxUuJW+iMmK1EoAcgUAnK5xFsu7x", - "YzbvReA4bX3tVJtN0/q5W+FxQEjfCtc0iwdBRBoLGGgc2yRmBTPUCYpCxCfzYuAt8pi48CYYkOBZJt0O", - "mlrgBiIdCfLqwSh92n34qy78DX0EGwwRiyhUyGjIjKDHAO4nJifgF0D8uhSiaBLTaCV2se+cKyrHoNIU", - "YBgU3Nyl92uCtYFsQW0IHPSpE8n4BhRwacM5QT7DpuLHehNa1Kj3t5+OoPNxSEgfZ6uRrm7bVwirWhT5", - "gkl3X7VMIl63rJYR4+ACXLnOM8oGvUibWfA12APvdYDwwKl/MhMgGLStR3/d2m28HrPliKol7itKtyLJ", - "Kw7Pzcdb71QzS3HHEBrVVWcwdknMdJ4J03uUgEdzdnFEq3hhZ/reu9XbX6udoCNrkent7eF5sNnwzheB", - "QJZwwtOtmeC5m6W7pGHhsEmUgP4vLLyiNBXOkMimyB9gdOlEaAScGYLTDuMkGC+JsTxE5YSEOvaiDJyR", - "BcJ3/lxCnjGXV0IJSxnEtvPxXNgnUz9+rNXU1P5XOo5YV19+BxBN6OuVKqYxel+bImrI6aFfJD3wRgyZ", - "1PJKugUD0//2it8luYE7eWsq3awcERL+fSlIX2AgGEAHWffl79lM3HiTzdjexvmOTnDDhNoRFOXSzaAl", - "ZFFwa0NGOf3T4MdyNDiTU+g+E4NXb35fYQUA9PQIuUIGZz/uvXrz+9BgSfsOIODZpVhEouNY1PKiQc4X", - "aO8R5j8dsnfUei0yZsPoNlGxEOblrrdEQ8t2ioQcNZ6PIfugGGdo5qRFaWcp8pjAAhso4mEjwxVSVIdd", - "LSpSyWU6yUR1s2VSx1FprAu8JVJYJJkmZoK0kGqa1n4NZTyvtrexklhpyGExMZlAhttqzCcCpwEj+g60", - "gCa5vsakajv4LSA9vQVJJOKEu8CMGqt2FfCSdLboe1kcCDXWmcio5HnGX735/XfUnTlcBUbUIi2dOxh0", - "VjyHiq0QHuUOIf9ch4JnmcQS8xPjp9NBXgh3FQ2DMFhP7UvQAu4RME1rCgOcMsOUHugiEud4TfuY3Ij3", - "eJGDwNkTYHRYN9Ij1tgRpZdgOZ01KKY2exogg1GQxSaWw1NATXxUMVsMBTGgvx/qLYlxaaRbdHb+489N", - "YzcgvZHuuUWp1EUrqY/Kel2mvKWy6QGVTKHMEnDG+96n8ccCssUwzOQPrmUmkkBHdiWtHMncH8yEnB5A", - "Ua0Qtl5XQqAKgSUfYClX5B+fpqynUc+zloIoTk8uH1AX+Dh+N1bF5XltamsCUfsjhLJaPel9WIk40oaC", - "PEujfFLHwMvHX+T1C0vC+VCDef1N+1pNcvkwONzHECFcGcSZrMRolRS1KpatX2S2lkDpVMz1lbBLFYlA", - "4x3/eRFrBWtFgN5oxLYbzIvJmhKh2hv/5AyqDT+8ZweHx4fnh2x/72x/7+BwlyokVSZMvvBPqEq0msSg", - "VLOl1SCT9hIp6myi/AhQDgJkEF38POYAriGgKSyXOlIFaaLAAcuE9aLdW03Q1Nx596RoesqKwscQssi1", - "dKeArWZSWjNR20+sIb626X8rXIVGeI8lWM97Hjfg0QHrfjw+OoAGipBTiImt0YIUabxhlXcss0/2jQNx", - "U1vOYtNn2dIoz9T9tlZSAyPS9dNL7Fd1+FHeojpTQiXxp59/8QBYS3ce3+gkXP0UIkKD3d+0DX7Pg0zc", - "Z1J2YBNHYyIWXzNsN8Dm3cfUgY+n1NrD58oK4yzjrFvZSjLrh0+88MP2vDEFdYGJSm+bVGmzxwRif8G7", - "hzQxWD8j7/ElKsUswHcvqKXjRTpkByXKoKi3szUfKp0V+QRKFUrldAnhP+8F1rw+sKfAoY8mXg1/zLZ7", - "gOoykqtvWrPXBntuD4Veo+oiaduwxyDS/9LtK/SAusS2MKqDiJIa00EPcHeWWXTXuT8VZ+xdW7htS/UT", - "BW15QJutFfMeSr9BUkqShz7MchuV39JQwru8EXmeVzu1tQIEOrRqBPWf5qp8pAavr+38QGe1xgtLhfsN", - "37MbWZyjUxjXrXfvQ+Xhx0b/HimqR8tJrdkNFQZma4RxGcSSyMwnXFkGZBPXmvm5yXPM8Q8IHxGhXWhe", - "d1gmlBWsO9ZW+r0AHVtIDIaQZrYHW8AW3Pjrzv73sXSC/XB+9oZ9/+7Vm0TBLYTrOnG2N2TUTwALDeml", - "ax2QJHMo8fJbZVJakSXK+/mnYiy9uuI5O+Xqkv1QIt/J5Xe/38YM0t7YaGtrlJKK/f1vg1EuAPNwzFUm", - "M6DEAIzHbvr3v7H/+T9sNH/15kJpM0/Ut6z7cvD3v/X8n+GL4e8pZnP+/rfvtodv+gyIGyFCnls2l2ow", - "5zeJ8hfy3G8gaFuAue4Fyg8jco4Z1pkRdqZz6DCvXugf/+//hyCU//N/2PbwddoDEMval0AzIIR6mdKJ", - "ilg6xOCfixsJfcRXwuS8iJyV+BpDdlIaMYAPStSEq4Ff+Ogt+uveBwzTsJzMiCk3WY7or4niI6vz0gmv", - "Ax0HUnyr63rN6NJJJfJFoOPNEiUNwXY6hgEf7pjS0ooBdA8zkiYr5zLnRroFVh+gwEyhPFXehFbI0YKQ", - "iABm07FccIuExZQ8dddA4Yvr4jQw+7K54Eqq6aTM2cRwMHbC9X7CQWyAlAzRP6EpFwlUFBuVMsdxoVLB", - "6JFUALFkcsGvpJruJMoL7OAlKioM4tvSXMmr+qlHbHZcLUC+B6/6TLjxsJ8oIvQsajvBavimTM+lChPn", - "RfeFY45fChwkUTbXbsj28mu+oPY4b/ApDYUYU3hhZoT/goz9rEdAWZ+JkS5VO9Rn1M0R67NNYYI4VXrs", - "P9cqsblUx0JN3ayz87K/Mom59Eini2g7NzKYhArb2Xm53e/MkQOos/PG/0Mq/Ec1SoXEuGYYXPL2QV7V", - "B3m1fY9RlihFAdVVK2b49W0xH7J9FLeRyPU1HnAA/Ot3PTC8ksRMp34bIkIwEd14/YBta4v5XDgjx4QG", - "3hAixJ8JSLpWY9Y/QgrHfZsoRDYOILrkYoAeHYDowX7FHRjiWPBDuBMxwqBj3Ag/uMiI53G7HqidaJOo", - "GjwZDRFf+FqIgja6Ev4E0Go6cFzmwMfkDaauGE6HLOnUknCxxpGMF/hL0mEczwGeqLm8Edkg03MO3GYx", - "GlYxAy0JRoQqbpeL7eHrfmfiVb3r7HQmueauU5OUlzU52Y5ygv3IG26jWNrA68kHQDqeHK/2cYzEHxcj", - "IzM4JL5Fq4SkPax6ntNpI9VnxRseIaKwzlzzh/g9I1VneO0Gpeat0WVxlN0VnoLLmMyW80jSMso/O/0l", - "c78/YnjLT8CVFNeDgH++ajq++PDWeWMpgRaVzfmCzEwI1sE3+k9eEC8fcxoqyJAscs4XVIAQKC/hhiH7", - "Y1WNoFWOJQmhJZwCAVDB1ZAm6rWReQ5VMdYOoAKUDGpknWyFhvQvEOftXIOwbgp7z49FQ3xSnKrFSz9D", - "UWnwNX7Be+g5A0owVY2tBuJZ7cPPiCKhIt76ZYoqcCmMtByPsQ0h+8HoeSVmd4dj7Ne11I8VzbnSl41V", - "+8df/xs1CuqMLuocbVCd9L4YlXnLnP9jFLTVY5AcfbqhgJX3d5aILXHb6Muk82taAUhVCBiIn8MoBOOd", - "A6nYy0Qhy09FQfxm+3fELNp8cqnwjRbIsCi49Ub1TtIZDodxTKzvOPieFYAzzWVuh4wqpMkTTffqZnka", - "QPnD7Kzop/wRZ2ODNg+OsN5AhrmUltFMPDYDwqe8QlwO8oAOvl/qLVhT4HgceikAyidUM67C+PFOkhOK", - "q7G4CwWKVto7X5nI5cgvIsR+9JAdOQB2tkCxiEGcXDAlblzoIsm44yNuRaIAkAbi4JYFWLP6FUzpAPCC", - "FFTQvg+xJ+kYV/ZaGEBjgzYEJM8VMPwADI9rqTJ9jU7ilFNV5VxD+zW1EhKbqLJxxnNpnVBSTYdsTwVe", - "lwaXNzq96evtl343+M+j96PKpVJdG0mvCmOHxxx8T0iY9IhRrseXbCRmEpF52MQI8ReEiTtyjFjv8TvZ", - "N1np9cY39TenTMUO/k2XLlBUyNwlqmL0jRMqLTAqE5ANdHHD5yKmSwYxOuhhVRkE/mCUsoAp54EQE2pK", - "eewH0Qo0QAlfHkjB43wmyn/ykO2xQkNFqrRM3BQQKgD8BKEyYSjEYJlXOeHBapp0mOEBMswbnhDVFcZo", - "Ewtdf9YlgA/KIHjSsqw0UVIQ9tNYlyhbgmU5KXN4mRrl8ZQXtQZwwBEsYJ4oRCgtEuxbzWYQswl9qIiQ", - "6jc5hRdhb1LJXYULxP1y5DkulC7dWM9xMbyYYslz9TKhDSw2NMblu+Z+PiEwOTF6HnYCZNpsLBuuth0G", - "iBFUXuSWCJmQOM4Pn8osD9imSqOQer0HHw9bIIWIpykLJ7IU4qkQWSmMuJK6tAB/kQXuUCA+jJIFe36k", - "tWMj2J9uafcH9jiQJOjwVw5fftd/FADaY7AFYkbYJhaX3+lEreyhfivcu0qdbb7DsDbYh/Ai69r0a7oW", - "v/g2IHGkXuMsoNfV74rf26bWsXDc6/Xm6fBLZyS4EcYfzf6w8N4hbtQ2y+qMz8VAGzmVCmDi9CATDvdt", - "Bal1egxbPVaJ2kLAq5Qm7+x0tgB2mF7rVi8NHGyYOCLEJ38c2QaWycj7EStybSyXEzFejHPBuvunHw96", - "jTsxiHz7ZgTp7tfYXPoVxnwf9g2q7SXKgurh9O/bjz6fGSEGsL0qqMHCaKfHgFgf7NHAjXf7CXsnRyzT", - "49IfUQEdge7K9Lj1c+jo6bNcT6XayvVUl64PYMnX2mQIaSD6kb6vtPWuIX+ytb2HN8nxEAWc0wq3pHar", - "v6blXug7xe5Q9AHA0B/YsS5ExvwXXoqFReKy46Ots4M/+DFqzy3kwF/R8ujK66DgBHVuQMRPOg2pS//g", - "pfhycyWHiao1UYSgDUQpsCuncdpHnFfkzcPSSpCQRM11JieLJlDrkJ2cvmRYA+ClEnT8bvWKC4Kk9ZPZ", - "T1ToiOxHvemu9cA6Po2hzdhzmEOdgQKOE/GfpVAuUUbkglsRWRprSbmJwC4e7NhDPUlzXPOw1vk7dged", - "swjgYYWDkfyk2CE7XEL+tTgtSwUPMSoWYkp9NjV+QfxpU1VNwPG9FbkT4ageMgw+wkT6r68VMEFQIMrp", - "LrQ5b1FcztsdeGmQuwkwjkzLnBt8+2AboDdayPElrTMRoojGhOFzWyaLJPBEGAuZjT14b3auL4WyfqTQ", - "xNm2MpAXGedaoaKQV/7kpmSpylhXF4HOpccC4Km/NAjNkJ1BGj5RQo3Nwh/SA+4GmMqVnO0dng3e7r/D", - "xCqARjt/KHs9TWlaJm742OWLRGk4VhQ7+XB2joZDEy/Hm2ECjJTmxED75ABwUNrm5x1JDkFtUgs4Ichr", - "oDN0yGSoSzeCzCU1xYNJOJVXwoYGTzApeb13Hc1v6Zj1gkSW9Pu98yHbj9BWNHSicE8qfb2LsJOINotN", - "BlgAlNd68v3jJSEQwfkA80znoZemVV1jH0+PbWOKQif0r3/+9f8PAAD//w==", + "7L3rkhs5ki74KljOrClTTTIlVVWf7pSV7aRSUimndcmTmerusY46DDACJFEZAUQDCDJZZRqbX/MAY2O2", + "z3GeYf+fh5gnWXN3IC5kBC+SurbX7PyqUjIiADgcDr9+/ssg0XmhlVDODs5/GRTc8Fw4YfBf10b/JBL3", + "htsF/DMVNjGycFKrwfngtTTWsae/ZQvxwJIFN5bpGYtv31w8PVlo6yYFd4vTeMxuhYhULJUTRvHsrKCP", + "2jF89pq7RTyO1GA4kPBReGcwHCiei/pfRvy1lEakg3NnSjEc2GQhcg4zEg88LzJ49Lvpf0ufJb8XT/k3", + "s989+fbZYAhvw5CD88H/+AsfzZ6Mfv/jL09/++kfB8OBWxfwknVGqvng06dPMIgttLICF/6Cpzfir6Ww", + "Dv6VaOWEwv/lRZHJhAMJzn6yQIdfGtP5RyNmg/PBP5zVRD2jX+3ZK2O0oaHadLxSS57JlBkakJ3k0lqp", + "5mwmRZbaISvVvdIrxe6lSodsylOWaDWT89PBp+HgUqtZJpNfYZ43wurSJILxzAierpl4kNZZdiLG8zET", + "OZcZc/xeKJzXa22mMk2F+ttP7KJ0C6EcfFUAgUrHMp7cW+YWggXeYUZnAiZ2pVLxIMxHxZdcZnwK3PO3", + "3+JUPMCWWmGWMhFMaec3sQS+xmnZcjaTiRTK3Tpt+PxXmNd77ZhQupwv2MwIwWzBE8GcZgudpUg++BpP", + "nACeK9aMZ1rNrUwF/BgpbeRcKp6NWfySOz7lVtw67sQ4UH2SSns/ma6dsDHjKmUxjNP8a6QSbswaB1Nl", + "PhXGgjhAipDAoNn/zWnxUS24SjOR4iYJwwQ9OQQqvdalSn/FIwb8McMxPw2rv9pflWfr454kulQO2Fda", + "nNkKD5RWzC2kDeQ6UdqzNJOK2IMIalgqCqFSoRIpLPuvf/tPtuBFIZSFBwtunOTZyIHowxU9OHvqWeCj", + "4qVbaCN/Fr8C9d95uasNk14mX1xfsXuxprkURifC2l+H/O94NtMmF/W9MNXpGuYWrodKstE9AXP8kzb3", + "eIbtS4nz/FV41k8jyDbPJLV42+KUROe5VtmacRUpoRKzxo+N7sWaTTWwPpdZaQQrjFgKYr25dItyOnH6", + "HhhnZnQeqZWE63sIROFtRnopCuQupdWoMDotExigxV+XC5Hco9jx08r03KKMMsI6bhzy4KegbaBacJE4", + "uRSv8qlIU6nm10YvZSpQOBVGF8I4SfoDLR4pnqYSxubZdeMJ0mPahLwWxkoLorbw3w3naZrp6ZjdLngh", + "2JIbOEXTNaoDz4FDI3Uv1pZxI9j7D3fMOg1Eh3OGRBZqOVpyw0CnwqdI3fIqkJ6CMgbMI9NtHS8OSxxf", + "vTw5jdlMqrkwhZHKDRne+/FSr/lcnNN/RolOxeib86dPnn17Pss0d6DcveMuWQjLYhEoN8l1KrIYOOPM", + "Ou5K25pU0MuGA1gkKnqqzAfnfxnoLOM5HwwHuhCKy8FwQAMPftxW6pqK41/oS7jKHzsWf5GmP0h3Iwrd", + "0Pvaezo1XCWoB+dSvRVq7haD86cdc/asWppsm6AL5wp7fnZGz4wTnZ/plRLmzIhCs483b8ddVCh0lk1Q", + "gV7ybGJFolVqtz/+oSBOY4UwI/wgvMgSDrJXwHnwr7KTJ2c6lw6Y7b/+/T/CCUjFjJeZO23MAQadCxMm", + "AVsnVCVZ2sO/wh+Yf47ZtUrGzIsHy1ZiutD6Hnf++0epl0+PInXif2F//nATXj59zrRbCLOSVlRqHBxs", + "aZkRsGkiZd8+e9bimqnWmeB4caCYmHRxdEUjmYK5wsNx+UG6N+WUXV/csZNasmrDCiOX3MEMCm1PO7en", + "uTQaEek4OB/kXJU8Gwwr/q3+wEunB8NBoMN+/m1w1TDwYh8nG10W7+CwmV5uLq0wnkC7xw0Pdo5VyD+I", + "dYf4MwJ08QnHgeEeg/8bpNyJkZO56CJi51yGg4xbNynt7o+pMvNaEQnWHV+RBXzliBdKftALZLF2LACP", + "96Sf3MNBYcRMPmyz6ktpi4yvRyjF6SFgWTgOszLLQDHxxlecyIcJfzp9lnyTfhvD7fZWq3lQ7Z1mRiR6", + "ruAwScUyMNuGzC60qdR/t+COSQfauILbG15Q1pkycThgpel3i2kjlvpeNJfXOIz+xy/YwA2WlCDI23T1", + "G1ARc9jkwXp+/Ux8SY9v8zIv5OSemHyXfuSPwqfhAPYmvNHe0LuFYEXGJSohuH1LnpVizB4/vhGuNEqk", + "TDzwxGVrplUixo8fM7AFBe6MFUlpRLbGmx2Eo1e12IqvaY+dkWIJD7OMO2E692qDlGF1jWn30+ittO7G", + "u0l6CYX/L53I7eEk8+NxYzj9WzueNZipuoW6Z28H4ZXOuZdO/5EnZZn3CsMguIOUVloJ9EglRuRCtb/c", + "Q0n8Rtf4L7R21hle3KKi009AJURqJ9PweAf/mFKw1UKgecWA9S1zeOdKy0ReuPW44zbcmOfmKF1Tvlxw", + "NRfX3NqVNmkv2ZLSGKHcpPAPHqAbKbFqPb5pgSmZlzn7HfoTeeKEsWP2XrOyKIRhU7CIYYmNQX63b1+2", + "Jrkxic71A+UuuRNzbdY3wuJlvrn6VGQCBAxax37pMPvB+ZMu9QlsmuOeLo04/DDhlD+ULtG56DpSdPfs", + "+sKNSDIu87Dsq5RkN/5RpOSuaQlwqdxvv6Xd2LESey+LggTrV1mI/15NyE1zdISmO9q4jDtG28TgomGW", + "yxTE44qj4Mw0WDPM8pkY71lH1w3UZoDNmW3s+DYpexkvLH6L4/yl0nH7cm/L7zGBUND7p3uH7z3uXPFs", + "baXt02MS4hx5BNd28lwu1RW9/HRz+zflf2NGrfF3LK77MH/G3LuERAe/pqVBXpzQF/cce6mkXRypOX+F", + "M+q4OU5f39iIxgfai2ivf3uu+3cN9TJSFXo5MyjfbVHwpsy5Gs2MFCrN1izjU5GB1rtS3kPJUm4XU81N", + "OmZ3Da06UqiXwa06F0oYUAy9jTxC5zd5ibo0NlS5dt6Bm9cxTL1/4T+g1XcH5uzfcPX75jwc2EQX4dor", + "jEhIV+5yY13NFRjURFHvWFB6xVJh5FKA+c4zRp9DN563vB/ZSP159OGidIvRLf0aInJsIXgK1/+aJZx8", + "Cz+8umNnoACxlXQL8jbbsigyKVKGxv+QWY0q0qj6Ow7KFlI5cpZVN0CkwNgpMwfT/oMoHBr+U57cr7hJ", + "LQVBnJzKTLo1jaizFN/LJMgEMp+sk1nGrFBwx/iYZhAkWwTdVnnvKVa2y2S4vrhr0dX7Ti3eaTCti1e3", + "ox8u37GpmGkjIlWQT1Gq+XNywUqKiqFJ2XIs4woEfDThBk5jpFxrbDJVPo+/w/J28LnRZdHL4S2a/NJv", + "fH/Fg+dD371TqiLc3XuGXD6TmbBr60TO4Ek2FeS3n0vrhBEpO5kKuOktS8nUp5B5p48p58lCKtHp07oW", + "ZuR/Zx8/Xr1kFctPKbB2+faKneBh+9ezcSIfzuqvnY7ZnxZCRaowwgpF1r4P0QO3vP1wefEWBZ4EPkuF", + "cnAIwHgF65PnAgMOaaQynfDs/Jf605/Of6mo9AmOIzrbeS6IGlqxVM5mArTzSPnX7BmZNakWIYyQZTIV", + "Y/Yhl3QuxQPFBckj1+OQCLNAsbdNMW3Hb7R1MP2T0+BUkSFKG2gJhrbfGTwx4733YM0V/Zz10e5wy2EY", + "vXUJ01+6HGZKOsmzHebUB0XXNwuPUJRVrFAwsry0DgwtNQeBwGaYz5HpuVTjSAET8zSXitkFN8KS+NCl", + "G+nZaMpVuiUKftelmuisZVjjFwdDdCruN6nD0rdW6j/cT+MqEHaoSOlxEs+MECPYCtZ4oPN8flUR1Aqm", + "dyjipdOTJbo0ugwgnrJMLgXdrnTR0+dQIA2ZQjGPv3rntxUO7ggLl2akJB78RBsDMgCsKLUON44Rc27S", + "TFhM9lnoFVw9c+3YQoTA0g4nCnmZOjZ+OJhmOrkX6aS2ZdrL+tNiHZIRMJJHbkrUO5mR84VDJQNOLAcF", + "ZzTL8I9JppVg2kQKTzf7SU+HTDZyLeCA31MIUTEgsjccfY6LKZWSaj6O1HtQDtH5Ih0Mz6og4QH+Z2Gd", + "zNEd2Ru8eRUeoUwLOLdDJh6SrExBJlEQhMZkL1GXSqsdjlS1xVb+TPopZ7ngFuOvbmF0OV8UpWMUkkXl", + "iLaZq0hpkwoD5zrncyVdmQo2L0HPNdwthAHdQDEOt0Iuba0D7LRjMvHZ1shGishnfiGTKIrnHcd9cA1/", + "JpUHFEGewd0FpJ+WjnJflMZTA0f9yLErHsKTmmUfZoPzv+y2JN+hjqW4SsSH6u1PPw679AriR8wD0Bb9", + "9MDE1aBDJmdwXsdbXPlpOABq1H6TI9eFLwN37bdig1rUZ6nCpOq93Yg4t/cuZv/P/83iauwYT/iKWwdy", + "TDviShCbcGhDcESC6IbHP2P7mlMshEl84kTOH+ilp0+edH6CcpcGDQm/wcIbUlrrnPGmREO/7LlPuipQ", + "u1oZ6ZxQfVlXeIqn2i0oE49x17BBj1z2UpjUZxFWofZ7std1nguVCrh1SzMHenTJb/+BXvn9QYHVAmye", + "CCYe0K4Ics2/O0RPHIUyFBh83I6k7Y648qyPsrcgAD0nxKMVz2IGpEu4GbO33JCuA7osWsK25qJpJnJg", + "r+oCXIjkvtBApJTBlZtzJ8EApQgMiAp6ELdsIXgBxpRbSDWPFMXcgJOqO4PG+ozd2dAafFJsQ8Q26dE8", + "qK0TvyUVu45jN/8PWxpHzSxbu74tu7uPQ9eF2KkHiRnqfhodgh3askonmVSiK1TkKdQri0JmSUfoV81L", + "n365/WPvaL0B4IJj/KH3dyvnirvSiP0OXG9M+1SWen1+XsOaII1l7CZsr4L8udSTuXStJIinT/a6H9f5", + "VGfHas/+rX3L64u2GfTYHu7u3eDFXVHLIw5zmMWuAOZLaV4pZ9Y9e3RgRKlnK3cIF/pwz4xE4rTBiDB8", + "Ztu8kqbbMZJRilsavsBOwIwfGZFxJ5fiOcUx2ffMaO26XSFCuaMc9XdGCCJg16aZUiUh8r8r3AqjgglS", + "KoofJbwoRHpAwBVIUU+6OWI3ae39R+vFz4a0w3z77hyLmZ2g6O3XmDvko50g0x31kmyl49m/ZtKBtEkW", + "RmNeHjplBsPBfF7OOhWFyiPTJSjBzN+1EZiRg9NlJagIuB2WTUXCSzB+dE4bJYUFK69SmZwRgq2EEaxU", + "YNhh0BysJB+zG7OLKXq/csEV3ei2zJm0kQKGyoQTfalmvSKynutuJQXmVo1fGZi6RCvEsalgK57di3TI", + "VguZLNi9EIWNVDSolxINwIyFk1hKu8DFoWkYDfA0RQPyGt/+97fSeU4GgxhsS0GrBUUJCTKyizwoTLZH", + "WTkkOErbPAwCxfNuF8dvpdNeqZnuCH5+fiZos2Lo8HzcS0y+vcU3Gbfsn28/vCeXID429ZRD7xClYBOv", + "Vdm7PElE4WzI3JWWxb/Qg+fsL7/AXT6ksMyQSnsiFag4DKmYQwbLHTa9T59+/BSP2Rtu0kSnImU3gicu", + "UjANyyQGX9BX95xJ98iCqq2tT2WsPKdO64y8Gl2JwFYkRriJUMsuN0UrmxidgtWCgR0tjpQYgZ5intkh", + "RSZ4pGYZnzMnKIKzWgj0LQieLFDbpjSNbM2scJQxHsIc40h9tLUzuwpbNSwF+LvPi6cscq6UMJGiOAiz", + "fCk2AjI7c903OfIWKfJKLbdvkO4sY89vbVoexPxwo24zfyDx4Zde96naN/16nIMmW9PlwKBkk3tC7uLl", + "1Z8nf/zwLxc/vJpcXF9N/vDqX+Lue98Kt/+SXjL4PnIlxTNOQKoprUYoCk83WOuAfCnSuWHwTpqE6qRN", + "97Lzjvzd+pZ/ruvLr2UmLuvSja0Cg/BDh35UW0VtYr3l1jH4qY5enzwdTTmcLrwOrFyKnvTv3fZA02ja", + "yGIVjtK1wyPobFVlljE5w1ucfh8f4kdF/2fP4qge9TNXR4oQvNwh8+7gR/oy6X+VoykTFMay9z2fPUS7", + "1KUryqZSCZLO+9vBjj/DkdGwsgcwbNMsbNCrZRw2l9uc5bBirD6O9IHa7iyaqbBuYhNNtmylN2AhyKDD", + "VXY4T3UE+LCm5GB5CHPHOpS9MrBJv8aC6iH7SEOf3z6qi1LdT+iNrqypA09yh8dBgCk1WcgjrNj3+M4b", + "2ZmqdMTOtQ9in0nf5/Toyh7q4NJAmjCzYZOWfbtwzdeZ5ulOkblR+Xn3evQ75sSDG7MXUnGzpsg9s4um", + "Fm7LKZXMdF5O/uuTRWet/O2bi9Gz76hUPpVzYVGGxP6luPOLO9m/99Ac4qHvtvlrarfW4j/ZR+4bwfsT", + "hIVKd1xCQ7YlnZthfoUFOrATreTMp32XU1cotOFZ8EkHDcmN6iM/KPsJGHTXjbN7KRSlPHAxHdJoB/FB", + "Dnf7Rv8O7usdgnWndxKWdiu4SRa9nLXtZny2183411KYjqqM23JKE2Yk4FPG51wq61hczTgeH5n2RGPt", + "W9zX8k1u8MKv6Jt8rU0ibp0u+heTcJWILNutA3HFOBbXMokVu4mwlhJymBXWSq1QP8KaeMZVioVK9Nkx", + "e80z67+jNAZg8OEqn+cEjvxPejr6aylKEakE9Kay8Al7his0660QLP5JT+0EfjcixUKqTp9P86ntVV0G", + "HbEQCqylsxC0xWSECRZm/oZmR/+Az2E0OlLoo/KJ5XUCB84bRQka3PBSa2pNNy+FQ/dxjE9v266WqDZr", + "Y5Vdm+8rZjsogFkXvwk1nywXjqfccVwCV7Uj4mQu3QjJkp6GSO44Uq98Su3T86dVgiedTiBjAJBhRq+e", + "M0w7q/+24EsRKaWZnxw8RLTqyKHx8+u4o8ScJ2vGM8nJoRE3azzZ99+zCL8QDeJxJ4fUxcLbmsJnFEe2", + "S4q7qxVFsEQPK260iwMLG8WDm2ABMu+4Ai+mVmdlqK+ouBODneLBsdTzLcfK3jGrsmgiFeqEJaY/YhXs", + "mL0LOSsV73s9AP4X5u2FginVhnPyqHpQkOk9qtrT345AS7t9c/G0UXTp+QsvgyG6oZlU7OPNW/slBdvX", + "e+q0Pb22S7QjdXJ59efJy1evLz6+vZtcf3j7dnL1/u7VzR8v3p6O2UW24mvLkoznYE6WBeg6qPdkWhv/", + "8rur95sv7kpmOqYS/E/oj4G3ydeyABFCiwQBlJaZMGwmCBWgZhpMNYtUoBvJbZ6hrMC7wekgUuhUKq1G", + "lKToq7Mj9a50JcboMXUKFDGSIK3z+398z+oK9F7HfmPLO0rVPKZBhWZVpX7iZZJwpZVMeBapaNBZ7P9P", + "JCKiASO26UlkbVay72XrskiPFi2bxetfXqneIlzzrA07i9iHbVm8MaONQt7GCnfcSP3FvDBSSMCeKO16", + "CyeqKBGlNngtpfk6iADLZqB7dMqAjYd3aT8t5gTV5RG8/IhdvH/ZcFZGypYJKEazMsP0/Woe8AxetMjq", + "VFDRx9Zz6VDr2KchhMv9M3SKegvJ/d1B43cXl4x+bFU/a5B/WjHac/Yb+sNS8khVuHFnvwAzfTrzY4yk", + "munx48fdxydMpBOM47qcZjLJ1rDZCYXNrj/c3sGVg3k/ZCESlUEqe4wIur5SjXqm13CscGXB6Mxk60Mq", + "rwNRGzvSnu4WFXsYflFOL5KeasiLMGcP4oSccn1xRwmsQlBcELPB9arKx4IHpI1U5UbF1PAhm+ks0yvy", + "T4qlMGumzRzDXNZKoN5ScirLOdNmbn0WeRWveWQZT1O68GaZXmE1EgbNCJeAs1uRicRV1SuU81poKzFT", + "oJDJvTChkIBSHrXBpaQGNHmpnGac2UIkciaTSMH0wJITHHUII7I1Jk9SCIDPZjKTmDZpR3w+N2KOSaBL", + "KbpVxiV33PQrYXouOxLf/Abgr+wESY3RD22QejYr593hjuAxbH8uwtz5aOCtAdosvFWes2igzdz/pM2c", + "K2lpde3EbEy+H8Kz+2U5Lco/1c+A3WbARXP3lpJ4hLaI8s6vL+7GW2T2Ks6kVqG7ymsK/cgGbYjRo883", + "woOFEaOZzDKK0/rrVkmFznayKqRt19Mjj1nGSR+pdNBMWtdzP+8rTUJYhe5gGOkCoUBq88WFy7NeXvOg", + "NF25IJtOl2r84SZl6880Ruvf47tQofZ3jvXSn2jXqGFs7sMLYd1IzGbaOF8jiPvNrm+eEqMCk3CHVROY", + "64lFf6HIyj6PFMJtgHgR3IJOqIsS/kQM1ixb9KWOvnYx3DORqozcut4OFb/jqgi78jBC+JLW3tKm9mz1", + "brQPAmI72EPVZKEvAPzwo+7ySGGA6uuw6c4aodedpUHsFaaveYciKY2Y2TM+4hz0cvDx+v0Olmgu50gt", + "G0h8le5mkDk8NJFpm0eO4+D6G73TOGASR3Ap8s4X8Kcfby9/EjhXh32Spkfy6BEFg0cW4w2PxwgbVoPj", + "WMN6PXsosXsXc3zmyG30JP6CzQzD7trNN4Jnbqcnv7sc4haTnLI1iYiYMBBjzBEr1QI/uu4OC9KjW4Ua", + "1Vv7dTr/ha7lIFDxCzGXO/LCyyxrBV7QAh72e4BWshCWSlQa3lsGsxBe1QdlvrI+vL+/J9Nh55R7d6FU", + "faAlpImif6I6gh0Jgr/suR0G73hB+iKGFskP9O//wULgV8/qDLeR1359mNXfGZGqNNFAogW3vlhzKoQi", + "z6dI2Yk2LIZtQA0oRodBwa0V6WlnRt9mWIeIsbn0Xna4xJDAgfGdPdpo/WzvcK9lJuzOqoTj4mIhHwBz", + "Rh48sMx3T7bFQs0kxwT6KmrSzPYtq5eIi1Ld20lSO672F1jaCWWYHv68D6yJdPI58cCNMYebk+4bZQdN", + "lLSLHSXahK8Eh+koLeLgvQyJUDTvVNpEL4Ov7phAKY22d51fd/MrMu9/YfvOCFnt6cHXxfawWwzQS4Br", + "o+dGWPtq2ZmA80EJhkDPAZnm/UtMtrbOCJ4z4ZFqp2sWo3/uDCXhGc4n9u64pmEmVGpZfIGMes6amNcP", + "I5X+ZLWKyfEV46gxpW9HChjAyFwq7nxy95IbyZXzaLQhzZsbUdl4KeMWLb8lV67LazTlLllUhbPbe0M0", + "3PVbkzG2n0FMZQ8udUD9hghbEDgBcxw8flNV+QPj1v8k2Ob63ylV9tNvGHUcDhaCGzcVaD7Qkv1T9ECX", + "ejnjbT2sWc4Cn8Zd7i/ta4u/I0Te9qO5sPboTKsdSoWzn2mf0e7sPUehQmLDne1/ZQVdecTjlFYxClkU", + "nqMDyo/34yJjP6dTtJCgGMiEZ6MZz7IpT+6rt1BlDa/GGxSOh5Hyf0Nax0Nq+dDm4rjrkBwrAQOoY6UO", + "bChjjdJ4SuUj6B2vQQ2ZEithHfm1n/v46Ddj9lY4yzj7eBUpu9ArD6OhzYqblOUa67TTEk17jiFob+7r", + "0PCgn3TH4kGJjBe2jRdR85Mup5noS6g95iL7jLukscEHFB0uuG3ZnLApcglrHu68g3Ycr0/7Tkf/RVv4", + "J/bpjduHrXWJbiAWyDTzwARKV9lKqLafsYoNyrp3xxi7IVFuUuxTj+glOqvwe0Wb+CyulOb4LCZMyfgs", + "9jlF9H7GrRuZEjFJXOlx1WKfYVQqG7cDADBhRGahObS2YthKAaLhBrgbMNwXGZf/rKcdHg/nRF64A7AR", + "qzl+kXf48/yAaVmIgOm8d4hd3u3Dk3Ry/jA5nDhFnfV8eIXbDV9RVZt/m3gRi9WoZYsFyRbDaPGY3TRA", + "GZj0KlcVbXnOUq0eOcatLXPBCDq87G02EdJAjtuIA3AqDykh2dCFfZpeg8vbB8Ifgh93xOgO8LriI8Na", + "m672dmOrN2iz12P/z3q623v2k54ebjHDGf0ClxmOtctf9laq+33QeiF/pDs/C3Qan6MVV6klMfZyqF0k", + "IZWwAZYYKSOszpYC0RKxx1VI2EF0O2WFcaT1n6wCfNhEpkOs6KwSWk4xoRC/G9w0iKM2pbwt3N3vH/l5", + "+NyinD9UNuhv23nEvz00mQaJ0UlRPZfqrU7ud8vWjcis/6VRdlmXSzObScScWkmV6lV3ZVPld95InNQr", + "YUYJpsLjI8+rQjzUHTFqvS4Ei2UxwQe6vZzioZAGVPwuiOfXl998883vCQoo+Mx0lgrEwMGFMQR40qXz", + "UFg20w7B3Oy4L2dwW4x3ILDfUkusq2sKC+vknknL7sUac1e6yzjqTPVNNk54QSBUzmAVevXRnmKyzoSA", + "WBZxaAmAjV+urhm2zNLK8WxkV0IUVLYmDDvJuVrTxngtQSsRKer0dTpu7ErrkydX10N667T6FCYZqKo9", + "2IaGUYB+4b+1X2nwshHfaghCIl2LGXaegN1yEAh7uCCsj9UXiEMacqc81Lt87UcEdw7G4O/BWNyJh+9n", + "2UdZDCDtoedH20GfjchTPWADEa3yGW3KtsPVhkwsCXKiYk9Q6YeDFTdqp4dip18gmDZ7GmwAA9MEwjv1", + "d/cs/UMTSu4ICEjKkCH0KUwppBTbUrFM8KV3bVWIgVKNKZwQU7AhUrwoBLZzVZgJQyjBIA8I+TDUwwkX", + "kAVpgIvrqwoZixNwBq+A2Xg9YMBWRGcfdzRFvCUM3c5DqvCwfG2Z1cdCS+7VABGTg9xTXT6z/fnP9IFK", + "CGyUuD8UVD9lGxgfiS7Ww5AIH9w0U246Aer2T+BwKwIdSl2twkxKmaOGywxmuYKdoL6MIh22wAXBcBmz", + "iwKbOCLqAo8UhrmmotIZwu4GaElbBCAVj1rpbzoEsKtgNgthCLgi3DZ1Lz5EvUPUDFtk0jGeGG0tcysd", + "hU6MLJMzAYfekqeJQHydgBFBf1nwbAYfKC2lhBPyNSYWcsdSmXqE4Vwg2PeY3YVC6JAv3ySDx5XxXjLE", + "iifqWQeMGurAD4eZ2BJyHTfNIQ0B9jIBKD67mgPsZ7hN8BV/tmu8OAwGBLy8fbJ0Z75148fq9JCSyqwT", + "xUEwAQcBN/Z8pXamVRJ+Q+EqjCi4QeeLtCw1HskQWGhusEAN7IfnLIZTHx5DxodFUjsvUq2fsxhF4cTp", + "iV3xImZaEeR86JDKTV3x5j2PLSRYaqttysKJFMfh2DRT6pIEfpPT4dACN1WHrZLIK27ZFCW5w2VQkmai", + "FTakVCFYj7Ooc289Di+zMi+yNVwJRoyqwpwNZ1NFNDSukS7Ui6RePZi+K14U4SdcIv0jRByCO6qx6j3O", + "w43mBIhKM5NgHIDuSrrmmF0FIGw8x+QTNyU1EeAhrpRwFSknsoxxsCXsokEH1KU50CkTnlooITMxc2y6", + "JrEPMmtD+NjSLOVS9CSXtv0dW5VXFWBUqHpccEtu6Qv2szAIYSvYCovYgdKMA39My3mkWsi+lkWD5ifC", + "NRANPr/Gqi9M7gFy6Gx16j9iR4ux0i0muXAL3VVOIUKqb50CHFCGnAZKz3giWDTI9FyXLhqwE+93PUWA", + "6AVcZdKxE99VzCe21+3WHtmK0E7jFQUGpp6dthnefxRMGd9crYtD68y09ir+KMVqRD+S7ONZhgkgiAHL", + "nPZRivY6KW0VZUw0wHIrmCJ+JhqExPmVdAs0iX15GUPZMgLjMwQyUEOJFGanYvtc+oZ9Tljw1pfMogjL", + "JMKdCom1A8xnSy1kYSOF3elOqnsSP0IvUL8Davzx6o6d0fdPj7g2e5P0vswMGba4q9qgbhbNtVl/7L7O", + "ftBwBLEcMsfnxmwheDFBkGmPXedBY3PB4eaYlZnH5a5KfCNFitC5x8dNHMIIaCMwJmVBGZHYDNBHzGGI", + "oHo5E7rfR6qphGaap1i3mIqHMbNr62eD3Wxs+BfcGgs5XyDTIayeL8Twi1roLLXoS0m9kudzuKhaxN9Q", + "edCgPtx2xfs2KfIZeNT4CbhcvvALwOxf9gmf9/357ZdUmU/myee+qI0unQdi2RMkEPx+Uu10hyYm54vR", + "yueyW+z1gVYeMhEe2blwpgS+H7OLEM6CG/OtVOUD6Qc5Tz7c4o1K3aexa4C0wgb2DO5ShL7LhL+VYG70", + "Acp4s+XUOulK57HrqmkfiEk4HOxY6KVXKLdW2cRhxLabDXsXb1AvQyOVzDU7wbUSICM8OxULqVJsH/LI", + "Msft/USqmT7FK8Qj1UUDdcajwTDY2s4IDiI5dHXHSDyQBC7wg9daHeajmWizgcnmwew4Jttnr/sgNKdV", + "Mfkm03bKV52KrAfytkure/Oa6nWuXvouU43y8YQncFNWcLMN5Dmqv8FO5N0VWN2Vv1XJO8YOvJgbz+fl", + "rA8QrI3O83U2CpWpgPJVf7WXnN24gkidSS9Y7+XVnyc//PDx9eTy4vLNq8nLqxsyKMBesHAyRBo0B7zc", + "sYylwtpj1dfZ96BK1DTyQB7djY5gtof7aRu8sq+awH952Fh1F7lqiKxjobx2w3X93aFr1YsJk+six3Vd", + "/LtJDKNz3lMef90AW/Kws16LqYxOUCRIW5BVGf0jBNzNMmroMGbvP759W5k42C0BLoQDm674CR5x5vY7", + "QhKtHJdKmF3rboTSqufZiZ45oZj4a4kArHX4sVv2fFaeQqOL2F4/CzxEcczOXmVw57ahVYakB1IJf/1Q", + "heWilbDjRlTVK7nNlmGROqk7hrFCmKrVVjWcpdsyNNbzEGyYjQWM0mMoY5WjXaukt//ExuqxA4Ui252a", + "IlYleL7HFXyNTXk6B+O85sPmSHTGahSAw9hy+wu7UU2bp8Ojn0iEv8i5G5HDge4zVPgpddOciYDV6p/0", + "/o2UcLOZdKeo92PXEd+IJBMOfVTTUmbpmF0pepM6YaEy52MAKXVybJug0YCsYQZLiwaRQtpRTS5hdDgj", + "53PsWU0erLVKAqY1ogx5POWMz6uEOsSQqVEPyJokDwfqaWdIeI/aLRWO3Qd8sKNjX3Wlb54CXykeas0f", + "2cCw3eV8FPWfACshlkNXeKDaF3zAH4+ZRLxoBDnZ2na8bjNuXaROjDj1o3jhqBUzVEXPHVY8o88nNXLm", + "DUEYyrswItXATgb5Yukb6AD6qO6VXqlowDZ8Q/itA3k7wLodma6PqVGBel/i6T6qSyJvC7gaM4yC+USb", + "pog7yxAWanzMTL56A8K9I6NaO2nU7m2gXWLX1IUG9TcAeAtj8ZCfIEHQq4BkqDG5iBh4ziKFI2QN31gN", + "dxN8n9qwV3++e3Xz/uJtjc114hbaigoPPOBewASEOQ2yADtygcAgaJ+AcUI9dwNOB4ojxAXhiNdALjNC", + "GjqQV3dgQBGw41PC60wWGAudISTUSX1tU7Yc9d//ePO2cZLHpJoD0wzOB//jL3w0ezL6/Y+/PP3tp3/s", + "AbbG9nsHAqzchsfhVWzA0CPWbijhKQ0aWKV6VZe3qxLESRzVQqWSFF60UHYfocmjJ4rPgWFn+jDcZprm", + "V1XFgNMOJhk+210JU1UvNfJhPf/vTA744mLsJjBSfTlt6ZhNqdrglUCARkphXw33toDdlBI7FP/dSTXh", + "bB9srzVAgr4GRmU1/q4km81z09FZBVthTPyhPfL2yvnDhKpojsfe3Rp583O71hMOwIaHxG9zVRux2ylI", + "FXB1FdIhTx/1aTJW7ZGEaQ403FjTxqQ3B9pFsjLPeZdbqaUcfi215u/nhqEUhuZWHHRYb/H53t5JtTDt", + "qKssJsHpdkyvpqrXVZ98+Pvj1D4xXonlpvhus/VONt4m4tY+7uD0qgNyjx/1eKSJZlJ0557XDxzmmGp9", + "cOv1PeARm8vs9m9W3zz6gtqg3z7fYmOgrtneCG6tnKsPcOv2pnvu0dzfi1VA+goxFMTSJWCFIfNwnAhk", + "tb9V+X4F4IZSey4Uz9ZWdtw03P/SxxAJd2J+3Pn3Y17Sm51iIC0p58LXvO4+2O088gPBdoUCc//I40GH", + "t7PD7ZFRvRXHMGpH3Oq9ViOsmg39Su2QoeuHNyIcqK63+ocFlPvPROFpbvMGdVoEbm9NP0VajLGD8Som", + "6EAGICAzi3iBPQi03onRSpMgb2JhYBuTe7J+/WTW3UDOOzGgbjFIP+KlWyCiR8OrOKROAFRHsVgzXo3T", + "BDz2oB/YHbaqsJKOiqyee0cZtxrD6hT5b/Uvwn7tFJ3PBBWmpyKRFObHFKci44nox67DMi25FN15rArx", + "c7VhMi+0rfD7jAhM8Dw4uWfS5CyVPNNzBvxrmXhwhnfTtG7+anjejNEWRvhuOV3lai+r3yndkxPKscfX", + "YlcOo4DGiRTTIRh3zshpSfqTI0JZwfAUUJpbVT3SyrBYiCz1OWCpTsqcgAkiRekcz1vZVlZgF2ArVeJr", + "Z/ATuV5SEilZ3kO2WmgrIjXT2hVGqtANGM4xTJl8rE5TGq6kJuR2zP4gigoUgURkpND1YTX2q8+AX2r+", + "Rs8tS7Wgj0+N4PeUxd0KLw8jRWUxCVepTIPDxohcL3kWxkPfK34CXry4vmJGLCVC80Tq0vvn8SKCsYKb", + "SrrDgtbDwcOo3vBRcNsPLpqb2qBra4eIvHpGbrMfNO73c3YP1IKVrKQR1FvbyanMpFuT06mTXpQAYwnM", + "4Tg61NWeR9xpV4hFA1w46UFtpX5TxKfU8QAMgZwjqLUIdUJY4YFtZPDJeLz3Yqnk/yZMp+V5kRET+H0c", + "hlZUJGdGVqZizC4zyliqjl7iKoGErnkr3PjQPC5Plr6OFfiNSauB1rYk6W8m+pUj7aGdZRs0r7GJrRGH", + "2zdUW9oGEm0v84D78KrdB9MUC64mddTUIrYr/jE0Y8Uo0aQKvaFRgq7sCcZIRV494ws4S4UwnD5i3uX9", + "au7e4d3vhgNfRN15j2LUtIoXYhiwWhWjfpmVsTVsaDza+GBK6K9zRNPZr8cnsLKaUfbkZHjy9ZoCOX/w", + "bdq3HfhFIQybolzQiuFTdce7APRfB7KkCnpOlo0jhV2JnKa8dXx2tdDYMJa6w4/ZC/g0wmU4n9uETxnp", + "RKSw5tEutHF0sdQ55OjVZ3htZthnv/oiFcHu7nnUT6GeVnNYg4NpiJ+hYjea53/Gy+0++h26J8f7HXO/", + "g3T07zA+c8I0NqTrqtxX6j//7JVvuS7qTw1bJG2RaGvJ3Rw9E0aoRHSLhHbqTA0H4l/qlDG9Hdu8kkBZ", + "RqF3T4Xa3sRj7e4f3tcfM3w35GjEde5MzE6MmFlG2qSHBSY42iEqQAbLB764eeaelpdfmAt0SG/IRh+4", + "VoZQY5w9/bsqVtgJ47eDHlvNvb7b29yLpnds9at/a88ivlabrvYR+RW7dN0IdKC/UthIKt2BtOqbgGwK", + "NJksmKGPsFnGl7o0TSt0JUNahi8klWgZNSAtQxUpds6J/y946ntCrzxpfMc3mwr4OCKd2AWPQ4K5oOlL", + "NT/1gGwraQWLG2WgMVk2BAanlRj9pKePLGoHo1Q4YRDcDSsPpc+Rx/SnSGER6gmVN4Cm7cEiQAqgZstd", + "lQEPajIiR41wlkN6ORtRhg22y/Ot7EczLrMSbBFuhd2oE8F61XYRa5cQPL75xnaM01MOxLovvp18DUCf", + "G2GFqwrl91exb0anff4UQeAiygwC4FYYBM99P/O03u3x8UgNVfUD1bUhjx3RcIGYP6AehA94iyx0MBJh", + "/u2WAV8DEaGX8B+tMNe+ar+X9kqsJk1ogC3gRfRnsvBI3QgBDXviaNBh0AjGZhC+qweldGG+hc8i8Xlz", + "U67SSuMLgvh3+9bbmmfPknFHq6qbDYtD2vtjokz2nj7UIYQ/z/1LzqG92cqNyqFPwwE5NSaoTex79Y/4", + "7C086t/fxB5vO2b9hIaeNBuD9ZAYVIKLBqri1n2HOkNXcOJDwf9aCnb18jmblQ5k3lIYC+aod1xg4kiB", + "Dc+oHryqgy99C39pmUz3By4as+hcBUnpS61mct6nh4J5lWjlC4s7CkQoi5KdOCPEyEoHZ3/FbX6K/WS4", + "SsSoej9Zs4QXQ5aKRJdFFqoP6gzMxpNj9ooni+ojvprqX3/7e/ZOvhizJ+x7ZkSi85xq7U++Od3v1qkG", + "6ss5bNRHaMN4X7IjFv3WSfr9KY4ECDohmM86HrtBQqOtHWFFBD4+wsd9LRF62QrdaNFJn8GcYHwcraNT", + "oohWVAeJv2710O2kSZbxnE/a2Ku7WwjTG1QaYHg+yeV0e1H40MhrKwttkYvzwo2ozCThBZjb7+QLdjKi", + "v40Mz/0ygtMfLIl6izHBMOwgZdTRN8mdT9VQRoDiRG5vnMMjy8rC4+P+jv0gX1S9cOaYDnpze8vgIGQb", + "WegfPryzp0M2esq+Z6VCRVukLXKOdlHHPRxFTTWZF+Uk42uP3t8mJk4CtpUeYCfvhOPZ2eXHlxenQ6TY", + "5fXHKu+xfwy3AJWmYwD4RCYca+0aL50eURPj/WwEcqI+Xo1zvJ8CjT3eaxc0RdZN4z1Q5vDW2wVMHzSM", + "dIrqxnLQHvvHfY1NMDGWys60kXOpqDIv9NmqveUJV6GKjbNo8PJFNGBnkYoGr9QS/pdFg8bkse44y0hz", + "cJoJkHtLnpVizP4g1pa0Jw8IUqMro5/PnrN4Q6rFQxa3mTAesvG4B16wnZvX1ZqgrmidhJQ6ZvSqyrVG", + "f5cTqu5+jWoq1TSq5VnzCMM5lYqJ2cwz1eclL4dJT9ddk9ZMWlsG5z/M8PrjHbrpXbthqs/nbLRROK5U", + "f/M62Tr8nad7+zjuOj0dAnrH3TLsvrW7ZXZ1ZPYqBzftE3qYnnDQ9XvUtXnY5XXwhXWI7D5UXh8kc4+U", + "mvsSIv//zX17me4jHvIu12MW4OV1QWJ+zG4FhmlRaiKGhXBnRmBIn+pSlsIYmaJC5YFFKMCLGPgsjgbR", + "IGYnvhsVff4UBFr8JGYnqsyFkUn1d6cjdfn21cVN+9snKMGx+nnGs8xWCDFCLdlZU1099eEFDKTSWu6F", + "KDxIREDxoTugDwS848gdgIe1fQT3o/TuOJL7R+w6ooe+tUfHfCdfPGdPmiVR9Vbs2YCGktmp5B08w4as", + "OPSdTdlx+HsNWbL/pZ2yZd/rXXGmW4+223swKROh0foB4zql8gV12w0sE9PTC7LRL/uQzkZhZukdt/dd", + "tdvAfGXRn/zkcU9zaS262cAWa3ptuWWpXilQhEDpcWLMXvOM8lSyDMwIWIrjUyaoIf9zbOnEsF83foTM", + "F8ftvWUJJt0Ipcv5wgsjey8RzokQQSrMEkIkmgq20saKvno9zCqal531kTjNxopgBuh09vA0TDqL2R1u", + "JBUIPKGW0miVC+Ui5U2lIZNjMWZKT3W6xmyeZKFtKL4LKMy90zOdGWGOqxSU5ZlcCq9V10QsDCllwyqn", + "B3fikaVKtAjviZ+1EmMW/1PKZbaO0eSbGYlo3Fga5d0zn9mPdAcPbkKUdyOY5zLL5KHdRvANU6ovKurD", + "j/Sj728BoBHuWcXmG1BbNdQgYeVLGymv64dAAwLqcJVmcIZUGsISPnYrHcGCqTWlgSHPRSrn5l6kzPvV", + "GUfQNOPKooHdVMOAtYDZfPS+HR6oscixv9vB0GfdvWZvKW5ZVxAMKRTD6YZB/0/BDc+FE4bQW0pHCYdG", + "oEkWKY8SiQ4JmRcZRi1sdf56GBK0B2xC0A0xgcrFTKxQCA2rEv9m3hGbrv0kjaW6VOzcIRBLTeQBgaKK", + "NQVhEK7CDWypA9N5tztK4Xb0lWo3JdEjW7GPtEw8iKQEA7M7GdJJl+3u9tlnD6IRaDlmGrYkVg1iA7PC", + "5NJaMvLgKdrvaQ2tiXGGXuLVEqQloYf1ddTc8rYE6FKOb0XOlZPJreAm6e8F5uukurr3Zjy5x9QKrJY1", + "umA+JEpJl8TbIVTD1ZoVRszkA9wKCDJk/NYcU678eVXOW2Htp0+22wgiXCUjLBCmZ+z11dtXHoWNnSAK", + "Bqqpp5SIi3JjvxtLqkkFP7KpbeKLLNFWKsGszGXGjXTrMcNMIbjfgyD1HsaTJ+NnQOxIZXK+cGyWae2P", + "JaULcaAqTxx7/5b9tRTYLKhChTklsyZSYIM4HU7pc4xBsfjJ+NvfxDSqMzJxLNGpGFGcnllkEjj4Cc/k", + "1FQpm5c6FTdc3WN1/ei//24jB7UXaKVqLbcV83Oi4im0YdDx87dmLCDW+tgcBnrpkLPVF/XHL0zQF9aV", + "H/onnmWjBOOn+CSqiipZDwlAkJLOnmLiec4zn3He8oL19is6NoPitcwE4v/5tLC/UQ7FcIMk3cQlmMev", + "0mn6c8pUeopupJ30hrXwrgp136HpUMKNWVdAPD5Ro/uuIk1MCHXUROu3KFB/oMIHL5T8gBe68nVbpdCt", + "GpXWGlrk2rHLu4ugPSWPqKv0vPMFrQWqMXfl/dwuuBF32p+Ynqs1NNnenz1WPdk5lkxFws1tpadvViVP", + "Znhd7IJEwSwVlorCLRiBhbNcY/2FnlGSuhepe4J/u82Yfjdt0RXZfhIAqFiykFg9RBo82BrYIO6EdHN2", + "Vjtf9s8Rs3C6rbAqwN1PMjzIU+FWQihvEQKJqDumpZ04C4F2Qs6wBV+pgPPT01+ZcofaGZqVGdKAY25h", + "M1eWSVh9D1qCj7VU7eyOkM80q0C0YYOZOjkRWXAHijAGSCYh9jwJDZI6MSKz9SiYdM2Gagi6s3+XeSEn", + "Pg+CtFgstBicD5ZPuyTllCf3QnXw4Av6oYkWRJhP8VzH3QBie7MCLihOVBi9lCmG39RcGKpOArUHhLsw", + "Hjb/TTmfSzV/zRPhI/fpMFJKr1h87T8wvnp5chpTPWKs0bl33tLLsD+kLoTi8tyJBzeqpjj6ZmRznmGQ", + "b6nXfC7O6T8j1P6+OX/65Nm356jFxeNIfbTUDbYdnnSaSn6MYHzOpbKOYo4NaLl4G6Ep9sfD56AIgvAe", + "UT1BgP/aTd9Awf0kvpcqPQ/EgcUSNWIMMvqVx9v9wOtMEtDEZSKa1i3z1/mM3ws2kw+uRMu4xk9lHJXv", + "2/BqsNwxD/LgxU1yrjBb3Iu/3ZhlVRSyWjqiDWGHrlEQoChOI3VS96BCJTuQ55TQ6WZaEyAt7JBlnM2N", + "EOoM00RB/Cr4VKo9Ejq1nKfqtkKYnMPFS6/gQ1UiYaRO3tzdXRMaf5gl2O1LAaK+ctOAgUI+oxtsWuIz", + "OpHCBCjOXZXdCiKOssErFHk//UJn2WmfKxH0aeuagmKjqRb+zgK8arDH/PPsJAB7Yx4i/Xi2jL05MowU", + "Hckn4+/GT4Gq78ssaySHYCprE2QN5loBwtkD0ZTwwEwIYnp3OW4rG8PfVb5y0zLqDAMLkop99+QJy2EC", + "wd3reSu8hG4NuofgFKAH13C72HCONijdhKvpqrkzYh7wpsKjh1zluC+T0nRI2R+ke1NOw95htQ5ieYAW", + "Hrc3PvZbg+ssCY7tMHQlpGWTfw7oeqjvd6AaTapmpfvOuR90hN0/KvkEW0GVi7zmyMdxpAIdtGKeMmTm", + "Z2vsyetR53x2hgoiTxhgAnSmS8u0DzHW8QSYTHsivrx2JVFOnFjhWHx59efJH1/d3F59eD+5fPPq8g+T", + "V+8vXrx99fJ7RBFseiPwDEg17z2yfrQJjrY/dRMfvoRnvX7c22Q9qABbu9pWJjYO3LAjbt6ARurUeDpV", + "p5V0yaJSyMPV3ms7JFWW5a6uo1vDbLaTIYk/GA7oPhwMB3QX7s+T9u0k/Dw6l9TAsDmyDiVM87iCncMq", + "cbxv1c+/WZezs7SGVkPej+5Cq90gxH+LBfeP1kOK4QC0NOUmvb9bOVccNJkvIuRWadMBpN3jh4Zhvtjt", + "+2xvNdPXdgS2lva1ypm2ePFXrGi6E9Z1iKm+paUyF6pbuaq9D9VDjNsaSiIEn4Iii0HASN1S/6Un9U1Y", + "PZEJBMBA66T6ZMZ/ltl6oxS2c+/1fU+CQAu9576bLEb0AT41j/AmroajjnXhEV/hlAlbZT8epIfsOM4/", + "d8Uk5c+YN0l9R04IGB708efsicciqLCvTne3RK1qqiT6mdvQ7nuiW/BQHy175UAnZP6NKPTIiIyj6VPV", + "p1Mk5UxUHXao6qzQzGjdE4bbnk2plMheSNUJX4f1BFlvXJ9stz16eajBQyMNP3dGyiA5hqc4dJ1m+vGq", + "24PUe7k0zeQKfDPTZTrLuMFEgLnpUUr7NdttOEgaY9ggSb3+H/cQthtEDFd+DIhVa6/2oStVX++f3CV3", + "PNPzTvRNsluPnFpQQvdMrf78jrn11NnszDNZeOjjDoQrnot05PDTrCinmUxYeJoheHxK/UJBTpz2NpBo", + "8hi+hCEdmdz3lVh+LmdiBfjECrcrCyvRShF8DT5OdivWNZ6E8g5CbzjtPlLtbPY9eQJ7krur4H3jqCDF", + "GrvSXNa+na/T5j5z//8O9q/D+Vu9jfUIFUdu7KSPkTtd5blGKuDx0RPPKRM3GkSDunY0YCQdLPb7gio7", + "rHRymcBtUzu3MESPa9KOrYWrfYEi3RHw2x1J2R46oi2LBqA8RLRv0QDnUu/K88qM313bq7QTu2I1HfgY", + "n88NJHC6HTnXJIwQ6+TjzVt2ojT1f0UnRcbt4pR0wUwuu5eyFVupAiZowAJjUUQlI4ggH1jZHU05KBGv", + "/5YMkZWak/oPPOjcfXAsOxB/fLbAJD80abBbAd7cnR4X1yTxPLmHBj0KNImzH6QDXe52rZJeDbDQWTbB", + "ZLwlz5pxre3iKtT1yKvKU6ES1Hr9G+zkyVk4Cf/17/+xkWuDGfUqWzdbBSKTYf8Q6lWINfPeXxWzE55Z", + "3czti1SYJJMzFq/EdKH1fbyRy9/p72rBa1TjddSXYnMWUTVS8DCLU0GIbKEvR2NwWGlR2sUIW4moSJ1g", + "WV5wvw5xdpuTY1oF9/vp88aS/+vf/yP0S2QzQek4YMQpFlb+nMU5VyXPaOSAZKEVS0XOEUcp2GbhbPqp", + "wkVJ45AaWfIDivebxDqQybpPVeiQsLclJn2qKS93dr+BhxifhpiBLh02wf6vf/tP5jvJcMc8BSLV2JrQ", + "7ZYC3gFqo6L99p71XXSttIKwyr20uoMrtfdEktLS3fY3oEB4vcv7nq8v7tqdVjznllZ4X7w2kQrHE2Rc", + "hfdAmtxJGBP56v3Ht29PfcawVsJWhl6kuPXa7Jh9++xZ7TSQDbhGKojkYY6kYRwSffm0i26LcnoA2fqS", + "c1esyLBN+YMDao3ZH3mGiJFpFWT1tIRlC5WYdYE/ukgZYV2oHGCZvBcMs3KkVs+be4G9aym33Qjsq+yr", + "GQnZ9M+jDxelW4xu6bGF4KkwFBrEmT+yQETsZQQ2Anymrs3YxKHY60MjYuxgxJ2pNPuS74POu29GPYN7", + "jJfe4T+7j0j/kB/tDu9/pb/01NhiLLZCFEk4JtFkWs2pDHshlJMJlp3ciBleWb6OzRe4BkBrwhqhe0+g", + "ug2f7o2m6oRnE3+iJ8fNMedr6jSH6WMbDdBQGODNcRZuEMr69C1yyL+KrFjBgVcsz6zj60jxLNMrkVb9", + "lDFbwaOOPVAv8jfcIp24ddQBh81LbnpDmkZ3qf/vxYoZndXZfggK3qQz6yazTzvwdI7htbh9M+JjA99u", + "+ccj2LeCVG8w1Eaai3aLZqVVqITEmwnFMJZW+ErxTPDQhiwYXZGiWslaArBrTsi3XHm8uFANqQ2LG8PH", + "vuIuUtKNWQxHNa5w1uuelkgdr0CnXVWMfzsZ4Ntbf3m2af+ZqGO7cXhowl3A8xqzlyHpBDbf+ua4qCFU", + "h5ktJa/xhD7cIGLuvVj38W9jnM8vEaogpqrX6S+Hp83+fyQ12MkUeJ4cQOzbJ9+cVnJEz0BcYELEKLQl", + "qz7aI2QMLFtVYmbMLnrEDDNizk2KTbxQNZIWO++NI/WSTA/MfcHA+PPqeIVtJ56ry6XgZTiSkaIeas7o", + "tEw8MgK16DvxUzrFgwda4goxJZogzX0sAqdwQge6BWbVLwwPlFVfoYmTpOA7MRsO3zPfXe2ZGo6AHj78", + "sUcg7E5M7k0mJooc7jaGof4k3aLqo7XTb0zf3hW+a3/v/JcBz7IPs8H5Xw7p3j/sSegMKdF9sNqX8Gfg", + "dpTmmBOehix4W7Xer9pn7M/svBfrwwYzYqnvRRpEocU2Hj60ePCI6ItDCLZOWJN3GlPEEmpn7lP7g1wA", + "VraO5wU7uXl9+c033/weLH20cOSsFmQLUD3QI53p+RybCGxU0hwhlTe7SHRu0hYht7nlx0/DwRb4WVev", + "O2rlTjBnIwKKRw60Q49eYHTOCBYNFQql2dXZh+36bT/TCi67P1+iCbq9txVJ6CXwxUDTbajv+rPDnpl3", + "HcCOlKQOCBSR3PeA6LwFxRFdWg3W+nh3OWQ3ry8ZMRhZ0I2aWso0hLc+HySnEVfoD2UWwkidyiTYpjhR", + "aUNOWU9XiuDm7lgp/sZyYYH5huHM5I0jh0OQXyS4DlRIqvwMDB7VL/b/RH6ZXTB9he5ryXMUNuhw4PE3", + "4Yh8KVion/aVmukdqfml05M6CXNf8mFIIq1yV7M1azkSPVxS7cryPgtsG0G8ATIZvY8T/xTi0rIgPQX7", + "DbqJFjyNFKoT50hfePJ0zFAdRA1n2OoUTJZDmAYjz2HWq+H4oSdWJKYrlvjm3cUlox/H7A7mxbD5iLLS", + "hZp2ox135Pqk5AJBC+iMRYQBO0Mdr4F9P968RT8et06ASqc9wR7ZQE7q+zIP9dcgazCdOhhhuE2XV3+e", + "XH988fbqcoIt7CwrFRiXhHMnCqFStkY4YYqwEQTZIW7D5hK2KDjcYqUdPPkBx+zox1X9fds1VmyEgwJN", + "UpFJTPz+ePOW9G6EoQjeskh1xI1aFeyEXgWEigZKKxENenL0a2S4LUFoBItp8jFo2MLi3TdmMRE5Rupz", + "jFQxnxvh6T+OVFzHWeIKLCHw9Qj2bmNPT6SaGU7NLUojIuXN4+DyDG17Ucd/znjYap+jq4RA7CAWw3Jj", + "qixWOrzsMemkZXW5OQIweIqjFfoIrXRP+2CXBwlHww1aAaQh0na/QPMssBNUzHPRjUi0SmQmPpA7vbvw", + "yBcC+al5K6A2DmCke1kUBMHfHwDsD4t6y6FHZ+lunxoyJv0ED1lkb1+GRib8tr7kF9mdmkqr7fzNRygO", + "N1j69qSri6Wnd+fAuywov3f7Y40VTer64nq7axbYsfGNfQhG/1dyBe1qb/baCDGC77RaKnhh5d1oKLN8", + "C65+d8tGB5O3Vy9HGBDQhBrcbmx6IKTJRyXh3doLAo91vn9oR/Tg10CVIXy2aokdCg2otw6Zbxztpkil", + "whdVsj/Kqk0ayk0YeghWx1RQj1eu1s2O5YjOESkE0k6Z0754Cd02B5befB0fhs/8aTc46ndZ7G8f2WrN", + "+Vk+is/o3lkfjyM6du7yVFQfvK5x+DfbZwVfXga6RurxrHnNQgQNTWgV2CYHjQdiiYUsCPYJbSiKa1Et", + "h0iZHxNL9KV3dWM3gjCiVDMdqRNSu4dVAi/+b6vj9+k2nGuqhVWPXKTgAmbcJyQQpoMzsuhybh/fNfbY", + "9gXdF9S+brCbu/SVm5ZvMcEXFO4f1LF8c8B3FbN8jU6+e3SEva1+d/fx3dQpDto38nkj7HpnlZFrw692", + "N/P5qn139hKpB1Dnhq+2wXSqWn04ggSiQskToNbCoknrxYI6tJ08HDrYDzbUIYzZe40I+uH0T7W2dH3w", + "osikSNkJB6NqKXVpqy6FLC8zJ+l3yuReo4qOq8NFDNkKm1xkwiHWOpqPqcZeKcLXsVJRRqQQmQfbfwVk", + "H7zIC/T4uxHF4BOj1ToPDU/2w/B81R5HG/x3SMsj2sq69dEBrPoaVbQbn5zTKV/6WSd0u9yqiEDCVo1i", + "Z1VpN3WRAWMsJv8mujcpMxPL6uGo69LFQyZcMmZXuA6MnWYUm8JUFL5qe7KwfSwmSSmeMcrhsyzVYE9l", + "gt8/Z1RN2fC1ZHpOHBQ3j31czxWuJxzkEBt+Y688XQ4g/7XAFnWfSf8+MEGP+dA6ZPTsmF0EvD/t3Yxc", + "sYAnEEcqF9zX/IQXFxyuV8Texza0oTkfoWfi1dS2UwtaE/BhpoMDzpds7rIGd9N0l0tug6b1Vb1h0+XP", + "vuvFDxNcheQqp4vRe+SyF++efcfwDVv1H6winlbOVaRmGVo75JWnNrmPLIOhTlBZKbT3bX0PwtMJA9Lk", + "lmru025wervQKxYNPIkLzXyJfhoprVgmnTDY1+0etPilMBkvogFb2jGLBgUcMOsBsxqSO7hf9guxVCgr", + "jiPT5j0ha3JVInrM7vScXNuoO8b1bsTkc3QrjV/DosnMBiBogfEbp1ncEvbxoeupemZ2yahFO6GQOgak", + "wshlE4q+yYqPbKQIsVDMEdKHAEwisiTOYL/+CYPXA8yliwaNv5z2YUuW+WQhu8r5L+n+9DNpcB/h0hIU", + "aBqCBeGwR4ou44QXeD/nPCX8ROXNuXmmpzwLt3Pd3LIzC323DGptSofHdz01Mg0dmpM18MVfngyf/li5", + "5P7X/xxNM6EwtRHWgGpFpHKpRjl/YAo2OJM/i5ROI6wHWTTwCTv5X//z+yfj704pS9jPZ2REJpbYm2YO", + "t7/hsFJQPsAyiQZ3uqiyEKJBpAqusFeEcbaKZzYQvvex2W7ZFZqptmnV2PdhUza1j+ABAq/fQqihwI+z", + "D5pqbIeNQCLct+XtLB+ssAUbN1BofU1pIVWnV67ono1UWppN/DZ/uhKNULHNPrpasVTae0JX8UmaXjDV", + "rpQKMJTP50YAI6TPgzT1YKO+8XwV8LhXehUyXkFVpNboIM8ClMgGDusRBG0oWx1U9ffmbrLiuSck3uCb", + "qZc7LR1bCSPgvsZjBEItUmsfp8BsXmyDj+k71cVOy0rZSTObjjsn8gI0ZZw00VmaCigSeyXyohDcMO27", + "m6/JRR6pmG7r74NeEZxtclap4YUm/G2erj+foE31qYuiO2BS6uOPsoH8YJtXDPMXtfWWBW4NqJobhK/N", + "IWlDOBXFj0NXGJABdhdRuxF+WGLzrFQuZVrWghgmwhZyvgBmJhmdfQl1+q186pI9c/YAbgOOYiFzqxEG", + "R3GcSzq7JzGtAb4ZnyImO1rM58gXj4yoGRIz61DERcoLg6nP4LeIjcwWPJuFw7ygC0T6NrnexosUiAJe", + "WO9N4tlcG+kWOcb6SiNGdEfMuBrp0gW1HoYUoMcKO2Z3Rs4xhbdZSIFQW05jZtcMWBy+/vruNlLUOp74", + "GBmeOLlmAuTpBbdsChay/yYobWUFyqXEitFmff6u3sLWvb677WP6XoRxLFj5t/+s0F8JHH/MYiQs/Vav", + "hszilM1As5uWLlJKk+EQYMQRpqmC5I0JP3fMYt80dOLNvToSFrg8SH7YdY4Wmm0Y7HgZiJTBttGNQBI6", + "bOWJFYLFzSso3uhIiuUuuKgBQnQ0Z9NdPSnzQ/LNNzbizr/Vi4HTcJH5i/iAy7y1vceahLvUkAPGvqvp", + "0BWLxQuTxJwtQPWVznqsMKkYAepTNdiY3YRNpkj6VgPdgls4u7EnvW+ie/H+ZdO1lDjMNgC5GCkeNAcK", + "+taciuaE0g0BEzSOgMblOTdShtdGL2dTxALWM/azMFoEaYeVM5hoxS2LqP5eObSfqGVoYMRiwa0ICBuM", + "M1vmuADOcv5A8Q9MiqX2dyIIw0jBg9IysPLyMlngUhrLTmWKridrvYjxyyErCXRiMKvgbdQCpGMrLl0j", + "G95meiXoSI3Zn4A6hTAzIEjBDc8ykUmb01RW3Kc+wYT8558HiqoAuo4fLfPnsM2F4YlDAD5y2TU8gjkt", + "Gt7zuN88o56HChsX8DUJb6RkuGrtxMLvIo0R+qq0rP4lSIo4TALrtrztw9eeCkjDhupHDUGkTbgJNK/2", + "DFuE0NUAU4UrLFJgQ7dJzbZueF8jBn82wlK7S7x4lF1h5EyvVFDIqrtfK7hn6vVINUE7wzd4sAL2w4lh", + "pBA2UhiGhwPUW2ByXBxXa1ocqNPYBJGexBs2HBC70Cs1jFToyi9Y7HQR0m9tDJZgxb+EPuC5N9W+qQTe", + "nDHwwyS3MZlJlIadEv4/8UtFkKW0Pt2ZnGtDBic6n2a+T0qtXuGSh/Q+RmVGRk+lr2rE4gdKAzfShXeD", + "Y4nOepahfKGMR67SIfpBqBkyT0cIKugRI4fsX5/+nuUYofzX3w9/H5D3tiJAaG/lPc0cqlU+st4FhJJn", + "zD4oVAa+Q28aVpyzBLgH5o0MVNmfQ+qdDR8FYgtTsRaGW1cSOIbNSpVU2w37wEk6IEcGAga3tgQtdA3n", + "bkGlNaiKI6LrJOcPcaQweX3M4ncXd5dvkMkE2LAYsMNPIk8vdNbo2IJaCgho3xEDz3qiC0HdQew9Vawm", + "orUII4oqxhwpy/PKp4An8D2lU1aVle8rLzu5ptMwIa2EryoA5cvpgnid9/ojyDTO+UPv5gXpZ6Wa13fB", + "o3Dsw1a+4+uGHcWrjfOGK4li0jKet0QIeo45W3GDuH56BtdgpOD8kwDWG72AtmZvy7xz9n9spvz6y86W", + "OZxWfx/ULCFVkpUI97RYp1TOiqZFGyoW+AxnafTKknSuX6wOKK44Us0lV94ZvBtA1tCNLyR58Pjay/pQ", + "zCQdCAe4Oougy5e5rbFLM6I2FVjXtw4ajeIBizP7iEa9zboIdoPSxBlZBNFbQ8zWeMD+VqTlOfHgujGY", + "Zlzp0vV2DqD5+msm3KMt/wZK1J5vl7a7J8F78m5Z3MBhw02GLjGq90rrO6ed9LvtpquvmI4K/koeG4Ft", + "VaRqydpHjZs3ZmhOUGepSIGWf7J1Kw9Z8445jfe4EOsXd8xt261Z+/EqKiBzvv9wRyYVahpkIjSuRrwR", + "x+ySh8ZdcOO6zWvRc6zwJobXRM4rC8XrGxptxKC4+u5CiC4FaqKeYdjEkvdqUyPxYgPO4pZuspdifkI7", + "CVZLJt9WJO1mESOszpbdbPhW8zTcvc3LL7B2neRBMp0XReZ7LYhIwSHAuguO+ctsJjPsvnTr9ZvakY5e", + "uqlIeGkbumWEqmZhxKhaiUMYGh7aQKNCWqlDRWPx2EIl82W5Vogha+r4VZZKnQbVBwxROUT67pXm/Aqj", + "p3X7b1uZIA2XSPcmeP2qR+3Am9m3XBoS1cDEonqDDAxxp/EvGbedUmy73hJhdZPSSLe+BTPWaz9gwZuL", + "kozIjTwgKu4hCw2NoBge1Eb+jFLqnL3At1lUPnnyTXJ59efJxfXV5A+v/gX/IGIMRMNQg3M/UD3ThXPF", + "4NMn7Fs50x3hgru7a0xlD1I7TuSDB1mO67gWwqmT2pBykSP+Nym3K2lwY3KO3DVdOzGy1DTM36Ft1GlL", + "l3vcAKUNihQIyPiMF/Js+fSMjPqYOZnc26ZDL/NYcnEb5zbGJDuwWkONoB2RC5k7TKyjujqWcTCXYfb/", + "8A/soq4/xfs8UncrXV82vMTuZm4R8E5AZIHqRXmHbo2QYtk5vDhijx+/gLMjjGVndYDx8eNzFlN/Bb8y", + "+OoZ1mPFxMdYBch+EylW179iz2YEPn/jXIFQLonW95I2KFRDebPB/4LFtsrBd3jpdM5hYRnCo2NjIRCo", + "yuEKRj472gtZC7LD17MZnWXwiZk2WGn59FuW8rWtZQp6q0O7F1r45dsrdsZuX/4BV7uLe33Vludc2DPv", + "3IQTsOIWRvb9qqfrTcIVcnQv1mAsYSNwrPReKWFGqD8TmAOIpamAz4TiudrtmxFCOAgUjj2367B8kkmh", + "HDGGR2wSKi20VA7bZxEvBF/P6TmLf3h1x84WgmduEQ/9P1OdWEyrwH8hKHEhx2ueZ9UjTSaYau2sM7wY", + "eW6HV/t4BbaIVD8EpL74ePdm8vLqloCoyZtj7+G+ICGN+RcVmnxYCztJxVJkuqDmKMBW3rvjtWtpfQnf", + "KZLiT5sVM44b58VvVRhPxb2+UaALRLKRwom++PDh7vbu5uJ6cvHy3dX7yat3F1dvY/Yb1vnr9cXt7Z8+", + "3LyMqW2xSL0vEiUyoRqczLRJKCnCn+nq1GgVZDeQ7HTMLlgm5jxZ+7l4uRljjAl72iFuCEu541iRIS2T", + "uQdw5egyQKMmUrFQy1G1X3GoyGwWZHI/wSBcQhImT1Ns36nmyFz+r/FCW3IUxhT3JFONS0XFHcE95n05", + "00Z2p1SR+njzNgTELTqIVbZGAzKEY/2RqJnY8XvBOIt/gTE/xezjzdtI1coVDua9Po8fExWf/pYtxANQ", + "mVKQ49s3F09Pqomfxo8fjyN1Sa0X0S+GiQYhMeisgsJ/w+3iGpYaaHPrjOA5MpxPVEEXVYv3w9tnNOMz", + "qoVHtNeYLbTSpW9zHFNJW+wxXs4jZb0u7385R/3JS/mzh5FKf7JwY1hELa/wdrwLhKxTJVagAIxS4fsg", + "M4tzRjpcwVSujZ7Dxr5aCuViRgqAHfrDEal4IbhxU8FdDKcQjDk8i0+fsMpr+yFLg+jxqrhQKSg2NPFI", + "0ZIwUhg3F4ELOGVzQbYfcbnn1tE/335438wVQpK/AiXJwj8uQqZV9QyWEtfXG/a1tQteiHMW/xJ5kLZo", + "cM6iAYlxnwdGYjwafIKNbUnEwEooYsQDLCa4XbBqWNFza7bkRnKFp0SGxDPQMSlxGUan5C4afTwe+9Gq", + "/pLng1pjgWM5aMC+DpZPMY+fBPHgfPDN+Mn4m0Gjf18laOHkngU5gBhaXfV0LzFf39v3NQKTXRip7hn3", + "+UiIgUxXc8HnwrK5ZuRAi9TMCOoQiGo9YheVvi1NxuEgrox0wpIzqBZMyBwLDtLZukjl2gj8kdRu76Ky", + "ktIvpEJ2hVs742ZO7r5cW9CdUGTD3KSNVHUtBMf9ljGFjUAt3MzOew19uSwZEytt3CJSqSbL2We6EUIY", + "olVG6nIheHHOYqAE1WpT65w4UGKCNIqRGJ7dsa83NUQBi8wOI2V9ChwoOXwmAlQYubaxZm/Jk7LMvavX", + "Ox/XYSEVIWlFzopsRgME3CyQGnS/ApPS8jAqkckl+eakC1gYRswyimQLjh5ZON94oVSAchTb4JaVxdzw", + "NORUEeaFQIz6qnKtzgHE2SVcgYWGHhxgrVLdkxMTcySNmJYyS5+DnE0MdeTJwjeApp7n8CvNUwVfqwxt", + "775Babq2TuSItpcjep5vzkghNGyCITY1sdZpOTMiybjMY9IZYgzzYKyQo5iV1MVTVT1KUCsnP2wIVWDD", + "MYNtQZVeUQQTgfjodmCEAaoE+0lPKTuEUSevYdXHqF5KdS4XfIk+cJ1X9k2ii/WY3VDfJkzRCk4FwjMK", + "Vvm0dC60OaJcFanVVTo4H/wg3Eu/8tuqv5iXoyAxnj15spF8vim4EXMLY5z7IqDtgdCO6848rfibgDk/", + "DQffPnna9/VqumcfEYIHlHWR0kvf7H/ptTZTmaYCK5C/O+SNG0FlkPajqgFD0VQu8xyByTH/wbggQK38", + "WQxJ6qSsDgg1GIen2DDmhHRCRHWGK4HPbY3f8iMM0cOzWO5NIgPjvGWHtL+qDxiBR2fCtc7Y7hPlbQ2w", + "Sf2hbYiWFciHnN+jnnzI4WKFtlgihl22fQ0szOUcfcE5l6g9WoYSXZhRzguaJgovH27jcBtkmU4oRmHw", + "C2koFMIzJx6c4XQNDRn52RCYj/3m6fi7/7NCyKSDOUI1gTCFMs1TlACPH19QGCMcOiReSwxjE0Rq2iRV", + "dWDh9nj8GLYaZmJXYFfEz548iccMTWCufAguaP6JtghqRBcPDn6x8VtLamLg3PvUCWcf3cDBQ+aVYJ+h", + "hpV7fB14EixL/+lqTXoGd40uygzpGZY3ZreoRMbPnjwDtdSIhownD0Loxk5tI/bwgOf7+Bz9ojyl7mMU", + "0xpSmJRTSh42l7ijzJ7ELygw34IXhVCWGkAjU/mwqEwFE2gQY28+UimRjbrk361wF6XTf8Sz844gwL07", + "4IWm3pRfRfbVgwTct0/tfAswej/9DYXvOwSwVFwl4kOgQJcMvutn7EYGhgegH7P3Pq0HPdNwpSG705Mh", + "hyMMhxi9Ms0o+/rTcPDsybNffX0XDQ7yndNI3qFXmm5VPBfjX/He+fbJ778aIdA06ly530kPbIK6xkJk", + "KbkDg9RAxYTMU1BPyDElqVRhvnB+57599uwQuvgGgnRDftH1Ci//t/0vXylbzmYyASP01mnD55tX82Ut", + "9AKbP7KVCEE5+NnXsJdsBIzdBe14Sx4ncuvOMZ2hYkfqwAlnxjKZ5yKV3Anv5GMoe8fsGt2ZZJvmNcNX", + "/mWPdI+WvHd5+Q7CtCQ4z3MjBCHzD70N5B9BZAmT01FNJc/0HAFcImUx2l3D3ElLwbIUncWP2euQ+qvV", + "HDMXml42adnjx5Wcf/yYDJUUA2yUODaMFGNTMEJDyDKk+fKUegrAhQxXInsvVuQOs43n8HYHPiVH6U/U", + "x4fo9t2Tb2LfCTO+Ec6sRxczJ0z8vNbG4Vc4PGmZgeHEKqTtgiME8quA+1ehpMEbFYpZc4LBJSANfAfB", + "Sy2mN4XSZjBrc5Az1umCTXEjGni/no5pSfILCVOlb0i19GX1AcCHs6ffjlK+rqDOMjkTMNYYduVuw90J", + "u+BdnmQyPn6Mci/VhavyS0Hj86F+dF8TnFBZH0WYD7lGCVMFhUdZ4IivHgpM79DlfAEKDsulKh3lfvyO", + "/fCCvJQrbnJ2e/uyZcUMWZGVOCLhsIFahJMtC6TecxYL62SOtffe4xTD9+KWVRHDF5qF4r6tKgsvV6Ge", + "0Bfa654cwdZhMcNG2lou4C6QNh+z21UVDCa9OGTmegQZT2SKCVNeEnFn1Q2SIvCg9ulCKB+pi0JRjc4w", + "920YirrsihcwjUIY7EGMSutUazcMRzEoipHSpAk1b24P7VZoncGBC7NAPwScdKJL4itavJwGrWskHkRS", + "OhGYpBURFQ+YZeX9Od7FKwyeAXyBXV+9ZE9B30XHXyByoTOZrOHCsWUBM7HaeJbW2VKkjV1p64Ms586B", + "4lwrtaKSmIkuYJs5e2wVL+xCu8fnMLR33CQ695jldWIbsBqbChQgIAwRsqOOBlUngFZgw56jae331sqM", + "Anc+EO+T68jTFcwZociv77P9wcjK+BoW0iQn7ndzZXpKVcs++6hiv0iBkWUwYVP5vNDqQAJ7WG8tGA3n", + "jWqWEOXYIwPXjnP05JA5RKecLgDcwUj5NhXMciUa9l2ScWvlTPpkNBxl6EESp2uGcVeC+h+S367KzUed", + "HK0BPxoWkfgSQNgmPyVECmvJ+ki9qC9JzH6gjkFFCI6lhnyDjaIC3F6ScWP2Wnr/K/2CVzOoPJh96W1C", + "5gxXlldFQVWbIvw3JmcxvVJDrPSpOIuc25ksLCKDNWD93EqHjUaP2yy4Q8nsilTHoco4+fd16dB4wdPv", + "GZqcjrWjoHIGJXvVGGRF/wcbqdrv0ECMRnUngNNzOMG+Ya10/iqjOEqdbVo5Fod+CphfBAIfrL6PvUZf", + "w0NBcZNLw+0CvZ9ufV4pLTDYT7o0CtULyp1BJxA5E6ciUChS4dND3EkHVqPTOQWJhw3tINH5VCoedpT2", + "Dj9ZmSrUdK/Khw0pTr5IjOW8QDszEIqKDTB1cx0p2pkxu1C1HSxSYkXZSNd5XvXi8E94CR/iA2kVpNYG", + "49aCNHOS0BTi5ogXhjg72KnQOhTFKhSwBxu8scXhXgqh4oZZH2w3qsUucxBXbagzb7TxLBtpMwolHP4k", + "e03GiJEp6xQKIi1ciejX5LRx7dR55/PsRSKtQF9uJQLrfQ13MI0ZVKUi44lI0XWgVx6RaRqu5WrSkUpQ", + "VlNyMuaDVZfqkNW/TUXFPFX5YOMSilTwSpC/rvEa6qao+7ReJJ9Fh5/Bn+GgrGy7WX99S7ghVrwZ/L8t", + "3i6Ld0PFHG9GBJrqJrrDKMLoLeW/B5v3xitMLQ2x4ThrOIE/2/b13+q3fW9KZVl8fXPxw7sL1oj2hFBX", + "SFTP9VK0/NIhOVdRsmQQ2ENfghYwyP508ZZqCcjXx27XKlkYrXRph1UYCEV+QkG8/5e9/1tuI8fyReFX", + "QfDGZBVJyWq75xsp6kKWVC7NyLa2ZHf1dyY7lCATJFFKAjkAUhK7oiLOVUec24mJ2E9wHmCeYd/PQ/ST", + "nMBaC8hMKknJlijZNX1VZYpMIIGFhfX395Mu2AKKaZMhsydaBdDaE9wTnIV0ODPKs6lEiZtxXlqvWrAT", + "SNUK5KvLCVNt8d5GmCmKcOMhJVM3atDqJcD+9/YzGUxRpYRJYHh7yI4VvH6ITCcKQuO4kqGK3plS4fFC", + "AzLHem5y3Kf9Ck0Cp54o0NzLaeFSOZmHH/kJTnSe0VXlHwd9gHTZSGxJgERckLWQnEW+WwzYo0sZU4bh", + "3gWY1IrEJgH2HmlELbvotFfG0A5SZTdqtgYsyR5LX23/c5qoSLJHdQ31DEyom5gLaCn2WyGjQWVZ1SId", + "FyQTE8MRxQHLjiA2P65rVZnn3qeC01ebNaz8vDCwP5iW1WPIvZI7Bpvvzc/qrfxzYpjbaci2UEQcEu+1", + "C9ab51KJtmvoDJ/xoxHi1Iv1hqLdNEwt1L3J0HYcDcA4W+6Bs6DnpJJ29vu755Zj9l467zgTj5vtBN6O", + "2yq7mZBvpuI/+6aJzRmDBuFva2XLGcVRvaUulDMLdFsrPOzY33EpFYUKvauH4LqJwhWPqhOg5IMSFuqK", + "XXFjsTqQZ0iaMDYCymh5bvuJKvLSxuISjA/En/nTSlGpfBFq9ypLfYiXQIXpQ068pFRX7AaHmWdGFxk4", + "qLGWHj6faEP9lA4ASwJg8Z8+/P/33x6F0sHgOlt+JdU06SRqxJUCHFDvTkFNoLRsLqHgr02fnMgWBnq7", + "ySKCW6MBXXXLofCf+z1r2XX7ZAqgcU5gRrXp3JZo2NRHORpbyIJw5wkh7B+LoJ1xHlYEpJQuyNP3DMrw", + "wvkY5XrUi+IF8c807Mfw+LDbS72unQpTGLDFYrAcgvQYN4qjRssDn878w6NRLdTV4Iob9n7/3dE5FRL5", + "2zaUTUU4Fk0pjlCAeCXMiDs5X1H0sg/Lc0uYNim5q4ZcXQiTQxe7X91IVvvktTDNicXZ2xjX9pfNNbDI", + "INqOVzNH794cHR4ev397fnH0Hou6gR6gt3Qi3lLF53j5fVs68e44FP32qpd9iAlRcSwVE1ALB3E4hgb1", + "EQWLUQr7QT6BCo3KEuw1x5J0FPlzWgHoRe1CONKGZsGSIoLIXQt4x1ilWidqq503qG8sldMl9GB5A/Qc", + "5gyRbLCBqWUcF2jNcdtbaiZ+YSNE34W/6S6gzyP1/i9kcKB5qbrrxI1LVFWeVu/bdYh5UqtDof7fbuqf", + "+YP/EGhmrFY/wCgDnHrA2FiqvYAXbD+Dj2+Trhjts8oxdp5DGeDEoT2uAAgqICQNkiMtCCOe8e27z/gb", + "nsVXfo5LkN4GIruZnEwEpuc++7zf6xL81V9ev225QNzZGph4g94Zh0D29YBf80Wt/zekhm4riDHPc5so", + "yOmzrj8vCAkLEwGjEPIicTZ73om8VkjeOJ7JnJg+dZ7zOe+Rjjl3GlJpAQSF8nrCYVY7k3OhINcLSW2V", + "KH2JnHXskw0sdGHOlRFJhp6fsh//9NPHZJ3ZEG5k3AIrsHkL3VpEBUNl1h1Bhn7Rp+TUp7OTfjAXg8Xb", + "qydovAZtUwUfRYsdifg72NJhgRZN+v0iWDzEwe34He4sn9x+7RRGXB9Y5g4V0MtOv3OlF3zaSn/xly9X", + "QVWR8mkNMgIndasLF15zYAsxlhNoTKwMoC7UIkJHgQBQQf+ivVrbZuzRfNIystaNihCCrVFmyKcxwNkA", + "jiKRBTXydaorb/rstHHZ1V4kcH/XmJTKgnpqgmHU9dNIVChI6TMlHOQxS0VcMLkApGK4HJtaMhBMY1/a", + "LTeBBCXE2chU8Yfuc7UmFNMMcj2+tPfyFaQaEJQK/RL6ifyMSgWJemwda/i43CGDOAGESLPUjO+fe3yK", + "kcru8Wkfe0h7rOASbA0YibLvI1GtKLZQxzJ78OJf7fzzkEGcnoonBlTbgGU+Ip8MZoLnFspUEe5Iwr2D", + "QVjLkBKRfBKsdeIYkBtIBWDy9eRsq0N84ud7Aku6wZMYR2nA/LdlQVCl4x4/n+PLwzSgVZc7gbKD83qI", + "3PoJiDX3+4mcuKr7EYP1ARMZ+nKxlXX5semQpV7VpqyQ40sbyghAzHdZKovAlRzxc49PGayYVo7nA3st", + "RBF+sOd/cAFynXrruvG7uszT90G0oZx2DJ+g5xKuY2jXwQlBThpyoU7jfOKRIj8HvW0akrrYvS8+TNRx", + "JuaF9qK4i19A2+RSLGLmv+Iqx6xXLFHc2X7VHmC2ojoAGwsv1wf5LAv+VRt8xfgywhJ2tYklz1gd1HvK", + "aPEXlNU2q1tBAnj7KfvsQwYu3Op74WeeX1rsfH779tOPFwf7Bz8dXRwen6WNYtZmEHY4nZYT6in8ZEWW", + "qNGiaay+sLUrD6YAJ9B7yTowxiM6/4xfCeZ0ovw5ZT/9iBCQx4dgL824ygLiJvQoRhTBMR/PRMQExfas", + "ylSeeLOeIBI0NlWLgRM3DsATmFRFSeg30NBsxapr4B2u3iZr+v0Iq+KfB/4tsUw/hwV4Rt3vxYNmgrU2", + "Y5xcyPR9rmCakJRYLZuhqrmqQi7BK4n2wi57q9lM8IIhnx4Q5BJ9rhUOurYAepJclyLnDmjAxE2hLWaE", + "IZOcLxBbcaJLA+2XfIr050BOC2YJ6Gpp4JloiUCyhHiRQujFrGHvjdnhgIMNf5MWi4qCTdYo/8Typ8CM", + "Cz2BGBeq4WBDNdFY1zo6Ba4Jd7H8Fmo5BaIcSrC7IACrE5XisEP/iwtoxLqAtG7KnOH+VocNkH+NFX+5", + "puZRIIDP59D+qxXwqw2gGhKaafEdL+Adh5HaN7ZhEzwkHEBC/4Io3ZAdhpeFS9NSS/IS63FYfsRuwcYp", + "/zlsTqLoN3mj+Q00XUDkAPAB61eE56H7HYsHoUGZIOnAhIz5GmnZSMwkFKLnhJeEpVL1zG2oPJduRdg6", + "ZOOQ/3mjOdX6QGt6NUv8wrfaolmTCdBE8Dpfro+2uOL54q9iTRlMIGOkwlsoicWxqSAjTIl8y9ifURVm", + "ICo9Ys1De3q/ceGiLoGUNxTaTbkZ+bcCrj066Ykigp1QuUTZEx7sywbSDMajq4IaLK+8KYSygXgXgGRz", + "OvMNJRXUG8SUExUP3JAdhDp0h8E0FmC5URlFk92OuaqViFNfWQprbaUF6h0olQHMKbq627sQq50CnJE0", + "FGx7NRpwkLztnXQwoSqBzbW2QhEeLunElnsoYq2qtXdJ9UpIfGNVfcg2VDED7BpAT7XW7kPBMqy7pkpP", + "rLevQCbR9KjaLwKKLgkfKir/elCmiYSapGeAF0Wxa1kEALmmjtnHR5xV4tPZfOXGPm1kq/dKf4tMA9+g", + "rvlRwum8fSK/XM/A9q7WMgDtESuAcwyX1E483Ic1mrRwlKjMCvUINCsjGgTYzjXKAYCeD1EwbRDlP0ga", + "OaBYjhB5vOhKHnNjyL3EOknUXRPigq5K6LxGwKBPomahLh7HrYgFGqTntMIiG2LBV9WmnKjwghTfwUnz", + "qr0VAEr3qJg5HCRYjYiSEilPK/vJAsAkQCGD1VQ/yE03PMGGEMs4e3uADSkA6wZHO71tQGVGF5aMVX7N", + "F0P2k75mE24SlRprw9ewBBKiVME+HURwuV1/s5xIVd7U6hinulKQ/tMP53R3kO0ZLEkYPyIg+gnu+cfN", + "+fjDOQKvekNpzs2l9+Hqwg2BNscX1TOxZtDqucBibpFbBDWHzZkPE3XkFWbtj1g4fwkno7Wc2ot/PHkb", + "CjPQIM/Srh1ecEVFG/w51rOxLsEpULeoP3dw+RBtge09fcXb+l8caDXJJdIIPHkvcUM3o6ZsKsq6NNeU", + "5mdr6xI4bQcYrb9XeJ1gVjNGvw2R/u7hGwjy/f1v/wF5Nf9fI8Z6Pgc4bGKv8sdtzMHZraLvFWF8MD2w", + "15JxluLqpGzOC2Qvy6GfEGCgAcXyhQ3U861cc7VytKRz+CbpsC2WdI7Ulf+/RCVg7dIckw4rvNmqxI2D", + "bhqwkWosWbedHVyDA1y+TZohjYFajtsRAixeiaV9eZ6Qypn3aMWKKX1hncwZdtaQFOITL6jVy3rRG7If", + "vUBYBO2u89uR6qWwNmV49JUwxt/a3R8gohYDal56t+qy628VkIm3AqplDrWwgM8cmkcxiICSHS7VSgwD", + "tmitQ43MfjiD9MOt8DBsBeU1w4GdfvrYJoCnZYsAbiCWXR/jE5CLP/Vdc6f447Sypxb6R4iHn/PbBySI", + "5ucr9IASsDrs+KER63bcNmhj0WOfYbLG25kQNApPJS465LTBvOPYaOX4yButBlv8KULmD8yFKRUGxcYI", + "WknQnYFSAdiwAnYb8K2FBseJRA8T2xawYhI7cL1DG1gfQoIpDB2B5rSBYmlwHsR19O8jKcRIzLh3KwzC", + "tmXSAugE4qlV1xFMHcoCqQGE44JF2j0cko1KmbuB3yeTKKGupNEKgnyZmPAydzWGC7+0XFUodbTu1HOS", + "qIAk7FWOBmDIGbdMQzumVE6vCuWfx41/4DFcYv3n9jP4FcMkso/cXt7J7I/P/ktruciy6o8Vyfijb8/B", + "hiRDAA3xpg+4kpxiaui8OrxlgMPniw/+1q+Kz8VvK0HW9glQAK9K7yHlYuKW2pj2Vf1UwfmEs2qhv8jb", + "VRhuB4YWOKR/2GY/ipEpOXIEwhmfaSX8W9/weZFTDK5qOUf0k1c7O2mz95aKCAEPGPsc/HtRV3RANsDh", + "vZawQ3YOzQHeM+ZmTi253F4mqubsxhLusMRVlA7yJxE/VTR7kvMc+qdRK10KQTRDUQNCU3NhBJ7PppaB", + "linABvkrUFUdoK5MVMB5twjptOtVbK5dWGhgDqs6zkJ5CLXNUKsoAC74f0Mpb8YoDTKB0EHOnegTDix2", + "CfpjE2Ia4xyK3NOx372LskjJaNFWYF0MRAqE0uV0Rg1cOCYsT06VK8j2TNE6znJeOF0wbnMhIDmzvb27", + "vU0bFX7vfWn/Nw5IorcUGd7iQYvcLq5b6ikCVQwh0YkUps+guDvNRsOA4TcEZu5bBXnwn3UFeY9YbHcf", + "dfk8NtWSsm7v2UINBZlEpa+D82ef1Fl/dfcv3mv3oy5V9vzOOoF3UXvSkqr/Ms3e9A/WRFSr8v6qWBkL", + "/bt/2EYmQ126Xp85YeZSxc4C8sQThSW/tDIhYwMlwbVAmz//1i2ZrMMqJruzvVOPMu5BypDVKpjCCyEk", + "mDc09Qh0JqHyka7AAAK8c/xnUCOGKysR1OJYDbALvgZtPiIOX0pLzXVILwcfC4zNCZc5vtaRMecV6g90", + "YIOtCSgKg8zIK6HIcq2IlLvpWN5EalXE6Q785Ei30FtRfuSncI6LsEmUBRppny7UdsOKfM74nac2rDbb", + "dlprBvI2/iivQKoCVWyA2OJM6YEubgUyKg8fqrMD8nlw97/0NFcstuSl3YotkYScB7bVzV0EjYHa2jvw", + "oIai0mfqyqAFD9YQaY/PXX8gz1i57N5K/2Q33B/qB7irItZ/57lLonieE9fI3QG71gupXhAKsMEslUoC", + "W3GgEME6PDvjBsuVdOkGejIYcZVRm7ES1zALcMVzPp2KjKVeO1+gwxIfRaQs4E559T4SlNGqU5dIt0Ra", + "0pq6MYI74bdgU2mbOMBn5W5ePqoItiZtYGLZ7y0X05DtY3UFYJyVZH2RDtn6VWa/ocxDzv92Nwa3Y55V", + "UAP+Vy9sO0TnkJ0FLDxNJRVoDAH5EwDl+5uL6G5uCSxmiKLA3lVOjF/Pvl4jvi39xe+1V/fpx5LZ5zp/", + "hbcn2zIStT2bayeYNsHOYO27F2MEULW32gfeoPKpBnimxPEq5UMx/N+bb9nmHhqd14Ulg5vtAXoIG0wG", + "4SoEH3ET56AdMhoud1CnMK/BeKatUMyJeaENN4uKMIwj8EfI4sGJhkq/+u0METpsD+iuvOh7feKhjCyd", + "EvX6JNfXQ7avFnTg5ljmIRx0uPsR6+TCHBGXomrFmIeZChf0tbiNrQzlt0B2t7K/xMv4adiNDbaY1Mf5", + "yo5zmBau/u/6VJ+hfAWRieL+GQeauBbXeib7hfxX/5074qIpz3MM6eJx1MBVF0pvMQcBuDlLV9G+qtc3", + "dbVhpbLC9RpVu8hG+yI2aIUIK7BPV7oFeCI7bU3OPM9X9jNvCtYE1u0ul+tfxeK5Pa75osKhwTavHP8h", + "J7iXDSkKIrPaAavX7Xz3HbQhOHHjvvuOpZMyzy8uxSKtYcaOxTJn9i2oJzuDWj6iHuSEVg3cTYRhn3RC", + "U2ogdUkwULjQJTpmVhAKAFTWJJ1Ahjlk5xVrKuJW4c9R/pB7sDBiIm/S1W4bbvZGHTcc4plcNxw8Omrt", + "cjx+qB/3YCfL2jL4WCTS7aLbogPv9KwAbdgrGKrw8ldxKOq5VuRQ7atawwB9h6tFoi4FEJJd6UtK4UUO", + "/1i7Y/Q14d3QeUA44FBKiplLMgEuuEsjGEWZScec4RJQkAGl2VyJrO+PSKJqpMBE0gsst9x5S8nVQuxw", + "MGrx6VfbL9stDT+DKPCbsPjudiZxEt+KM3kWBOH+UtnGHHxnpWT6a9KB8uGL+NOks8tAOaRVn2eDype6", + "PW/pXCxhtAimWuRccWwGGxshVKPPk3WTDhX0AOUCJirAPS1yTaW3bTTA39XA/1QWqa2STm/I3iMZc8Xb", + "HWmZVxRFvglvvPnQ9dJQ6673+FWKHDeo4zu7//aXupj8HIER6xuB9eAQPIQa/Li1rFsAh3Xjei7drEWS", + "0Jdpemqtd/efhJET4GSl9FwVM+2zsiBcMM1SJa7rf8I2+kS1xkjTkNTzpyDYgugBBYIW6IOUNlEYbnEV", + "3zn0RYQOiKX3CCWWXoovgUmX5/JK9IYsFksCcFhl3zRLDtrIw1vveBh2w55Vc5CHNu9HP6h8aHzjkYIP", + "ddKZJY/lbvkFv3y11H5QoW61j0H/9Fy4wQEI0C6rUen/gAlTmWGudC/y7u8l6pzPxbl04odzZ+TY7bFT", + "7mY/bKVNwCmQz4Ivcs0zKhdfJfUYXgHcZ9C8/t5bOtuh9yXEIirJJj1bUdrQgSFWhbZ6PFijzcgmPPuZ", + "PH0ae7WOPQEqewYvj9TCMIdKBNqCR6h2SMd0gxj02ZIU9DrrTJXfnvpQrbg4jm4olAVgKez7KhYw0VDU", + "vfS69743cj3VpbtvO525EmYARCRhQKOvqXvXOlOOHX4TKM1CDA5q49PaGU3ZleSrT/Aee8dvBvtT8cN2", + "uuIY+CnfR0cGKYBy+y9UkA1VdxQ6eknP0ZzvXuf5/QBpQflw5xAkgjI8AZmOXubDWbjthok6hpCjv87b", + "NdSt5hWsxI4IzTpRwJk0KQ18oPiVnBJWfujab9dcK6y0dxttm30n1iKu1W6fx9jt8LyanSoyfPqdGx7C", + "unduOxpLSqtBaNBsmEwhNNb3Pq+wbgB2Yp/ANdKcW3dhhVDeX+yz2r9lQVZZ7bOSR4kASYMqdVtox0o1", + "4XOZS26IZRBxl1JpL0jW6bbzzmpQBzBNZPEhAw7qcOESWYkNszgPK7PJ2hMc467Y3HlkD/zi+FxDYPYb", + "JzUCp9etovtLTku8oi09Gxf02Vz1x9CyD3O/vVomerX5olr+rpVTxZD8iQAVMnElx2L9xTiVbmBEoe3q", + "a/FYWYF8plSaB51irIs+4A+FAGKzXp9xrGX3p2Mq3QU8NlEmMCsJBYWU0JIIKBHwjbTOkvqLHkVMsvEM", + "iN+AOhHLXjJxQz/x39PRTMbja6CNlsBlAUAB6KmAmy8NrxveIA30VlRMA/iF1U2kdKKutbkE6B70s3Kp", + "LpEh0GkWsTxrX7qS3nYOA1V/wORiNbCcsExY8v0TlTp96TUYNsoEaSWOV3mFHH+FtnssvRajmdaXgOyc", + "JooaY9CDnXNV8jyNSwEyFPHVuZeYgZ1pl6j4mNLkKfu+eqwVYyOqQFyMfWDwD7tHsKKCfsGkYm+l+6kc", + "BSQu1k156XSKHDXAoSJdoNiZt9Zy7mfZW+nORKE3xckdB3imaDONvibcfEoCSyFn9j2iqIQT8xA98/V0", + "af8cT8QhZexvodr5d/4+4EYF0QLplxCu4xaJAlXG4bt0qGoKLhyzW0puVo4GcNLuNlLmwvGMOw5yS+yu", + "TkPjlH8Akq7yuegzO9aFsP0a8e8wUachRYQhaMx1vz/609FZ1S4DddBAIIqMnXsx0QPPSlTMM0GfXICx", + "lLaRWfIHvwF+03jPVUbJW/jSR1yLDZoltXHuMk3gSw9LHD6OCEIGkTabxO90/6Nl3SgTy3nopmitTiNi", + "BTlconFrUZyalPtspLNFg8FAqLFZFMhQgNHn/aPzwduDd+BZQu+U4vkWam8siQukBiRRM5GosSxmwvhh", + "V1wRjTeMWZy6HCYqwKhI1cxje9Vvh+zcH4fAuADg5TXgZFzORHl3DjCPJsKYQP6ZcwcNnICyssdOz17i", + "LgTWBi+E3lmA85aowPoBOV21WJ3IrMngRrOZtXGe75KJb7ryhKFk/8+4Tc4RlQyyp9VRZl06TiIbcG/5", + "WrfuNK+6Q+5Mr56GfCgAFiFFH9JK0egA/97I1lcWY8g7gV2PvYMCGBTGIlBEgeLIqh4c6iqPZvcwWJXs", + "x39FbqkP79nh0cnRxyN2fvSRvf90cgLtnKE0C+x0W9GowghGXOlAJlg6Sg8bMUDrZCvYgYmaADwRTBWo", + "D2nBITVb9QkhQBAP00fItGB2r67JXT7Emy/N/czap8cR2Fije+v+WX/dbKJG8U7skEqO8d6hdLueYIaV", + "yv3iBQe/7CfqUojIrg/oBRLrGf1M8V4C1pO68XMLUi9RtDJd6IcrrTC9WICTy0uBZnREUYCxcEnx/kB6", + "KiMmRgCsUoQ4+fPgw37pZoNz/Fq8IjEMP2RvahzuMgMBji3ffbIUxQ3ex9EJBT48jG7Gk4tE+kSBjsjE", + "QKM/GEkkCY/Z4Yrug7ohvQ7D2JllyK5vxFQof2razxBWBG/+Irw1zjMlWO51ERrtQKS6t62ehvARIO54", + "pkXW+z027O5svn/wIBJqAx9G0GtOx6ON5UG4Y4+lTc9ghyl2CxrK66bP0q1rL/8tguy/H5+GP7VQbfUC", + "SqWs9hqCnoB5VyojMFOupCV28/BLhGslBXm7pgXUAw/EswY8GFFEFmWeZRDhg6ru1uBOZvwVHcBYKbSV", + "qDA/ytoWcnyJzAE1j9yfmNKKSZmjggJ1vEWRv0wLq164yN4X3xER3JFI7nz/3cmgMJqYj7SZhpJFXhSC", + "G1YC+NiW/8PWrxCv/w0H6EWwWL9I1bnFQ3uLHH9vKTlEg2SZEdbSN1E9jxZMZqvcZ1Ag+2HzHxUopi5S", + "98KKQV1Hk7mNFdPvAKp47ZL3KzSFUvMmjAyv3gd/ch9EmX0E3Pfnin7Oui8x1/I92x4O38Nm9p7ODiMl", + "uFl1FqNSxE6zpLueTaE+chCkctmr3b2SVo6Qjjtq0qc3T9er5ZjLuAcIY6Er7Uw6ts+0yQAkabRgcw1M", + "rmNw4xJVlN5cRD4L1kJn0VS0TrNCF2VOt5A3OAtNFBeguH72+jKlxYXgf8Dca8T4KObHJxOZS/QJB4ni", + "06kRU7BhAJ2rsoVJN9a4AWDg2nsmygrhLwy4kfoIxz3ScB8A7BfeQn9Fx28u5iNv/PrpJqoxXysa5Asz", + "6SxLQ0dVXVOnSIFH7YfhXtGGpS1qPcUyD+Wn0WcIcwsOqq6W6wIoe4J1HPlzBmDgZwQqfOs+mktQ9Pho", + "uoPcopBjnsOYLVfRhu8Y9qnwgvJ6e5vEEev4KErcfY1ovonSE/Zye7s3ZCfcTP0S1qSB2RkoBCOgAYGA", + "3rByxQHN50TmThgENwYJZJzNgRE6pLEC/9C6O+8MTtYdfTMfCmTFg1LagVRWAMjIFTAr4hlmOB3oIS/z", + "/AJ8vxUtMP++tmCpv3L0IGLYAec0en7E6Rp8U5RpL8X9SrBRshAmj+dWs5HfW7e6S4d+93kTPQtpvutb", + "SsAKt8fkVIF2hSq5axnYZ9aMD/NubRaiRLw20030DDUtmKh9P8N8gdTfA2wXQ3L5JYZLtGTlg/CY/mGn", + "PJudspwklOKrtVO2KFxqtxBseY2VAkCWmICB4sYAz0yQ+Jh9z0Qu4YL/dHaCNwdgaHrjgACgCRYRYUCR", + "SWJi9HyXceSlmHPFp142SqVE3m/2PAz89X5w/OeL009vTo4PLj6dnbCuHIrhEuHSnGdxmqNFoqSaGI4F", + "kqURBBF0JYyFfO3Nos+kmhqsbnbcyTE7Pu2B3aG0QvjQ/aWZ+WE+nH48/vB+/2QXdebSxFBx9sPaWCzf", + "iNSZXC3oUctONIIPYGiO8SstM2w5UhgUTzpK0y+TDuafC6NHuZhXDSi0N9CsBMydYB3CMqyoG/wZZ/kB", + "xWCDwbDmQGshr29JFQnpYxSSxkGa0uxtLjq/y6PXziqufmtApjpRJvDtrKPkGUCuPcRaGuU0P2DZShXZ", + "eGGXp8YdkUNSgVsQeW4Fg7MRGeNJaP2HICQkUH1oCk5UU3R7Q1YRNA7ZWaksCwC2YyhM0tglA6cZwkLV", + "44kyfq9ZTVAxuEvHsDCovRMkshSRnNgnkMU45mrag/gVpkvn1dYzVwWciUGof4J24luCE4z3Wrp7SUo+", + "nZ3cKdJGl8Vq13UfKRG964byC9/fg3DctMy5QecKWLxD2h+/AxfJIlEjkWswfFl3uVL6hWVJBxCk/J/h", + "VwDkD3yMY6AsQ1e2t7KqBGe/ycC+H+GuShL40qOVuIKF4f3rwTS8XrQg8IN6wUdrAYL/2mZLD/wIz1V0", + "AG+3chvG/wOQsHATGK+JySrIiigytw79Z+FgYUDGzmRBxT6YaKwqTgMUXCgaAEMmaoM1efYoq79T8KvP", + "2KL+SkjJFau0/USH6htZ87eA5fJZC742rvSn6knHhysgqR+ASdaaMt+g6q6N8Fxp8lVSFlhApl+3tD2H", + "rseleQxdv0VKfC1UEWzRO/ripkUBx7nLvMJvPS3Ez8MUEWIC4SJiCcBXr5Razcn9LKvt0wZbJapBHtq0", + "T8LCs0xkrCujk9v7XeOY7WdZwNmE+OOj6YqtX/1Dj9e3yZ1BremypNxzp7BQ9Zn2asnj9jMJ60g0pl/1", + "wb2V4gH06eND4ICCt1kxDm7qF4eWf9Gj9bfIv/gvtMe3lzJJNkDN3M4hUQ7WzxLpAzr9TmDq9XNHiti2", + "/FK/faxb2ap7/i6Xc9nMtVEXXGf35fZ2vzPnN3Lu5/wa/iUV/utl/3YWaZNoef+iR3ddpf+iR19Nx0uz", + "A9OG1k62xfyqUcK2ftCqWvWm2ortV+sk8jR8aYMbQGPctQmnsVP0QRuxffePjqlPJ2TLWpHiTcXRVVSL", + "1NLbtj7odBrb4jYXdqIxninwFN7w7l7Kh95fm02Rfqx3UYWihRm3LAXeqwva8ouAcYz4/InqjrlSOrwk", + "kWQF+egNGQWLuRFM3Ih5AfULlc+02Zfaj7XvBMYnLUtn2roLf/OlkYwb2gTsQ6qXH3juzmJQH9sO7tlN", + "Gj7d+nXG7ey3LYA9Glini/thRvtfPQ5q9E/cZAM+isnicaODtpCFyKUSoZ4qtCYkqoupLezjyXrhzYfs", + "1c5OldcMuygDCxu0wfv/S1REAsChriSHnxycHENMcsavBFO6gaITp+N0ovxqsW5opjg4OX4B5WhszNVY", + "5FsHzuSDAyq+utZEg2v7bKTdjI2EdQMxmWjjdhPF2MshO0UDZSuwGzUABr6/BR5gqSddWv97xrAYzB8X", + "NKxrTSFI/kQJk/gOeP7ijx3kSwGCYC6G/uMdTDQTnI1Ug8hABwsWYE+62MHYA8ADePdcZH16LhJelmqU", + "6zHhkACuN5AmwXO2RmIqFYIZTHJZEGkg/DpsH93lgZmXB+aqHP5CCXc8koOxnlMN4nhWqku7ZRfzkc6p", + "h/nDR2a0nyA+rDtHiigMLuMe4juwSM7XI/bRGmynXajxFlE+JUpfCXNtJMEutaL5/+jP17nTxbH/ySbZ", + "nuJI62yGH+NxDzw6T9lr9pwtlbU358rLdJ00bFnLfLE6DYgkdxbxBm0XEvShl2fIXm2/Wq3GEtX1N6zS", + "FUgJM/q6hwe2CYaBCCGAOpWRDqiBgkJilFsnIi4IlQycB0btv//tP1jIra+oBSF7pY6BsbnOKKy1uy3T", + "jZX41hongSgd8lvxLRoQDvcWyv6mLu926pPza4kUezUpfWFBP/oXmOmMZdII6GuM11GQ5oJPxa5X3YNY", + "yIJ49SSCRWlnsRoqFIQEpr2hl1C4+hqlDC946fQLxCAPeMoOkGt1VQDhJ4FNv4x1Ec11qRZrK5SXUaEK", + "Ypqf7n8kHC+GoNK7fqsu/KN6Q3Y8IecHDwf0C9t+vdJswvM83mL+IYXOc6SfyJjlC+vPp1QsVdqJdAgL", + "Q19JI5wBZEazyMVt2ETgFgD0OoIpXHkVwVgXfn0RPvIKQavMpn2mqcq4h8u4tIbBVH+BUyBcHgQN0VVl", + "D2zzXqQ+1Ipl/s7MVm1NfGw/vrieTODyRjmCpbjmJCqVTNSMJZZVhSwWauJAbyKXOK4ysH3r0uYR//wO", + "bbq65bSp2M4XarzpztMwznOReNyex6oKp5Bkg7MeuNJhzN9ZFfKxgiZPeNELVGqPqfgjzXzghL/Vfk1Y", + "yl4zWBTBL7NGtrB2enMOXlvLPYINW3IEGiBE3fQ21kPaa1sCQKEGRQ4G0xaoPOyM1zHiFK+KfqyZBU+I", + "FCwgNWBbPXXvR5fxlFvLIojZLlNlnqfAvKHn0vUQOd3x8Qz1DM7ej47mW+jEYtxSrdySFxrBXyLoRaa9", + "z6G0wwjC3noFxbq3Lb3Q0xNxzCwSuHM7kHaXcdSLIU5Ri8rpCHgGLfyJonpV7NmfcpPl3sfTE9oyf+nV", + "O6R0ntkK56DCwShrLWXg4i1v1WgB+hpxcsK0Ta0uURuYjHCCdY0YWO+AIzNNZU3Aflc/wcZaV59M777q", + "/EmQBOJAzwclsMpgjlUSj2Q4f8uM3vepn8W65DYFzR3jLJMTqFVzaxv076exkTbptqJuDxN7zc38D1ek", + "5x5Nl39Q+YKdfDjYP6lgNJuqqRDC9MCpBM5sbq2cKpFhJ2eM3cUfc0MMLqByRgtAlZwqIicAtp9XOzvt", + "Zd/4bFqDD8QztRmCNRwKxnimY7wmWRCOcXALf98Ea7gV2LMC1D9QD9NIHqzKrN/v6JE389Rh8KMKWBYw", + "fZrR31/0KCCQrsKnvSvw7c8pmisxANwWCw/OXC0a3huyQ5GVhcAu68JCXW4BnlWiqj4NRfDnETqpiq79", + "okdgsLzXZg79IFWk379aJsYyExCmMmIulOM5u7LQWtvsI0lUt/4dxBMKuLoiu7AzThC0Y20ygcBMzggx", + "PJSTSaIAbFdkdg+fHSigB/D7Piu4cZLnA++5l9DFPNZXwiz6idKGiUAiP/DubE6NK71gPvonEve008gj", + "4TezzHNveeKqLkON4KeZkRMCybQFdDqifRV6YQiT37Kavdx4Y28gzYxW6PRSnF9B3zQs8veUTOHIl26c", + "DfUo/sGK2RyD6C1aFn5ZJWPX3kc/V6QaYcKNaYZ8BRrLCMmSKAKIBWrNauasynYwUyrE4cXlZDHw7q1E", + "2JXQDzWRufDWfyEsK4zUZikHsGXExG5BN7m48IdX2B514Ouw27g0cStwd1b3LvsZtRdzTHhuRSzaGGnt", + "17q1aGPnEe8qWBrSJtm6HAB9NQL0EtMg+QuUg/ICm8Hh7/3PyA+8o1Yz72WEiOPqtORn3y7Y1nCf0pJz", + "/Oama2aPs/u1I8nMLt8rFIh0+lsqowVavaobalW8Jb7bQ8v0Nmqjr6Beyhfs6M8fj87eN+x0YupcttXn", + "fAH4EPjC/rz7/4V+Gx5xvraaBlYggl1hm4Po0ot/1JvsQYCRaIiHlvqe4wr8jynyhfdtlf8H1Py267ut", + "X6eoa9bW/X5StiY4Pxo9v383F/3266j7RWrJsJx//9t/4jJib+rXqk/6X1JgTNv6xZW/y+JCUcGBVBN9", + "LwAsDLfmiwHAcwB/Zogsfjo7idipP73bPyAMxUQt51NXVhKBJo0KFPVnoiojPAUVil7DWBbcAW/yLRCC", + "6GYN/OLVfK1QYYS2RS4nYrwY5wLRZHV4UIz0zrjKckiAkvbdfgUo5teaZeBxjZH90/aBPhF8sFJaWBVA", + "ngLaC2nELuvyHtEoczcDQzhlAenQCKvzK0QeUYsqAs+hvhMBObqjXsMawDI4AFiOSTV2AEWBCKmcKMBU", + "djBVPh/JaemXCyCKwKBmKaB/LQlESmCQwEum1USaOY4l1Bip+7zDISD+VivPqYsRLBO3iUo6jUusX1ti", + "SCyENEzSWV/lQLUQx15ENw814IdZZ53R19hYa5NJxd3DQYA2m007kpFRNUgP+HYxF9JnH85WCFeiGuGM", + "+kkEhp7mjhJkZ2+YqMO60I0WbDwTCAS6Tuqo3vRx/Iqfa1rp+wDn6jURJtdDsNgKBzfek1V2tCpj/9C1", + "vdxQmzvwd9iQHRpdNH0DAIOVzjLyuvvMu9198M4Zet39RAGdUgip2CE7FAi4I68EE0qX0xlCBXlDRJgA", + "ixe5oiCjF1lkQZFUME3SrW4Rr1eW37NJHKRtpLPFV20RPriWODaZx7yGyqCmX4YiWZExKEm6O8K6uvl8", + "5fpvP2Fp/VPWhz1wV94Kx2pEQUhTBcf8PkqibdzqK2GlfvIPXFPmVT/vsTCQgHsQdnRmpLpEVHkvKBDN", + "Q/2bqK64gfLDi4I7/562z+b85gKCcFb+VfT26JDXzvFIMI74Z4myMkcWikwMArFSMNLuyvVuNL/7JQ0k", + "/8gJPU6w74Gn6tQLelVfHmT6C1NHcGFuTQJ218OO4Arkr8rpIemhXnC4VWNtZb1bYSodWjq6RIhrlagY", + "GApez4iPLxtez1J+1/us/kDCg7NQhSgNBtEtVknqJTKlOR/PpBJ9+CX9kepHoJa+lt7d/udEUWgqzYTj", + "Mk+ZE1gdWD0T4/NUIwtvTH05QhqmrxXNZmGdmDOndW6H7MNcOpZCviPdSoXK0sZTrmc6x2ftVRiliYLu", + "DP/SLwcjboHFepyX1r8lJM1UCZ3MQ/ahdAX6O2NeFFj7gu/oNdZfxRZ8HZo9LeumzpQK+Gh3GSaJIAE0", + "k663Ir2dhUCIl6rNaLEfAcKMZ8+kwPzwB/SgtiYFv81hIFAt9zjzb2pv82T5iqfoSPvYmtT1soXnMJzT", + "xmnrAb+Ks4mKekCiq7UQjvErLnOvUYfV0RM3ABAd6vUQfRNgAzXLNNCGC56FVrzn6nnjWVB8XTzG2uDp", + "NJCSpezm7dDmCxsV4mdr91qz0qaVPGepKZFPNUSw5rzwKpwg+fLFgIqMSOjIrUpUN8U/UHYz7YWkKkJq", + "g7Hmp1iCoZ+J3PF6AnuXmjOdhixqowFLBBaNYOENmddGgPpO5eRtqgw6nt6IgBL6+FqsGqCmxzapt+oD", + "3s1VrQuhfu89vHg2KC+PQLi1uxvJd+najW4MhXPg546A/Fu6fivTZqLNHHgHvl4lva9q7g/tv7Qxq06d", + "lLEOom6eP5kqvWdEC4RcmE8q3hJLWvhDIajhpPnCNcUa/nQfxYpdl5vSrDBZimj78yWUk2CNATrLkAXN", + "m/6adGIPa9JBW+03ILzmiQpbes0tu5TQ5spSKPOAbyjvpvu/4T5jEc/ByTGcA0utulIhb+gASm/Kwl/Y", + "gpsceskd8BlOsURdwmWODsk14JADV0OiTKkYttN6DxzoCLSJLjQyFPoD83Iw06VhHz+erNTLB7jqm1aW", + "OMw6bYnfyLFzylDTyTcTo8HZo3SFbuklNdBITH7ZEQFDb1Mn5FyozFseI3CM9QTNq4Ivcs0zyxB4F0kn", + "AvuWimbKMFHvELWGvd4mg7QAzZ/nkL/67rtzZwSf+wcoMdUOiT+++26XWaEyliK38C6rC9rNQGVe2FKI", + "AhkxFvKK6FC9sTfIBHhXImMWHu5nnR5TwRrAsx9dCeVShjQK3joCRvIrADUWaDL2MVzNWToT3LiR4C6l", + "arKX28z2huxnaibBPBYSKUKxFLigrTOHWffaSHASlYspHy+YlWqai8G/nH94T5P2/o4NZyStKFT4JPQs", + "wt4kKqAW2ZXHGh51V7le2r7WNrZ0IvWAX1mRxfegRWxd57CmUDEKVXC7LL21LrVaOlzMKn+Ba9kK33RL", + "A/U7bfNfyQ+yIbuTNu1Z/OfbUgNqqXVZ/ErecC83MA3cVvi/qMTeH4Iw0llqOSpebwFHame382vSgT8m", + "nd2kg5Fcx43zl2Y/6aBagL+ZwUv4CHLf/oM5l2o41fAh/BCLOTu7L/tJByQcgsJJZ3dn+7dE3R4ISjpp", + "oNanYs2nf+JO6wMw63TPJ/STDnz/Yu7//fpV+5wyrcQXTSgqHfiis/DhzvbOHwfbrwY7//Tx5T/t7rze", + "3d7+v5LO8k9xreLIoHUvOJwgsF12tuPQF9QIm3R2//Dqn+KXI/bDBRDP+L9u+/fD2+3+MthQA2uSvpy0", + "HkNBQ8ljXaqYhWAFr+lyFMhEwStb7+pTcTv5shrIe6WCYvX1N0gvug1fa148FPZ4f2gCMGYfzhieo9pn", + "W9F/mksLnQDP5DxsuuUWnA8W/E3wKN+efmJWZmLMDRuVdkHcV/5/+yw9E84sBvv+rkzjLU0EbxRftuV0", + "KqyXmWsuHetSOzwFYPEnoB1rz2q+zC0Avt+WaurK0Vy6ZSvKsu6c37DX219u+ClpZ49n+bVaDDDERm9K", + "P8LzXpU4g7tjNhFD6NvVGaW6VPpafT0a44HhhgPYkqUM84MiDoQVuqrM8GckK6+HccC1240FeHOZDbwv", + "XtD1R9ggaTHjVqR9luItm0kLjSUi24oX7hZcuP47zQs67ScqFdBkldXgOrh3kYKvhWoPkMmWp5aoBsQI", + "Ro4rBtCIIVaqUDKF7wJQHMDzmC5ZBjRRnMHSXAHypBbTSxSBEs2kBWZlLBnchagKrjYYLjLLRdL5LV3p", + "vpwHFNfN6oNgttyBrol7S54wOH7+BZ6sH2aph+FKVPKmJw2eHlMquChzboEqDaFt/cftJ+RhxSJrzpcV", + "3Ixnm4pUHGHHFwHbeTFT8JLQzMGLwugbOedOMCW4EdYNlJDT2UiXhuHEIrvcEmrRlRgDJJbOczH2gw0Z", + "wp9APDpRfjoDBJLFdG86l+rCjrWB0+7f3abeTJVO5FDJWBgxkTeDD2eDSB2aKFDCvT5LqSzG/2aU8/El", + "/sbyedXk2aOzn3M1LfnUf/fv//d/AlqdYnNhpmAAO+19tAFEbGJfS8YM936Sn+hIWIfPZDBdiMnUZl+B", + "3QEY4SDS+P79b/8REvdkpbN0e7iTsi42dhqRiyuuxoJNcg1hbU6IgpE4PRbvGF0w7leB+yuLu9LwfBBe", + "DLZSCsIyvJ5pK3DWqHNw2t7W/7ft4c7rPtse/uH1X3o4WXHj1YD0U0thxlRbAFEch6hCI30l2E/vz3/G", + "iS79EBjT/NHyv4aqQ3wdQHdMt4evvsfuRb+FY3rBsc7EACscSa6gFiqXIwOBZf/9A52JM64uQWQH/+v/", + "14N1B6m9cHIuLuYW+1X9Ucf66JfQDTvnOStyPm7tyjynzTrHY7ah1prGIM9kti1PYo2ebsg/FIriTymY", + "bL/+9sWv1hs7ilW6NYeMlCVkm71p6T17uH3qLlqiujVfipFXZoW70+datsvBEvLnA1y3GAmgUA54en7A", + "Njz21d5aEJEuvkyPznHttqQP1lqT+J2tTHg3DeisN+Wp4TE4rA20mbNfjfBM574+gTVMLgEJob70v8Nj", + "3iwH1gOnB9UbQ9kZ3kIQRP8i2X3kzFKb1IaUxKbq2J71nqpP4B7ySqk1N/v9i6tfGeh7q5HnP0TLBkLA", + "zSVC65W2uUZxsERBneJJS72bAQAXHJwOmQnl5ERCneqlUMNEpSRXKcLv+v+FCql8wcS8cOi0pEJlF1C3", + "9sMPCMwB/yIbn/hKYcWULArhLINZYDEASXcAxQCZAmg0nnlvIFFo+OxRtNwyO4PfTXSe62tWFhgWjXYS", + "LjBCgGOtDhbVRgDXdlMUhT5uyqYgmGiAZzrftfHXAWrEVfj9n2qo4A7vS6liOBtfdqypU2yzV9A5DbIh", + "hwme/rzuUmMK97iIwrL/3uX1vO6me4vJm0qsi+GZrXgz9T5XeMMAv97V7XZO39x8O1AYqS27Ef70zdRV", + "hQSHvhLmSopr1nW68BcS9JSOkYqBekwhUG17m+iLWyMCzoiN9fycQN8zZDzmc5FJ7gQTyhkpqPUHb2Zt", + "FoADcLv7p1b2D+0/7K7uH8QqsINcXoo+dB3m4krk/UQpwMcqjYWwKLbWZNIgSi4Ai4FZ04swdYjcpXXo", + "KGpgzIXWH9a1QlQ9PQHwrDdkR8qZBcNC5Nhik6h1fTR7CGkwnEqX1pfGhmjmKt73AFXjt3Izd4N/9BMV", + "px+GFz9BBIg2RRC/A6Rh8KV/tNe0tNdg8wxb3zuTqLuaZ1hb78wjMa7VBL0bz2vP64NHbn+p0eXdBcyC", + "1ZMVlzsE1GfcNoreGXeOj2dQ23GtRAboz7lUlwH9sM4SwRD+2//cae/IXLOkU2EuJB02nsmCWHyAbxP6", + "XHKJadBfynkR0qHVtHDX4PmQDzkCv8yfiDYYTfXCAdILatL663lxAIdNYJWTdFAEHhvmA//FleQV1ESE", + "KPeLxUYCMkP4+hBFDW33daZ8wIsZCaEIjfxOjVat0BMYHnGwE2ndauJleJVvyASBY1YJPwqzXzcOd26t", + "i2Ozhscd1Qrh7FGyLpDH7LIrAXd2vwK4qcFuMEAy7Xt596KL5yfP+ZwP6EEhmg+onQFiqZvC7y5yzTOR", + "pb0+dcUyPUlUCwEkJjHjd2pdbAFnMpZ0/KJHq1iGNl8VgCOsLRBCIjIqBvhCMV5KQsA6b8WVJq6zbqNN", + "ZcmsHYmmskaWmrulA2p6wbIqhGFYIFQYfSWhpH2c6zKb5NyIPlNTA6Q2H2ciUdTTEL855gaMUOj9xvlS", + "DzV2bnnbzziRAe5SWZX0sC2WdMZ6jlivWrVDLfkT95FeaIObjUMccMdzPV1R+hFel77z8N1GDErkFArL", + "aUOTpwybX9vssLNt2701kor7jViz7wUQK9L5ZgCOE8Z9YRmfehcKHrOo738WBACYHQojLNTbksJD1dCH", + "+8kbB1tRx6DFIyLCEymRMVeJ8uYPz/OtErAp/C0ZO/s+HbPunCs+FRloJIShFC7QSRzq8aXXTnLOp4DY", + "SxVQjtFDCenZf0LPQd6ljNvZSHOTgc1gE0VoS/Qz+C9gDmllWReNPqgkAQCI9cL5Jiz+xmUURlqsulZP", + "hRnEk0lbSWL0CAK77yVk0Hgs6I4w4ueK6tav4Ze/bdEugPvc6gAf6muF7UdwL3EnLLQie1XSEN2IPR6k", + "KFrFw0QBjF+lg8C2o9/h1+cCnUyhILedqO7B8Z8vPn56//7o5OLN8fuLd/vv998eHULXa6+GFlGD4vvn", + "9mI2eMH6LnbuA+NVW9zVUF6hMaY6tf4B/tSu7IF5Rik9Dgd1Ly5aYG15DJl9Ii/xTZvYZBHxLZqbm51F", + "OBbhADGirl9C81r+Fl+t+evKt/eQI43KffWJPhOD7J6Huu/NiJyPAxAMzTFRY10soDLPeXfM/ymQ9E2c", + "MNfcYKGIKVUUMbqgENIzUUvKYM1pX40v9Y9DHWGm/nGkH+9Ik3XUeqJxudecY7oE6Ux92alGN/F+4Lv+", + "VEE/DR1A/C3r4kW65cfdmmnr/AkIvsRYK4WlY4E6jCmIgcR+V2/8ERmbFS4lb6IyYrUSgByBQCcrnEWy", + "7vFlNu9F4Dhtfe1Um03L+qVH4XFASN8K1zSLB0FEGhsYaBzbJGYFM9QpikLEJ/Ni4C3ymLjwJhiQ4Fkm", + "3S6aWuAGIh0J8urBKH06ffhXXfgf9BFsMEQsolAhoyEzgh4DuJ+YnIC/AOLXpRBFk5hGK7GHfedcUTkG", + "laYAw6Dg5i69XxOsDWQLakPgoE+dSMYZUMClDecE+Qybih/rTWhTo97ffjqCzschIX2co0a6uu1cIaxq", + "UeQLJt191TKJeN2yWkaMgy/gznWeUTZoIm1mwbdgD7zXAcIDl/7JTIBg0LZe/XVrtzE9ZssRVUvcV5Ru", + "RZJXXJ6bj7feqWaW4o4hNKqrzmDskpjpPBOm9ygBj+bq4ohW8cLO9L1Pq7e/VjtBx9Yi09vbo4/BZsNf", + "vggEsoQTnm7NBM/dLN0jDQuXTaIE9H9h4RWlqXCFRDZF/gCjSydCI+DMEJx2GCfBeEmM5SEqJyTUsRdl", + "4IwsEL7zlxLyjLm8EkpYyiC23Y8fhX0y9ePHWk1N7f9K1xHr6ssfAKIJfb1SxTRG71tTRA05PfKbpAfe", + "iCGTWl5Jt2Bg+t/e8bskN3Anb02lm5UjQsK/LwXpCwwEA+gg6778I5uJG2+yGdvbON/RKR6YUDuColy6", + "GbSELApubcgop38e/FSOBudyCt1nYrDz+o8VVgBAT4+QK2Rw/tP+zus/hgZLOncAAc8uxSISHceilhcN", + "cr5Ae48w/+mQvaPWa5ExG0a3iYqFMC/3vCUaWrZTJOSo8XwM2QfFOEMzJy1KO0uRxwQ22EARDxsZrpCi", + "OpxqUZFKLtNJJqqbLZM6jkpjXeAtkcIiyTQxE6SFVNO09tdQxrOzvY2VxEpDDouJyQQy3FZjPhE4DRjR", + "d6AFNMn1NSZV28FvAenpLUgiESfcBWbU2LWrgJeks0Xfy+JAqLHOREYlzzO+8/qPP1B35nAVGFGLtHTu", + "YNBZ8RwqtkJ4lDuE/EsdCp5lEkvMT41fTgd5ITxVNAzCYD21L0EbuE/ANK0pDHDKDFN6oItInOM17WNy", + "I95jIoeBsyfA6LBupEessSNKL8FyOmtQTG32NkAGoyCLTSyHp4Ca+KRithgKYkB/P9RbEuPSSLfo7P7b", + "X5rGbkB6I91zi1Kpi1ZSH5X1ukx5S2XTAyqZQpkl4Iz3vU/jrwVki2GYyR9cy0wkgY7sSlo5krm/mAk5", + "PYCiWiFsva6EQBUCSz7AUq7IPz5NWU+jnmctBVFcnlw+oC7wcfxurIrL89rS1gSi9iGEslo96QPYiTjS", + "hoI8S6N8VsfAy8ff5PUbS8L5UIN5/Y8OtJrk8mFwuI8hQrgziDNZidEqKWpVLFu/ymwtgdKZmOsrYZcq", + "EoHGO/7zItYK1ooAvdGIbTeYF5M1JUK1N/7JGVQbfnjPDo9Ojj4esYP984P9w6M9qpBUmTD5wj+hKtFq", + "EoNSzZZWg0zaS6Sos4nyI0A5CJBBdPH1mAO4hoCmsFzqSBWkiQIHLBPWi3ZvNUFT8+Tdk6LpKSsKH0PI", + "ItfSnQK2mklpzUJtP7GG+NaW/61wFRrhPbZgPe95PIDHh6z76eT4EBooQk4hJrZGC1Kk8QervGOZfbZv", + "HIib2nIWm77LlkZ5pu63tZIaGJGun15iv6nLj/IW1Z0SKok///6LF8BauvM4o9Pw7acQERrs/qZt8Hse", + "ZOI+k7IDmzgaE7H4mmG7ATbvPqYOfDyl1h4+V1YYZxln3cpWklk/vOKFH7bnjSmoC0xUetukSps9JhD7", + "C949pInB+hl5jy9RKWYBfnhBLR0v0iE7LFEGRb2drflQ6azIJ1CqUCqnSwj/eS+w5vWBPQUOfTTxavhj", + "tt0DVJeRXH3Tmr022HN7KDSNqouk7cCegEj/Q7ev0APqEtvCqA4iSmpMBz3A3Vlm0V3n/lScsXcd4bYj", + "1U8UtOUBbbZWzHso/QZJKUke+jDLbVT+SEMJ7/JB5HlendTWChDo0KoR1H+eq/KJGry+tfsDndUaLywV", + "7jd8z25kcY5OYdy33r0vlYdfG/17pKgeLSe15jRUGJitEcZlEEsiM59wZRmQTVxr5tcmzzHHPyB8RIR2", + "oXXdZZlQVrDuWFvpzwJ0bCExGEKa2R4cAVtw4793/r9OpBPsx4/nr9mbdzuvEwU/IVzXibO9IaN+Atho", + "SC9d64AkmUOJlz8qk9KKLFHezz8TY+nVFc/ZGVeX7McS+U4uf/jjNmaQ9sdGW1ujlFTsv/9rMMoFYB6O", + "ucpkBpQYgPHYTf/7v9j/+d9sNN95faG0mSfqe9Z9Ofjv/+r5j+GN4fMUszn//V8/bA9f9xkQN0KEPLds", + "LtVgzm8S5b/Ic3+AoG0B1roXKD+MyDlmWGdG2JnOocO8mtDf/5//F0Eo/8//ZtvDV2kPQCxrbwLNgBDq", + "ZUonKmLpEIN/Lm4k9BFfCZPzInJW4jSG7LQ0YgAvlKgJVwO/8dFb9N97HzBMw3YyI6bcZDmivyaKj6zO", + "Sye8DnQcSPGtrus1o0snlcgXgY43S5Q0BNvpGAZ8uGNKSysG0D3MSJqsnMucG+kWWH2AAjOF8lR5E1oh", + "RwtCIgKYTcdywS0SFlPy1F0DhS/ui9PA7MvmgiupppMyZxPDwdgJ3/cLDmIDpGSI/glNuUigotiolDmO", + "C5UKRo+kAoglkwt+JdV0N1FeYAcvUVFhEN+W5kpe1W89YrPjagHyPdjpM+HGw36iiNCzqJ0Eq+GdMj2X", + "KiycF90Xjjl+KXCQRNlcuyHbz6/5gtrjvMGnNBRiTGHCzAj/Bhn7RY+Asj4TI12qdqjPqJsj1mebwgRx", + "qvTYv69VYnOpToSaulln92V/ZRJz6ZFOF9F2bmQwCRW2s/tyu9+ZIwdQZ/e1/4dU+I9qlAqJcc0wuOXt", + "g+zUB9nZvscoS5SigOqqFTP8+raYD9kBittI5PoaLzgA/vWnHhheSWKmU38MESGYiG68fsC2tcV8LpyR", + "Y0IDbwgR4s8EJF2rMesfIYXjuU0UIhsHEF1yMUCPDkD04LziCQxxLPhD+CVihEHHuBF+cJERz+N2PVA7", + "0SZRNXgyGiJO+FqIgg66Ev4G0Go6cFzmwMfkDaauGE6HLOnUknCxxpGMF/gk6TCO9wBP1FzeiGyQ6TkH", + "brMYDauYgZYEI0IVt8vF9vBVvzPxqt51djuTXHPXqUnKy5qcbEc5wX7kNjHZB2gCb/D4zZtxK9jICH6Z", + "6WtQUwgGVwFQ2wL6BpxNlJNzgagmS0f3o5xLNfUX7KHkU6Wtk2MssNo/PU4UaeddJh3a5xbEg9lYKQFW", + "Fo435sqrUa94VKJ4wU00dQHuZDJhpYI7gttLhCJGO5rqQnI9pZJrAJuu3m20IBMbiqWgLJ/TmNLCXPpQ", + "1jEOAFL+DoVaNL/nNQuwEJA8YFbPhdebM1DMicJyHDxak5zDjQEVNoAmPddzodxqKXC4hu0yAEVecXNH", + "WueCqw03ySzt8XpqCTj7T45G/DguwE+LkZEZmADfo81Juiyc6TwnW0KqL4omPUK8aJ0x7k20e8Yhz/G7", + "G5Sat0aXxXF2V/ARvsZktpwl9CcRqwuc/pqZ/R8xeOkX4EqK60FAt1+1HF998PJjYyuB9JbN+YKcCAjF", + "wjv6V14Q6yJzGuoDkQp0zhdUXhIITeEHQ/anqtZEqxwLTkLDP4V5oD6vIU3USSXzHGqerB1AfS+5S8gp", + "2gr86ScQ1+2jBmHdFLKiH4uG+KwoZEsM5hxFpcHG+RWfoecMF8JSNY4aiGd1Dr8gRoiKeOvXKarApSDh", + "crTNNoTsR6PnlZjdHWyz39ZWP1as7kpfNnbt73/7T9QoqDO6qHO0QXXS+2pU5i0r/E9R0FaPQXL0+YYC", + "9lXcWQC4xFykL5POb2kFD1bhmyA6EqMAm3f9pGIvE4UcThXB9OvtPxBvbPPJpcIZLZA/U3DrXabdpDMc", + "DuOYWL1z+IYVgCLOZW6HjOrfKc6Q7tedrjRQLoTVWdEt+xOuxgZtHhxhvYEMaykto5V4bH6Lz5lC3A7y", + "bw/fLHWOrClfPQmdMgDUFGpVVyE4eRfYCcXVWNyF8UU77R29TORy5DcRInt6yI4dwHZbINDEEF0umBI3", + "LvQIZdzxEbciUQA3BFkOywJoXf0bTOkA34OOHYAzQGRROsaVvRYGsPagyQSpkQUMPwDD41qqTF9jCGDK", + "qWYWfbvQKEpcscrGFc+ldUJJNR2yfRVYexpM7RjSSF9tv/Snwb8ezY/q0kp1bSRNFcYOjzl8Qzin9IhR", + "rseXbCRmEnGX2MQI8VcEATx2/jR7Jxbfk32XlV5vfFefOTnJu/iZLl0gIJG5S1TF1xwX1DvlhVAEUwQ9", + "+vC6iNiT2eAGg9c8SRSMUhZ9jAEQ3SlUDPPY7aMVaIAS3jxQvsf1TJR/5SHbZ4WGemNpmbgpIBAE6BhC", + "ZcJQAMkyr3LCg9U06TDDAyCcNzwhZi+M0SaWMf+iS4CWlEHwpGVZaaKkIKirsS5RtgTLclLmMJkaofWU", + "F7X2fkCJLGCdKAAs4bE8t5rNICIXuowR/9Yfcgoew9mkgsoK9Yn77chz3ChdurGe42Z4McWC9moyockv", + "tqvG7bvmfj0h7Dwxeh5OAsVpQlF4deww/I9RIpFbottCWkA/fCqzPCDXKo1C6vUevDwcgRTi2aYsnMhS", + "iJZD3Kww4krq0gK4SRaYYYHWMkoWnPmR1o6N4Hy6pdMfuAFBkgC/QTmc/B7Fj6TCUBpEBDFGFbff6USt", + "7JB/K9y7Sp1tvn+0NtiHMJF1IAw1XYtvfBtuOhLrcRawCeu/iu/bptaxLcDr9ebt8GtnJLgRxl/N/rLw", + "3iEe1DbL6pzPxUAbOZUKQAD1IBMOz20FmHZ2Akc91gDbQsBUSpN3djtbACpN07rVKQUXG6YFCc/LX0e2", + "gVQz8n7Eikwqy+VEjBfjXLDuwdmnw17jl5giuP1jhGDv17h6+hWDQB/ODartJUKK6uH079uP/jgzQlCc", + "NgJJFkY7PQY+gmCPBubDloDv6THL9Lj0V1TAvqBfZXrc+jp09fRZrqdSbeV6qkvXByjsa20yBKwQ/UjO", + "WNp6T5i/2drm4U1yvEQBxbZCpan91H+n5bfQVYy9v+gDgKE/sGNdiIz5N7wUC4u0dCfHW+eH/+rHqD23", + "kAP/jZZHV14HBSeoLwciftJpSEz7By9lD5o7OUxUrUUmBG0gSoE9V43bPqL4IisiFs6ChCRqrjM5WTRh", + "eIfs9OwlwwoPL5Wg4/eqKS4IcNgvZj9Rod+1H/Wmu9YD6/g0hjZjR2kOoXUFDDbi30uhXKKMyAW3InJw", + "1lKuE4E9WtiPiXqS1rjmYa3zd+wuOmcRnsUKByP5RbFDdrSE62xxWZbKWWJULMSU+mxq/Ib426aqiYHr", + "eysyY8JVPWQYfISF9G9fK0+DoECU0z1oYt+iuJy3O/CrQe4mwCczLXNucPbBNkBvtJDjS9pnorsRjQXD", + "57YsFkngqTAW8lb7MG/2UV8KZf1IoUW3bWcg6zXOtUJFIa/8zU2pcJWxri4CWU+PBThb/9UgNEN2DkUW", + "iRJqbBb+kh5wN8BEveRs/+h88PbgHabNARLc+UvZ62lKwjNxw8cuXyRKw7Wi2OmH849oODTRkLwZJsBI", + "aS4MNMcOAOWmbX3ekeQQkCo1+BM/gAaySoc8lbp0I8hLE+QBmIRTeSVsaN81mAeqIROg+S0ds16QyJJ+", + "v/9xyA4icBkNnSg8k0pf7yGoKGIJYwsJpqXyGuKCf7wkfCm4H2Cd6T700rSqJ/DT2YltLFHoc//tL7/9", + "fwEAAP//", } // decodeSpec returns the embedded OpenAPI spec as raw JSON bytes, diff --git a/server/internal/httpapi/searchtimings.go b/server/internal/httpapi/searchtimings.go new file mode 100644 index 00000000..08ecfc2a --- /dev/null +++ b/server/internal/httpapi/searchtimings.go @@ -0,0 +1,131 @@ +package httpapi + +import ( + "sync/atomic" + "time" +) + +// --------------------------------------------------------------------------- +// Where a workspace query spent its time. +// +// This exists because the alternative is arithmetic. The dense scan, the BM25 +// query and the fan-out's parallel speedup were each measured separately on the +// load-test fixture and multiplied together to guess at a 10.5 s query — a +// budget that happened to close, which is not the same as being right. Two +// optimisations were about to be built on that guess. +// +// The numbers are cheap: a handful of time.Now() calls per project and two +// atomics, against a query that reads gigabytes. So they are always COLLECTED. +// What they are not is always reported: +// +// - the log line is emitted only for a query slower than slowWorkspaceQuery. +// A breakdown printed for every query is noise nobody reads, and the +// server already logs one http_request line per request with the wall +// time in it, so the routine case is covered. A threshold keeps the +// property that matters — nobody has to have switched anything on before +// the slow query happened; +// - the response object is attached only when the caller asks for it with +// ?timings=true. In a response it is a debugging aid, not API surface. +// +// --------------------------------------------------------------------------- + +// slowWorkspaceQuery is the line above which a query is worth a log entry of +// its own. Workspace search is a fan-out over every project in the workspace +// and is expected to take a while; on the load-test fixture (45 repos, 1.9M +// chunks) the median is ~10 s, and even a small workspace on a warm cache is +// hundreds of milliseconds. Two seconds is therefore not "slow" in the sense +// of "wrong" — it is the point past which the breakdown starts being worth +// storing, and it is low enough that a regression on a small workspace still +// trips it. +// +// A var rather than a const only so the test can exercise both sides of the +// threshold without sleeping for two seconds. Nothing at runtime writes it, +// and the one test that does restores it via t.Cleanup. That is safe only +// because nothing in this package calls t.Parallel(); if you add parallel +// tests here, move the threshold onto Deps first — under -race this global +// becomes a data race whose cause is not obvious from the failure. +var slowWorkspaceQuery = 2 * time.Second + +// searchPhases accumulates one workspace query's timings. +// +// The DENSE phase keeps a SUM and a MAX, and both are needed: the sum is how +// much work the query did, the max is how long the user waited for the slowest +// project. With perfect parallelism the wall time is the max; with none it is +// the sum. Measured on the fixture, eight concurrent project searches ran 3.4x +// faster than the same eight in sequence — so the truth is between the two +// numbers, and reporting only one of them hides which. +// +// BM25 no longer has that shape: it is one workspace-wide statement, so it +// reports a single duration. See the bm25 field below. +type searchPhases struct { + embed time.Duration + resolve time.Duration + staleFTS time.Duration + fanOut time.Duration + fuse time.Duration + + // bm25 is a single duration, not a sum and a max, because there is a + // single query: one workspace-wide FTS5 statement partitioned per + // project. It ran per project once, and the sum/max split existed to + // separate "work done" from "waited for" across those. With one query + // they are the same number. + bm25 time.Duration + + denseSum atomic.Int64 // nanoseconds + denseMax atomic.Int64 +} + +// addDense records one project's dense-side latency. Includes the vector +// store's own hydration of the winning rows — the two are not separable from +// out here, and hydration is bounded by the result limit rather than by the +// collection size, so it is not what a scan-side change would move. +// +// A project whose query FAILED is recorded too. The time was spent, and +// leaving it out would put the sums permanently below the wall time they are +// meant to explain. It does mean a slow failure can own the max, which is why +// the fan-out logs its own warning per failed project. +func (p *searchPhases) addDense(d time.Duration) { addSumMax(&p.denseSum, &p.denseMax, d) } + +func addSumMax(sum, max *atomic.Int64, d time.Duration) { + n := d.Nanoseconds() + sum.Add(n) + for { + cur := max.Load() + if n <= cur || max.CompareAndSwap(cur, n) { + return + } + } +} + +// payload renders the timings for the response and for the log line. +// +// scanned vs returned is the ratio that decides whether routing the fan-out is +// worth building: the query does full dense and BM25 work on every project in +// the workspace and then thresholds the answer down. If those two numbers are +// far apart, most of the work was thrown away after it was paid for. +// +// `returned` must therefore be the count that survived the RELEVANCE +// THRESHOLD, not the count the caller was shown. The response panel is capped +// at top_projects (default 10), and feeding that number in here would peg the +// ratio at scanned:10 on any workspace with ten relevant repos or a hundred — +// a measurement of a request parameter rather than of wasted work. The panel +// count is reported separately, because "what did the caller get" is a +// different question from "what did the fan-out pay for". +func (p *searchPhases) payload(wall time.Duration, scanned, returned, panel int) map[string]any { + ms := func(d time.Duration) int64 { return d.Milliseconds() } + msn := func(n int64) int64 { return time.Duration(n).Milliseconds() } + return map[string]any{ + "wall_ms": ms(wall), + "embed_ms": ms(p.embed), + "resolve_ms": ms(p.resolve), + "stale_fts_ms": ms(p.staleFTS), + "fanout_ms": ms(p.fanOut), + "dense_sum_ms": msn(p.denseSum.Load()), + "dense_max_ms": msn(p.denseMax.Load()), + "bm25_ms": ms(p.bm25), + "fuse_ms": ms(p.fuse), + "projects_scanned": scanned, + "projects_returned": returned, + "projects_in_panel": panel, + } +} diff --git a/server/internal/httpapi/workspacesearch.go b/server/internal/httpapi/workspacesearch.go index d065d83a..757d0ba7 100644 --- a/server/internal/httpapi/workspacesearch.go +++ b/server/internal/httpapi/workspacesearch.go @@ -8,6 +8,7 @@ import ( "sort" "strconv" "sync" + "time" "golang.org/x/sync/errgroup" @@ -95,8 +96,10 @@ type workspaceSearchStaleFTSRepoPayload struct { } // projectHits is the per-project intermediate state accumulated across -// the parallel fan-out. Dense and BM25 sides arrive separately and are -// fused inside the goroutine before being collected. +// the parallel fan-out. Dense and BM25 sides arrive separately and are fused +// AFTER the fan-out, in the serial loop below g.Wait() — fusion needs both +// sides, so it cannot live in either goroutine. See the comment above that +// loop for why parallelising it buys nothing. type projectHits struct { ProjectPath string // FusedChunks are the per-project chunks ranked by RRF over the @@ -109,6 +112,13 @@ type projectHits struct { // (positive, unbounded — SQLite's bm25() flipped via -bm25 at // the chunksfts boundary). Normalized into candidacy via // per-query min-max before being blended. + // + // Computed on the RAW BM25 list, not on FusedChunks beside it. The two + // fields are scored at different layers: the chunk list a caller sees has + // been through RRF, where the dense side gets an equal vote, while the + // projects panel ranks on this number alone. So anything that moves BM25 + // moves the panel directly and the chunk list only after fusion has had a + // say — worth knowing before reading a panel reorder as a ranking change. BM25Signal float32 // Candidacy is the α-blended, per-query-normalized score the // projects panel ranks by; recomputed after every project's @@ -118,10 +128,10 @@ type projectHits struct { // WorkspaceSearch — GET /api/v1/workspaces/{id}/search. // -// Hybrid BM25+dense fan-out. Each project runs two queries in -// parallel: dense (vector-store cosine) and sparse (SQLite FTS5 BM25 over -// chunks_fts). Per project, the two ranked lists are fused via -// Reciprocal Rank Fusion. Across projects, an α-blended candidacy +// Hybrid BM25+dense fan-out. Dense (vector-store cosine) runs once per +// project, concurrently; sparse is ONE FTS5 BM25 query over chunks_fts for the +// whole workspace, partitioned per project by the caller. Per project, the two +// ranked lists are fused via Reciprocal Rank Fusion. Across projects, an α-blended candidacy // score (with per-query min-max normalization on both signals) plus // a relative threshold (`candidacy ≥ best × 0.4`) keeps the result // set focused on repos that actually share vocabulary or semantics @@ -135,6 +145,13 @@ type projectHits struct { // project threshold those repos drop out, restoring the cross-project // signal the user needs to scope an agent's follow-up search. func (s *Server) WorkspaceSearch(w http.ResponseWriter, r *http.Request, id string, params openapi.WorkspaceSearchParams) { + // Always measured, conditionally reported — see searchtimings.go. Started + // on the handler's first line so wall_ms means what it says: everything + // including the visibility check, not just the part that was interesting + // to whoever added the next phase. + started := time.Now() + var phases searchPhases + if s.workspaceProjectsUnavailable(w) { return } @@ -160,7 +177,11 @@ func (s *Server) WorkspaceSearch(w http.ResponseWriter, r *http.Request, id stri // explicitly. minScore := clampFloat32(params.MinScore, 0.4, 0, 1) + wantTimings := params.Timings != nil && *params.Timings + + embedStart := time.Now() queryEmbedding, err := s.Deps.EmbeddingSvc.EmbedQuery(r.Context(), params.Q) + phases.embed = time.Since(embedStart) if err != nil { writeError(w, http.StatusServiceUnavailable, "could not embed query: "+err.Error()) return @@ -173,6 +194,7 @@ func (s *Server) WorkspaceSearch(w http.ResponseWriter, r *http.Request, id stri // Pull the workspace's project memberships joined with the projects // table so we can split into indexed vs pending in one pass. The // junction lives in workspace_projects; status lives on projects. + resolveStart := time.Now() rows, err := s.Deps.DB.QueryContext(r.Context(), ` SELECT p.host_path, p.status FROM workspace_projects wp @@ -224,6 +246,7 @@ func (s *Server) WorkspaceSearch(w http.ResponseWriter, r *http.Request, id stri } members = filtered } + phases.resolve = time.Since(resolveStart) if len(members) == 0 { writeJSON(w, http.StatusOK, workspaceSearchResponse( @@ -233,6 +256,12 @@ func (s *Server) WorkspaceSearch(w http.ResponseWriter, r *http.Request, id stri nil, nil, nil, + // Nothing was searched, so nothing is attached to the response + // whatever the caller asked for. It still goes through the + // reporter: embed_ms has already been paid by this point, and a + // hung embedding provider is exactly the case the log line is + // for. + s.reportSearchTimings(id, params.Q, &phases, started, 0, 0, 0, false), )) return } @@ -263,6 +292,7 @@ func (s *Server) WorkspaceSearch(w http.ResponseWriter, r *http.Request, id stri pendingRepos, nil, nil, + s.reportSearchTimings(id, params.Q, &phases, started, 0, 0, 0, false), )) return } @@ -275,14 +305,20 @@ func (s *Server) WorkspaceSearch(w http.ResponseWriter, r *http.Request, id stri // reindex — otherwise the operator sees no observable difference // from the pre-hybrid algorithm and assumes the change didn't // take effect. + staleStart := time.Now() staleRepos := s.detectStaleFTSRepos(r.Context(), projectPaths) + phases.staleFTS = time.Since(staleStart) - hits, failedRepos, err := s.fanOutHybrid(r.Context(), id, projectPaths, params.Q, queryEmbedding, minScore) + fanOutStart := time.Now() + hits, failedRepos, err := s.fanOutHybrid(r.Context(), id, projectPaths, params.Q, queryEmbedding, minScore, &phases) + phases.fanOut = time.Since(fanOutStart) if err != nil { writeError(w, http.StatusInternalServerError, "fan-out search failed: "+err.Error()) return } + fuseStart := time.Now() + // Per-query min-max normalization on each signal independently, // then α-blend. Both signals are >=0; using raw/max instead of // (raw-min)/(max-min) means a project at 60% of best gets 0.6 @@ -329,6 +365,7 @@ func (s *Server) WorkspaceSearch(w http.ResponseWriter, r *http.Request, id stri } surviving = append(surviving, ph) } + phases.fuse = time.Since(fuseStart) if len(surviving) == 0 { status := "empty" @@ -342,24 +379,40 @@ func (s *Server) WorkspaceSearch(w http.ResponseWriter, r *http.Request, id stri pendingRepos, failedRepos, staleRepos, + s.reportSearchTimings(id, params.Q, &phases, started, len(projectPaths), 0, 0, wantTimings), )) return } - // Build the projects panel + the flat chunk list. Per-project cap - // is applied to each project's fused chunk list so one dominant - // repo can't take every slot in the round-robin interleave below; - // the projects panel sees every surviving project (its num_hits - // reflects the post-cap count so the UI doesn't dangle a "10 - // hits" badge against a chunk list with 5 entries). + // Build the projects panel + the flat chunk list. The per-project cap is + // applied to each project's fused chunk list so one dominant repo can't + // take every slot in the round-robin interleave below. num_hits reflects + // the post-cap count, so the UI doesn't dangle a "10 hits" badge against a + // chunk list with 5 entries. + // + // The panel itself is capped at top_projects further down. + // projects_returned is NOT, and must not be — it counts what cleared the + // relevance threshold, and capping it turns the scanned:returned ratio + // into a measurement of a request parameter. That has been re-broken once + // already while refactoring these very lines. for i := range surviving { if len(surviving[i].FusedChunks) > workspaceSearchPerProjChunkCap { surviving[i].FusedChunks = surviving[i].FusedChunks[:workspaceSearchPerProjChunkCap] } } - projectPayloads := make([]workspaceSearchProjectPayload, 0, len(surviving)) - for _, ph := range surviving { + sortPanel(surviving) + // `panel` reslices, it does not shrink `surviving` — projects_returned + // counts what cleared the relevance threshold, and capping that at + // top_projects is exactly the bug fixed in d511513. The test for it caught + // this line the first time it was written the other way round. + panel := surviving + if len(panel) > topProjects { + panel = panel[:topProjects] + } + + projectPayloads := make([]workspaceSearchProjectPayload, 0, len(panel)) + for _, ph := range panel { projectPayloads = append(projectPayloads, workspaceSearchProjectPayload{ ProjectPath: ph.ProjectPath, Label: projectLabel(ph.ProjectPath), @@ -370,29 +423,12 @@ func (s *Server) WorkspaceSearch(w http.ResponseWriter, r *http.Request, id stri }) } - sort.SliceStable(projectPayloads, func(i, j int) bool { - return projectPayloads[i].ProjectScore > projectPayloads[j].ProjectScore - }) - if len(projectPayloads) > topProjects { - projectPayloads = projectPayloads[:topProjects] - } - - // Restrict the interleave to projects that survived the panel - // truncation. Otherwise a workspace with > top_projects surviving - // repos can surface chunks whose project_path is absent from - // projects[] — agents lose access to bm25_score/dense_score and - // the response looks inconsistent. Filter to the panel before - // round-robin. - panelSet := make(map[string]struct{}, len(projectPayloads)) - for _, p := range projectPayloads { - panelSet[p.ProjectPath] = struct{}{} - } - panelSurviving := make([]projectHits, 0, len(projectPayloads)) - for _, ph := range surviving { - if _, ok := panelSet[ph.ProjectPath]; ok { - panelSurviving = append(panelSurviving, ph) - } - } + // The interleave is restricted to the panel. Otherwise a workspace with + // more than top_projects surviving repos can surface chunks whose + // project_path is absent from projects[] — agents lose access to + // bm25_score/dense_score and the response looks inconsistent. `panel` is + // already exactly that set, in the same order, so the set-membership + // filter this used to do is no longer needed. // Round-robin across surviving projects so rank-1 from each // project lands in the first N slots, then rank-2, etc. This @@ -400,7 +436,7 @@ func (s *Server) WorkspaceSearch(w http.ResponseWriter, r *http.Request, id stri // before any repo's tail entries appear — matches the project- // picker use case where the user wants to see each project's // most-relevant hit before diving into the dominant repo's tail. - merged := interleaveByRank(panelSurviving, topChunks) + merged := interleaveByRank(panel, topChunks) status := "ok" if len(merged) == 0 { @@ -416,9 +452,67 @@ func (s *Server) WorkspaceSearch(w http.ResponseWriter, r *http.Request, id stri pendingRepos, failedRepos, staleRepos, + s.reportSearchTimings(id, params.Q, &phases, started, + len(projectPaths), len(surviving), len(projectPayloads), wantTimings), )) } +// reportSearchTimings logs the phase breakdown if the query was slow, and +// returns it for the response if the caller asked for it. +// +// Both gates are deliberate; see searchtimings.go. The breakdown itself is +// always computed because it costs a map allocation — the decision here is +// only about where it goes. +func (s *Server) reportSearchTimings(workspaceID, query string, p *searchPhases, + started time.Time, scanned, returned, panel int, requested bool) map[string]any { + + wall := time.Since(started) + t := p.payload(wall, scanned, returned, panel) + + if wall >= slowWorkspaceQuery { + fields := []any{"workspace_id", workspaceID, "query_len", len(query)} + for _, k := range timingFields { + fields = append(fields, k, t[k]) + } + s.Deps.Logger.Info("slow workspace search", fields...) + } + + if !requested { + return nil + } + return t +} + +// timingFields fixes the order of the log line's fields so successive lines +// line up when read by eye, which is the only way anyone reads them. +var timingFields = []string{ + "wall_ms", "embed_ms", "resolve_ms", "stale_fts_ms", "fanout_ms", + "dense_sum_ms", "dense_max_ms", "bm25_ms", + "fuse_ms", "projects_scanned", "projects_returned", "projects_in_panel", +} + +// sortPanel orders the projects panel, best first. +// +// On the RAW candidacy, not on the rounded copy that goes out in the JSON. +// round4 manufactures ties — 0.71234 and 0.71236 both become 0.7123 — and the +// caller truncates to top_projects immediately afterwards, so a tie decides +// which of two repos the caller sees at all. Sorting the rounded value handed +// that decision to whatever order the projects happened to arrive in, which is +// workspace membership order (added_at DESC): the panel became a function of +// insertion history rather than of the query. +// +// ProjectPath breaks a genuine tie, so the result is a total order. Split out +// from the handler so the property can be tested on a constructed slice +// instead of through timestamps that a test cannot control. +func sortPanel(surviving []projectHits) { + sort.Slice(surviving, func(i, j int) bool { + if surviving[i].Candidacy != surviving[j].Candidacy { + return surviving[i].Candidacy > surviving[j].Candidacy + } + return surviving[i].ProjectPath < surviving[j].ProjectPath + }) +} + // interleaveByRank returns up to `limit` chunks by walking the surviving // projects round-robin — rank-1 from every project before any rank-2, // then rank-2, and so on. Projects are visited in candidacy-desc order @@ -440,10 +534,6 @@ func interleaveByRank(projects []projectHits, limit int) []workspaceSearchChunkP }) out := make([]workspaceSearchChunkPayload, 0, limit) - dedupKey := func(c workspaceSearchChunkPayload) string { - return c.ProjectPath + "|" + c.FilePath + "|" + - strconv.Itoa(c.StartLine) + "-" + strconv.Itoa(c.EndLine) - } seen := make(map[string]struct{}, limit) // rank index walks 0,1,2,... ; we stop when no project has a // chunk at this rank (every list exhausted). @@ -457,7 +547,7 @@ func interleaveByRank(projects []projectHits, limit int) []workspaceSearchChunkP if c.ProjectPath == "" { c.ProjectPath = p.ProjectPath } - k := dedupKey(c) + k := chunkKey(c) if _, ok := seen[k]; ok { continue } @@ -485,12 +575,16 @@ func workspaceSearchResponse( pending []workspaceSearchPendingRepoPayload, failed []workspaceSearchFailedRepoPayload, stale []workspaceSearchStaleFTSRepoPayload, + timings map[string]any, ) map[string]any { out := map[string]any{ "status": status, "projects": projects, "chunks": chunks, } + if timings != nil { + out["timings"] = timings + } if len(pending) > 0 { out["pending_repos"] = pending } @@ -509,43 +603,61 @@ func workspaceSearchResponse( // reindex before BM25 can contribute. A best-effort detector: if any // SQL probe errors out we log + return nil rather than fail the // request, since the warning is informational, not load-bearing. +// +// EXISTS, not COUNT(*), and the difference is not cosmetic: the question is +// "are there any rows", and COUNT walks every matching index entry to answer +// it. Measured on the 45-repo load-test index (1.95M rows in chunks_meta), +// this loop cost 53 ms per workspace search as COUNT and 0.2 ms as EXISTS — +// and it runs BEFORE the fan-out, so every query pays it serially. The LIMIT 1 +// that used to be on these statements did nothing: it bounds the result rows +// of an aggregate that always returns exactly one. func (s *Server) detectStaleFTSRepos(ctx context.Context, projectPaths []string) []workspaceSearchStaleFTSRepoPayload { out := make([]workspaceSearchStaleFTSRepoPayload, 0) for _, pp := range projectPaths { - var nMeta, nFiles int + var hasMeta, hasFiles bool if err := s.Deps.DB.QueryRowContext(ctx, - `SELECT COUNT(*) FROM chunks_meta WHERE project_path = ? LIMIT 1`, pp).Scan(&nMeta); err != nil { + `SELECT EXISTS(SELECT 1 FROM chunks_meta WHERE project_path = ?)`, pp).Scan(&hasMeta); err != nil { s.Deps.Logger.Warn("workspaces search: stale-fts probe (chunks_meta)", "project_path", pp, "err", err) return nil } - if nMeta > 0 { + if hasMeta { continue } if err := s.Deps.DB.QueryRowContext(ctx, - `SELECT COUNT(*) FROM file_hashes WHERE project_path = ? LIMIT 1`, pp).Scan(&nFiles); err != nil { + `SELECT EXISTS(SELECT 1 FROM file_hashes WHERE project_path = ?)`, pp).Scan(&hasFiles); err != nil { s.Deps.Logger.Warn("workspaces search: stale-fts probe (file_hashes)", "project_path", pp, "err", err) return nil } - if nFiles > 0 { + if hasFiles { out = append(out, workspaceSearchStaleFTSRepoPayload{ProjectPath: pp}) } } return out } -// fanOutHybrid runs dense + BM25 in parallel per project, fuses each -// project's two ranked lists via RRF, and returns the per-project -// aggregates the candidacy step needs. Bounded by NumCPU goroutines -// across the workspace; each project is one slot regardless of -// whether it issues one or two sub-queries. +// fanOutHybrid runs the dense scan per project and BM25 once for the +// whole workspace, fuses each project's two ranked lists via RRF, and +// returns the per-project aggregates the candidacy step needs. Bounded +// by NumCPU goroutines; the BM25 query takes one of those slots and runs +// alongside the dense scans rather than before them. // -// Per-project failures: a BM25-side error is logged but does not mark -// the project as failed (FTS5 might not be populated yet for a -// pre-existing install; dense still works). A dense-side error is -// surfaced via failed_repos and dense_signal is left at 0 — the -// project can still be retained if BM25 alone is strong. +// BM25 used to be per-project too, which is the expensive way to ask. +// FTS5 evaluates MATCH over the whole server's chunks_fts and filters by +// project afterwards, so N projects meant N evaluations of the same +// global match — and, measured on a 43-project workspace, 78-80% of the +// fan-out's total work plus an order-of-magnitude slowdown from the +// queries contending with each other over one index. See +// chunksfts.SearchProjects. +// +// Failures: a BM25-side error is logged but fails nothing (FTS5 might +// not be populated yet for a pre-existing install; dense still works). +// It is now one error for the workspace rather than one per project — +// the blast radius grew, which is the cost of asking once, and the +// fallback is the same one a pre-FTS install already relies on. A +// dense-side error is surfaced via failed_repos and dense_signal is left +// at 0 — the project can still be retained if BM25 alone is strong. func (s *Server) fanOutHybrid( ctx context.Context, workspaceID string, @@ -553,6 +665,7 @@ func (s *Server) fanOutHybrid( rawQuery string, queryEmbedding []float32, minScore float32, + phases *searchPhases, ) ([]projectHits, []workspaceSearchFailedRepoPayload, error) { concurrency := runtime.NumCPU() if concurrency < 1 { @@ -565,18 +678,45 @@ func (s *Server) fanOutHybrid( results := make([]projectHits, len(projectPaths)) failures := make([]workspaceSearchFailedRepoPayload, len(projectPaths)) failed := make([]bool, len(projectPaths)) + denseHits := make([][]workspaceSearchChunkPayload, len(projectPaths)) var mu sync.Mutex + // One BM25 query for the whole workspace, concurrent with the dense + // scans. Errors are swallowed on purpose: bm25ByProject stays nil and + // every project falls back to dense-only, which is what a pre-FTS + // install does anyway. + var bm25ByProject map[string][]chunksfts.Hit + g.Go(func() error { + bm25Start := time.Now() + hits, berr := chunksfts.SearchProjects(gctx, s.Deps.DB, projectPaths, rawQuery, workspaceSearchBM25Limit) + phases.bm25 = time.Since(bm25Start) + if berr != nil { + s.Deps.Logger.Warn("workspaces search: bm25 query failed", + "workspace_id", workspaceID, + "projects", len(projectPaths), + "err", berr) + return nil + } + // No lock: this is the only writer, and every reader runs after + // g.Wait(), which is the happens-before. A mutex here would be + // decoration that reads like protection — the next person needing + // these hits INSIDE the fan-out would see a locked write, assume the + // map was safe to touch concurrently, and add a real race. + bm25ByProject = hits + return nil + }) + for i, pp := range projectPaths { i, pp := i, pp g.Go(func() error { var ( denseRes []workspaceSearchChunkPayload - bm25Res []workspaceSearchChunkPayload denseErr error ) + denseStart := time.Now() rawDense, derr := s.Deps.VectorStore.Search(gctx, pp, queryEmbedding, workspaceSearchPerProjectLimit, nil) + phases.addDense(time.Since(denseStart)) if derr != nil { denseErr = derr s.Deps.Logger.Warn("workspaces search: dense query failed", @@ -602,37 +742,6 @@ func (s *Server) fanOutHybrid( } } - rawBM25, berr := chunksfts.SearchProject(gctx, s.Deps.DB, pp, rawQuery, workspaceSearchBM25Limit) - if berr != nil { - s.Deps.Logger.Warn("workspaces search: bm25 query failed", - "workspace_id", workspaceID, - "project_path", pp, - "err", berr) - } else { - bm25Res = make([]workspaceSearchChunkPayload, 0, len(rawBM25)) - for _, h := range rawBM25 { - bm25Res = append(bm25Res, workspaceSearchChunkPayload{ - ProjectPath: pp, - FilePath: h.FilePath, - StartLine: h.StartLine, - EndLine: h.EndLine, - SymbolName: h.SymbolName, - Language: h.Language, - // Score field carries the dense cosine for the - // merged chunk; for BM25-only hits we leave it - // at 0 (BM25 score is on a different scale and - // would mislead a client reading "score" as - // cosine). - Score: 0, - Content: h.Content, - }) - } - } - - fused := fuseRRF(denseRes, bm25Res) - denseSig := meanTopN(denseScoresOf(denseRes), workspaceSearchTopNPerProject) - bm25Sig := meanTopN(bm25ScoresOf(rawBM25), workspaceSearchTopNPerProject) - mu.Lock() if denseErr != nil { failures[i] = workspaceSearchFailedRepoPayload{ @@ -641,12 +750,7 @@ func (s *Server) fanOutHybrid( } failed[i] = true } - results[i] = projectHits{ - ProjectPath: pp, - FusedChunks: fused, - DenseSignal: float32(denseSig), - BM25Signal: float32(bm25Sig), - } + denseHits[i] = denseRes mu.Unlock() return nil }) @@ -654,6 +758,47 @@ func (s *Server) fanOutHybrid( if err := g.Wait(); err != nil { return nil, nil, err } + + // Fusion moved out of the goroutines with the BM25 query: it needs both + // sides, and it is arithmetic over at most a hundred chunks per project. + // Measured across the whole 43-project fan-out it was under a + // millisecond, so there is nothing to gain by parallelising it and a + // simpler read to be had by not. + for i, pp := range projectPaths { + rawBM25 := bm25ByProject[pp] + bm25Res := make([]workspaceSearchChunkPayload, 0, len(rawBM25)) + for _, h := range rawBM25 { + bm25Res = append(bm25Res, workspaceSearchChunkPayload{ + ProjectPath: pp, + FilePath: h.FilePath, + StartLine: h.StartLine, + EndLine: h.EndLine, + SymbolName: h.SymbolName, + Language: h.Language, + // Score field carries the dense cosine for the merged + // chunk; for BM25-only hits we leave it at 0 (BM25 score + // is on a different scale and would mislead a client + // reading "score" as cosine). + Score: 0, + Content: h.Content, + }) + } + results[i] = projectHits{ + ProjectPath: pp, + FusedChunks: fuseRRF(denseHits[i], bm25Res), + DenseSignal: float32(meanTopN(denseScoresOf(denseHits[i]), workspaceSearchTopNPerProject)), + BM25Signal: float32(meanTopN(bm25ScoresOf(rawBM25), workspaceSearchTopNPerProject)), + } + // Release this project's dense slice now that both fuseRRF and + // denseScoresOf have consumed it. To be accurate about what this + // buys: fuseRRF returns the UNION of both lists, so the Content + // strings stay reachable through results[i].FusedChunks either way — + // what goes is the backing array of ~50 payload structs per project, + // a few KB, not the chunk text. Free, correctly ordered, and worth + // keeping; just not the workspace-wide retention fix an earlier + // version of this comment claimed. + denseHits[i] = nil + } failedOut := make([]workspaceSearchFailedRepoPayload, 0) for i, f := range failed { if f { @@ -663,6 +808,17 @@ func (s *Server) fanOutHybrid( return results, failedOut, nil } +// chunkKey is chunk identity: the same span of the same file in the same +// project. Shared by fusion and by the round-robin interleave's dedup, and +// since fusion started using it as a sort tiebreak it decides ORDER as well as +// identity — so the two callers agreeing is no longer merely tidy. Two copies +// of this expression drifting would make the fusion tiebreak and the interleave +// dedup disagree with no error anywhere. +func chunkKey(c workspaceSearchChunkPayload) string { + return c.ProjectPath + "|" + c.FilePath + "|" + + strconv.Itoa(c.StartLine) + "-" + strconv.Itoa(c.EndLine) +} + // fuseRRF returns chunks ranked by Reciprocal Rank Fusion over the two // per-project lists. RRF score per chunk is sum(1/(k+rank_i)) across // the lists where it appears. Chunks present in both lists naturally @@ -675,19 +831,16 @@ func (s *Server) fanOutHybrid( func fuseRRF(dense, bm25 []workspaceSearchChunkPayload) []workspaceSearchChunkPayload { type entry struct { c workspaceSearchChunkPayload + k string rrf float64 } - key := func(c workspaceSearchChunkPayload) string { - return c.ProjectPath + "|" + c.FilePath + "|" + - strconv.Itoa(c.StartLine) + "-" + strconv.Itoa(c.EndLine) - } byKey := make(map[string]*entry) for rank, c := range dense { - k := key(c) + k := chunkKey(c) byKey[k] = &entry{c: c, rrf: 1.0 / float64(rrfK+rank+1)} } for rank, c := range bm25 { - k := key(c) + k := chunkKey(c) add := 1.0 / float64(rrfK+rank+1) if e, ok := byKey[k]; ok { e.rrf += add @@ -696,11 +849,28 @@ func fuseRRF(dense, bm25 []workspaceSearchChunkPayload) []workspaceSearchChunkPa byKey[k] = &entry{c: c, rrf: add} } out := make([]entry, 0, len(byKey)) - for _, e := range byKey { + for k, e := range byKey { + e.k = k out = append(out, *e) } - sort.SliceStable(out, func(i, j int) bool { - return out[i].rrf > out[j].rrf + // The tiebreak is load-bearing, not tidiness. `out` is built by ranging + // over a map, and Go randomises map iteration order deliberately — so + // without a total order here, chunks with equal RRF come back in a + // different order on every call, and SliceStable faithfully preserves that + // randomness. Equal RRF is not a corner case: a chunk found only by dense + // at rank r and a chunk found only by BM25 at the same rank r score + // identically by construction, which happens in most queries. + // + // Observed on the load-test fixture before this line existed: the same + // query, same process, same binary, returned a different chunk at rank 0 + // between consecutive calls. Project scores were unaffected — they do not + // depend on chunk order — so the panel looked stable while the results + // underneath it moved. + sort.Slice(out, func(i, j int) bool { + if out[i].rrf != out[j].rrf { + return out[i].rrf > out[j].rrf + } + return out[i].k < out[j].k }) chunks := make([]workspaceSearchChunkPayload, len(out)) for i, e := range out { diff --git a/server/internal/httpapi/workspacesearch_test.go b/server/internal/httpapi/workspacesearch_test.go index 54bb9c2c..d3327358 100644 --- a/server/internal/httpapi/workspacesearch_test.go +++ b/server/internal/httpapi/workspacesearch_test.go @@ -1,13 +1,17 @@ package httpapi import ( + "bytes" "context" "database/sql" "encoding/json" + "fmt" + "log/slog" "math" "net/http" "path/filepath" "strconv" + "strings" "testing" "time" @@ -42,6 +46,15 @@ func (e fixedEmbedder) Ready(_ context.Context) error { return nil } // vectorstore (real, on tmpdir), and a query embedder the caller // controls. func newSearchRouter(t *testing.T, d *sql.DB, vs *vectorstore.Store, emb fixedEmbedder) http.Handler { + t.Helper() + return newSearchRouterWithLogger(t, d, vs, emb, nil) +} + +// newSearchRouterWithLogger is newSearchRouter with the logger under the +// test's control, for the tests that assert on what got logged. A nil logger +// keeps the router's own default. +func newSearchRouterWithLogger(t *testing.T, d *sql.DB, vs *vectorstore.Store, + emb fixedEmbedder, logger *slog.Logger) http.Handler { t.Helper() t.Setenv("CIX_SECRET_KEY", "") t.Setenv("CIX_SECRET_KEYFILE", "") @@ -51,6 +64,7 @@ func newSearchRouter(t *testing.T, d *sql.DB, vs *vectorstore.Store, emb fixedEm } return NewRouter(Deps{ DB: d, + Logger: logger, AuthDisabled: true, Users: seedlessUsers(d), Sessions: seedlessSessions(d), @@ -1101,3 +1115,632 @@ func TestWorkspaceSearch_DefaultMinScoreIs04(t *testing.T) { len(openResp.Projects), openResp.Projects) } } + +// TestWorkspaceSearch_ReportsPhaseTimings covers the diagnostic added because +// the previous round of optimisation work was steered by arithmetic across +// separate measurements rather than by a number taken inside the handler. +// +// It asserts SHAPE, not values. Wall-clock in CI is noise — an assertion that +// dense_sum_ms is under some bound would fail on a loaded runner and teach +// everyone to ignore it. What can be asserted is that every field is present +// when the caller asks for the breakdown, and that the two counters describe +// the fan-out the request actually performed. +func TestWorkspaceSearch_ReportsPhaseTimings(t *testing.T) { + d, err := dbOpenMemory(t) + if err != nil { + t.Fatalf("open db: %v", err) + } + vs := openTestVectorStore(t) + query := l2([]float32{1, 0, 0, 0}) + router := newSearchRouter(t, d, vs, fixedEmbedder{q: query}) + wsID := createWS(t, router, "timings") + + // Two projects, one of which cannot clear the relevance threshold, so + // projects_scanned and projects_returned are different numbers. Their + // ratio is the whole reason the counters exist: it says how much of the + // fan-out's work was discarded after it was paid for. + seedRepoWithChunks(t, d, vs, wsID, "github.com/o/near@main", + []vectorstore.Chunk{ + {Content: "near", FilePath: "n.go", StartLine: 1, EndLine: 9, + ChunkType: "function", SymbolName: "N", Language: "go"}, + }, + [][]float32{l2([]float32{1.0, 0.0, 0.0, 0.0})}, + ) + seedRepoWithChunks(t, d, vs, wsID, "github.com/o/far@main", + []vectorstore.Chunk{ + {Content: "far", FilePath: "f.go", StartLine: 1, EndLine: 9, + ChunkType: "function", SymbolName: "F", Language: "go"}, + }, + [][]float32{l2([]float32{0.0, 0.0, 0.0, 1.0})}, + ) + + rr := doJSON(t, router, http.MethodGet, "/api/v1/workspaces/"+wsID+"/search?q=near&timings=true", nil) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d (%s)", rr.Code, rr.Body.String()) + } + var body struct { + Timings map[string]json.Number `json:"timings"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v", err) + } + if body.Timings == nil { + t.Fatal("no timings on a response that asked for them and ran a search") + } + for _, field := range timingFields { + if _, ok := body.Timings[field]; !ok { + t.Errorf("timings is missing %q: %v", field, body.Timings) + } + } + + num := func(k string) int64 { + n, err := body.Timings[k].Int64() + if err != nil { + t.Fatalf("%s is not an integer: %v", k, err) + } + return n + } + if got := num("projects_scanned"); got != 2 { + t.Errorf("projects_scanned = %d, want 2 — the fan-out searched both repos", got) + } + if got := num("projects_returned"); got != 1 { + t.Errorf("projects_returned = %d, want 1 — only the near repo clears the threshold", got) + } + if got := num("projects_in_panel"); got != 1 { + t.Errorf("projects_in_panel = %d, want 1", got) + } + // The max of a phase cannot exceed its sum, whatever the machine was + // doing at the time. This is the one relationship worth pinning: it + // catches a sum and a max wired to the wrong accumulator, which would + // otherwise look plausible in every log line. Only dense is split this + // way — BM25 is a single workspace-wide query, so it reports one + // number. + if sum, max := num("dense_sum_ms"), num("dense_max_ms"); max > sum { + t.Errorf("dense_max_ms (%d) exceeds dense_sum_ms (%d)", max, sum) + } + // bm25_ms is a plain field assigned in exactly one place. Drop that + // assignment and payload() still emits "bm25_ms": 0, the presence loop + // above still passes, and every log line reads as though BM25 were free — + // which is the number that pointed at an 18.6 s query the day this landed. + // + // Be clear about what this does and does not buy: a phase inside the + // fan-out cannot outlast it, so a timer around the wrong span is caught. + // A DROPPED assignment is not — 0 <= fanout holds — and it cannot be, + // because an in-memory corpus legitimately rounds to 0 ms and "> 0" would + // be flaky. That gap is real; the alternative is a flaky test, which is + // worse than an honest partial one. + if bm, fan := num("bm25_ms"), num("fanout_ms"); bm > fan { + t.Errorf("bm25_ms (%d) exceeds fanout_ms (%d)", bm, fan) + } + assertCounterOrder(t, num) +} + +// assertCounterOrder pins the one relationship the three project counters must +// always satisfy. reportSearchTimings takes them as three consecutive ints — +// scanned, returned, panel — which is exactly the signature where a +// transposition compiles, produces plausible-looking numbers, and is invisible +// until someone reasons from the ratio. The panel is a cap on what survived, +// and what survived is a subset of what was searched. +func assertCounterOrder(t *testing.T, num func(string) int64) { + t.Helper() + scanned, returned, panel := num("projects_scanned"), num("projects_returned"), num("projects_in_panel") + if !(panel <= returned && returned <= scanned) { + t.Errorf("counters out of order: projects_in_panel=%d, projects_returned=%d, projects_scanned=%d "+ + "(want panel <= returned <= scanned)", panel, returned, scanned) + } +} + +// TestWorkspaceSearch_NoTimingsWithoutASearch is the other half: an empty +// workspace never reaches the fan-out, and reporting zeroes for phases that +// did not run would read as "the search was instant" to whoever is reading +// them. It asks for timings explicitly, so a pass means the search gate held, +// not merely that the opt-in gate did. +func TestWorkspaceSearch_NoTimingsWithoutASearch(t *testing.T) { + d, err := dbOpenMemory(t) + if err != nil { + t.Fatalf("open db: %v", err) + } + vs := openTestVectorStore(t) + router := newSearchRouter(t, d, vs, fixedEmbedder{q: l2([]float32{1, 0, 0, 0})}) + wsID := createWS(t, router, "notimings") + + rr := doJSON(t, router, http.MethodGet, "/api/v1/workspaces/"+wsID+"/search?q=anything&timings=true", nil) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d (%s)", rr.Code, rr.Body.String()) + } + var raw map[string]any + if err := json.Unmarshal(rr.Body.Bytes(), &raw); err != nil { + t.Fatalf("decode: %v", err) + } + if _, present := raw["timings"]; present { + t.Errorf("empty workspace reported timings: %v", raw["timings"]) + } +} + +// TestWorkspaceSearch_TimingsAreOptIn pins the half of the contract the +// opt-in exists for. The breakdown is a debugging aid; every caller that did +// not ask for it — the CLI, the MCP tools, the dashboard — must get the same +// response shape it got before the diagnostic was added. +func TestWorkspaceSearch_TimingsAreOptIn(t *testing.T) { + d, err := dbOpenMemory(t) + if err != nil { + t.Fatalf("open db: %v", err) + } + vs := openTestVectorStore(t) + query := l2([]float32{1, 0, 0, 0}) + router := newSearchRouter(t, d, vs, fixedEmbedder{q: query}) + wsID := createWS(t, router, "optin") + seedRepoWithChunks(t, d, vs, wsID, "github.com/o/near@main", + []vectorstore.Chunk{ + {Content: "near", FilePath: "n.go", StartLine: 1, EndLine: 9, + ChunkType: "function", SymbolName: "N", Language: "go"}, + }, + [][]float32{l2([]float32{1.0, 0.0, 0.0, 0.0})}, + ) + + for _, q := range []string{"", "&timings=false", "&timings=0"} { + rr := doJSON(t, router, http.MethodGet, + "/api/v1/workspaces/"+wsID+"/search?q=near"+q, nil) + if rr.Code != http.StatusOK { + t.Fatalf("%q: expected 200, got %d (%s)", q, rr.Code, rr.Body.String()) + } + var raw map[string]any + if err := json.Unmarshal(rr.Body.Bytes(), &raw); err != nil { + t.Fatalf("%q: decode: %v", q, err) + } + if _, present := raw["timings"]; present { + t.Errorf("%q: timings attached without being asked for: %v", q, raw["timings"]) + } + // The search itself must still have happened. + if chunks, ok := raw["chunks"].([]any); !ok || len(chunks) == 0 { + t.Errorf("%q: no chunks — the request did not actually search: %v", q, raw["chunks"]) + } + } +} + +// TestWorkspaceSearch_LogsOnlySlowQueries covers the other gate, the one that +// decides how much this diagnostic costs in production. The server already +// logs an http_request line per request; a second line per workspace query +// would be noise on every query to catch the rare slow one. The threshold is +// what buys the property that matters — nobody has to have switched anything +// on before the slow query happens. +// +// Both directions are asserted from one logger, because a test that only +// proves silence would still pass if the line were deleted outright. +func TestWorkspaceSearch_LogsOnlySlowQueries(t *testing.T) { + d, err := dbOpenMemory(t) + if err != nil { + t.Fatalf("open db: %v", err) + } + vs := openTestVectorStore(t) + var logs bytes.Buffer + router := newSearchRouterWithLogger(t, d, vs, + fixedEmbedder{q: l2([]float32{1, 0, 0, 0})}, + slog.New(slog.NewJSONHandler(&logs, &slog.HandlerOptions{Level: slog.LevelInfo}))) + wsID := createWS(t, router, "slowlog") + seedRepoWithChunks(t, d, vs, wsID, "github.com/o/near@main", + []vectorstore.Chunk{ + {Content: "near", FilePath: "n.go", StartLine: 1, EndLine: 9, + ChunkType: "function", SymbolName: "N", Language: "go"}, + }, + [][]float32{l2([]float32{1.0, 0.0, 0.0, 0.0})}, + ) + + // A four-chunk in-memory workspace is nowhere near two seconds. + doJSON(t, router, http.MethodGet, "/api/v1/workspaces/"+wsID+"/search?q=near", nil) + if strings.Contains(logs.String(), "slow workspace search") { + t.Errorf("a fast query wrote a slow-query line:\n%s", logs.String()) + } + + // Same query, threshold dropped so every query counts as slow. + restore := slowWorkspaceQuery + slowWorkspaceQuery = 0 + t.Cleanup(func() { slowWorkspaceQuery = restore }) + + logs.Reset() + rr := doJSON(t, router, http.MethodGet, "/api/v1/workspaces/"+wsID+"/search?q=near", nil) + line := "" + for _, l := range strings.Split(strings.TrimSpace(logs.String()), "\n") { + if strings.Contains(l, "slow workspace search") { + line = l + break + } + } + if line == "" { + t.Fatalf("a slow query wrote no line:\n%s", logs.String()) + } + // The line is the whole artefact — if it omits a phase, the phase is + // invisible in production no matter how carefully it was measured. + for _, field := range append([]string{"workspace_id", "query_len"}, timingFields...) { + if !strings.Contains(line, `"`+field+`"`) { + t.Errorf("slow-query line is missing %q: %s", field, line) + } + } + // The two gates are independent: crossing the log threshold must not + // start attaching the block to responses nobody asked for. + var raw map[string]any + if err := json.Unmarshal(rr.Body.Bytes(), &raw); err != nil { + t.Fatalf("decode: %v", err) + } + if _, present := raw["timings"]; present { + t.Errorf("a slow query attached timings to a response that did not ask: %v", raw["timings"]) + } +} + +// TestWorkspaceSearch_ReturnedCountIgnoresThePanelCap is the regression test +// for the counter these timings exist to feed. +// +// projects_scanned:projects_returned is meant to say how much of the fan-out's +// work was discarded — the premise of routing the fan-out at all. Counting the +// projects the caller was SHOWN instead pegs that ratio to top_projects +// (default 10), so a workspace where 12 repos are relevant and one where 40 +// are would both report the same ratio, and both would report it unchanged if +// the threshold stopped rejecting anything at all. The number the caller saw +// is a real but different question, and gets its own field. +func TestWorkspaceSearch_ReturnedCountIgnoresThePanelCap(t *testing.T) { + d, err := dbOpenMemory(t) + if err != nil { + t.Fatalf("open db: %v", err) + } + vs := openTestVectorStore(t) + query := l2([]float32{1, 0, 0, 0}) + router := newSearchRouter(t, d, vs, fixedEmbedder{q: query}) + wsID := createWS(t, router, "panelcap") + + // More relevant repos than the panel holds. Every one is a near-exact + // match, so none of them can be dropped by the relevance threshold and + // the only thing that can shrink the count is the cap. + const repos = 14 + for i := 0; i < repos; i++ { + seedRepoWithChunks(t, d, vs, wsID, + fmt.Sprintf("github.com/o/r%02d@main", i), + []vectorstore.Chunk{ + {Content: "near", FilePath: "n.go", StartLine: 1, EndLine: 9, + ChunkType: "function", SymbolName: "N", Language: "go"}, + }, + [][]float32{l2([]float32{1.0, float32(i) / 1000, 0.0, 0.0})}, + ) + } + + rr := doJSON(t, router, http.MethodGet, + "/api/v1/workspaces/"+wsID+"/search?q=near&timings=true", nil) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d (%s)", rr.Code, rr.Body.String()) + } + var body struct { + Projects []map[string]any `json:"projects"` + Timings map[string]json.Number `json:"timings"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v", err) + } + num := func(k string) int64 { + n, err := body.Timings[k].Int64() + if err != nil { + t.Fatalf("%s is not an integer: %v", k, err) + } + return n + } + + if got := num("projects_scanned"); got != repos { + t.Errorf("projects_scanned = %d, want %d", got, repos) + } + if got := num("projects_returned"); got != repos { + t.Errorf("projects_returned = %d, want %d — every repo clears the "+ + "threshold, so this must not be capped by top_projects", got, repos) + } + // The panel is capped, and its counter has to agree with the array the + // caller actually received. + panel := num("projects_in_panel") + if panel != int64(len(body.Projects)) { + t.Errorf("projects_in_panel = %d but the response carries %d projects", + panel, len(body.Projects)) + } + if panel >= repos { + t.Errorf("projects_in_panel = %d — expected the default top_projects "+ + "cap to bite with %d relevant repos", panel, repos) + } + assertCounterOrder(t, num) +} + +// TestWorkspaceSearch_LogsSlowQueriesThatSearchedNothing covers the early +// returns. A workspace with no queryable project never reaches the fan-out, +// but the query embedding has already been paid for by then — and a hung +// embedding provider is exactly the failure the slow-query line exists to +// catch. Silence on those paths would hide the one phase that ran. +func TestWorkspaceSearch_LogsSlowQueriesThatSearchedNothing(t *testing.T) { + d, err := dbOpenMemory(t) + if err != nil { + t.Fatalf("open db: %v", err) + } + vs := openTestVectorStore(t) + var logs bytes.Buffer + router := newSearchRouterWithLogger(t, d, vs, + fixedEmbedder{q: l2([]float32{1, 0, 0, 0})}, + slog.New(slog.NewJSONHandler(&logs, &slog.HandlerOptions{Level: slog.LevelInfo}))) + wsID := createWS(t, router, "emptyslow") + + restore := slowWorkspaceQuery + slowWorkspaceQuery = 0 + t.Cleanup(func() { slowWorkspaceQuery = restore }) + + rr := doJSON(t, router, http.MethodGet, + "/api/v1/workspaces/"+wsID+"/search?q=anything&timings=true", nil) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d (%s)", rr.Code, rr.Body.String()) + } + if !strings.Contains(logs.String(), "slow workspace search") { + t.Errorf("an early return skipped the slow-query line:\n%s", logs.String()) + } + // And the response still carries nothing, because nothing was searched — + // even though this caller did ask for timings. + var raw map[string]any + if err := json.Unmarshal(rr.Body.Bytes(), &raw); err != nil { + t.Fatalf("decode: %v", err) + } + if _, present := raw["timings"]; present { + t.Errorf("a search that never ran reported timings: %v", raw["timings"]) + } +} + +// TestWorkspaceSearch_BM25HitsStayInTheirOwnProject is the handler-level guard +// on the partition. BM25 is now one workspace-wide query whose rows are split +// back out per project; a mis-keyed split would hand one repo another repo's +// hits, and the symptom would be a plausible-looking result set rather than an +// error. The dense side cannot mask it here: only one repo is near the query +// vector, so any BM25-driven repo in the panel had to come from the split. +func TestWorkspaceSearch_BM25HitsStayInTheirOwnProject(t *testing.T) { + d, err := dbOpenMemory(t) + if err != nil { + t.Fatalf("open db: %v", err) + } + vs := openTestVectorStore(t) + query := l2([]float32{1, 0, 0, 0}) + router := newSearchRouter(t, d, vs, fixedEmbedder{q: query}) + wsID := createWS(t, router, "partition") + + // Three repos, each with a literal token only it contains, so BM25's + // answer per repo is unambiguous and checkable. + repos := []struct{ path, token, file string }{ + {"github.com/o/alpha@main", "ZZALPHAZZ", "a.go"}, + {"github.com/o/beta@main", "ZZBETAZZ", "b.go"}, + {"github.com/o/gamma@main", "ZZGAMMAZZ", "c.go"}, + } + for i, r := range repos { + seedRepoWithChunks(t, d, vs, wsID, r.path, + []vectorstore.Chunk{ + {Content: "func handle() { /* " + r.token + " */ }", FilePath: r.file, + StartLine: 1, EndLine: 9, ChunkType: "function", + SymbolName: "handle", Language: "go"}, + }, + // Only alpha is anywhere near the query vector. + [][]float32{l2([]float32{1.0, float32(i), 0.0, 0.0})}, + ) + } + + // Ask for beta's token. Beta must be the repo carrying the hit. + rr := doJSON(t, router, http.MethodGet, + "/api/v1/workspaces/"+wsID+"/search?q=ZZBETAZZ&min_score=0", nil) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d (%s)", rr.Code, rr.Body.String()) + } + var body struct { + Projects []struct { + ProjectPath string `json:"project_path"` + BM25Score float64 `json:"bm25_score"` + } `json:"projects"` + Chunks []struct { + ProjectPath string `json:"project_path"` + FilePath string `json:"file_path"` + } `json:"chunks"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v", err) + } + + var betaBM25, otherBM25 float64 + for _, p := range body.Projects { + if p.ProjectPath == "github.com/o/beta@main" { + betaBM25 = p.BM25Score + continue + } + if p.BM25Score > otherBM25 { + otherBM25 = p.BM25Score + } + } + if betaBM25 <= 0 { + t.Errorf("beta has bm25_score %v — its own token did not reach it: %+v", betaBM25, body.Projects) + } + if otherBM25 != 0 { + t.Errorf("a repo that contains none of the query's tokens has bm25_score %v — "+ + "the partition leaked: %+v", otherBM25, body.Projects) + } + for _, c := range body.Chunks { + if c.FilePath == "b.go" && c.ProjectPath != "github.com/o/beta@main" { + t.Errorf("beta's chunk is attributed to %s", c.ProjectPath) + } + } +} + +// TestWorkspaceSearch_SurvivesBM25Failure covers the blast radius this change +// creates. BM25 used to fail per project; now one failing query costs every +// project its sparse signal at once. The fallback has to be the same one a +// pre-FTS install already lives with — dense-only results, no failed_repos, +// no 500 — because the alternative is that one broken table takes down +// workspace search entirely. +func TestWorkspaceSearch_SurvivesBM25Failure(t *testing.T) { + d, err := dbOpenMemory(t) + if err != nil { + t.Fatalf("open db: %v", err) + } + vs := openTestVectorStore(t) + query := l2([]float32{1, 0, 0, 0}) + router := newSearchRouter(t, d, vs, fixedEmbedder{q: query}) + wsID := createWS(t, router, "nofts") + seedRepoWithChunks(t, d, vs, wsID, "github.com/o/near@main", + []vectorstore.Chunk{ + {Content: "func handle() {}", FilePath: "n.go", StartLine: 1, EndLine: 9, + ChunkType: "function", SymbolName: "handle", Language: "go"}, + }, + [][]float32{l2([]float32{1.0, 0.0, 0.0, 0.0})}, + ) + + // Break only the FTS side. chunks_meta survives, so the stale-FTS probe + // still answers and the repo is not reported as needing a reindex — the + // failure is the query, not the data. + if _, err := d.Exec(`DROP TABLE chunks_fts`); err != nil { + t.Fatalf("drop chunks_fts: %v", err) + } + + rr := doJSON(t, router, http.MethodGet, + "/api/v1/workspaces/"+wsID+"/search?q=handle", nil) + if rr.Code != http.StatusOK { + t.Fatalf("expected 200 despite the BM25 failure, got %d (%s)", rr.Code, rr.Body.String()) + } + var body struct { + Status string `json:"status"` + Projects []struct { + ProjectPath string `json:"project_path"` + BM25Score float64 `json:"bm25_score"` + DenseScore float64 `json:"dense_score"` + } `json:"projects"` + Chunks []map[string]any `json:"chunks"` + FailedRepos []map[string]any `json:"failed_repos"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v", err) + } + if len(body.Chunks) == 0 { + t.Errorf("no chunks — dense should still answer with BM25 broken: %s", rr.Body.String()) + } + if len(body.FailedRepos) != 0 { + t.Errorf("a BM25 failure marked repos as failed: %v", body.FailedRepos) + } + if len(body.Projects) != 1 { + t.Fatalf("expected the one repo in the panel, got %+v", body.Projects) + } + if body.Projects[0].BM25Score != 0 { + t.Errorf("bm25_score is %v with no FTS table at all", body.Projects[0].BM25Score) + } + if body.Projects[0].DenseScore <= 0 { + t.Errorf("dense_score is %v — the dense side should be unaffected", body.Projects[0].DenseScore) + } +} + +// TestFuseRRF_IsDeterministicAcrossTiedChunks pins the total order in fuseRRF. +// +// The function builds its output by ranging over a map, and Go randomises map +// iteration on purpose. Sorting by RRF alone leaves chunks with equal scores in +// whatever order the map happened to yield, and sort.SliceStable then preserves +// that randomness faithfully. Equal RRF is the common case, not a corner one: a +// chunk found only by dense at rank r and a chunk found only by BM25 at rank r +// score identically by construction. +// +// The symptom was invisible from the projects panel — project scores do not +// depend on chunk order, so the panel looked stable while rank 0 of the chunk +// list changed between consecutive calls on the same process and binary. +// +// Repeats matter here: with N tied entries a single run has a 1/N! chance of +// looking sorted by accident, so one call proves nothing. +func TestFuseRRF_IsDeterministicAcrossTiedChunks(t *testing.T) { + mk := func(project, file string, line int) workspaceSearchChunkPayload { + return workspaceSearchChunkPayload{ + ProjectPath: project, FilePath: file, + StartLine: line, EndLine: line + 5, + } + } + // Disjoint lists of the same length: every dense chunk at rank r ties + // exactly with the BM25 chunk at rank r, so every pair is a tie. + var dense, bm25 []workspaceSearchChunkPayload + for i := 0; i < 12; i++ { + dense = append(dense, mk("p", fmt.Sprintf("d%02d.go", i), 1+i*10)) + bm25 = append(bm25, mk("p", fmt.Sprintf("b%02d.go", i), 1+i*10)) + } + + first := fuseRRF(dense, bm25) + if len(first) != len(dense)+len(bm25) { + t.Fatalf("expected %d fused chunks, got %d", len(dense)+len(bm25), len(first)) + } + for run := 0; run < 20; run++ { + got := fuseRRF(dense, bm25) + for i := range got { + if got[i] != first[i] { + t.Fatalf("run %d differs at rank %d: %s/%d vs %s/%d — fusion is "+ + "not deterministic across tied chunks", + run, i, got[i].FilePath, got[i].StartLine, + first[i].FilePath, first[i].StartLine) + } + } + } + + // And the ordering must still be by RRF first: a chunk in BOTH lists + // outranks any chunk in only one, whatever its key sorts like. + both := mk("p", "zzz_last_alphabetically.go", 999) + withShared := fuseRRF(append([]workspaceSearchChunkPayload{both}, dense...), + append([]workspaceSearchChunkPayload{both}, bm25...)) + if withShared[0] != both { + t.Errorf("the chunk present in both lists is not first: got %s", withShared[0].FilePath) + } +} + +// TestSortPanel_OrdersOnRawCandidacyNotTheRoundedCopy covers the last place in +// this handler where an arbitrary tiebreak decided user-visible output. +// +// project_score ships rounded to four decimals. Sorting THAT value invents ties +// between projects that are not actually tied, and the caller truncates to +// top_projects right afterwards — so the invented tie decides which repo is +// shown and which is dropped entirely. Before this, the winner was whichever +// project came first in workspace membership order (added_at DESC), i.e. +// insertion history. +// +// The two projects here differ by 5e-6 in candidacy: far below round4's +// resolution, so both display as 0.5000, and the input order is deliberately +// the reverse of the correct one. Naming the stronger project last +// alphabetically is what makes this test able to tell "sorted on the raw value" +// apart from "fell back to the path tiebreak" — both alternatives would put +// "aaa" first. +func TestSortPanel_OrdersOnRawCandidacyNotTheRoundedCopy(t *testing.T) { + surviving := []projectHits{ + {ProjectPath: "github.com/o/aaa@main", Candidacy: 0.4999950}, + {ProjectPath: "github.com/o/zzz@main", Candidacy: 0.5000000}, + } + if round4(surviving[0].Candidacy) != round4(surviving[1].Candidacy) { + t.Fatalf("test premise broken: %v and %v do not round to the same value", + round4(surviving[0].Candidacy), round4(surviving[1].Candidacy)) + } + + sortPanel(surviving) + if surviving[0].ProjectPath != "github.com/o/zzz@main" { + t.Errorf("panel led with %s — the stronger project lost to a tie that "+ + "only exists after rounding", surviving[0].ProjectPath) + } +} + +// TestSortPanel_BreaksGenuineTiesByPath is the other half: when the candidacy +// really is equal, the order still has to be a function of the query rather +// than of the order projects happened to arrive in. +func TestSortPanel_BreaksGenuineTiesByPath(t *testing.T) { + mk := func(paths ...string) []projectHits { + out := make([]projectHits, 0, len(paths)) + for _, p := range paths { + out = append(out, projectHits{ProjectPath: p, Candidacy: 0.5}) + } + return out + } + want := []string{"github.com/o/aaa@main", "github.com/o/mmm@main", "github.com/o/zzz@main"} + for _, input := range [][]string{ + {"github.com/o/zzz@main", "github.com/o/mmm@main", "github.com/o/aaa@main"}, + {"github.com/o/mmm@main", "github.com/o/aaa@main", "github.com/o/zzz@main"}, + {"github.com/o/aaa@main", "github.com/o/mmm@main", "github.com/o/zzz@main"}, + } { + got := mk(input...) + sortPanel(got) + for i := range want { + if got[i].ProjectPath != want[i] { + t.Errorf("input %v -> position %d is %s, want %s", + input, i, got[i].ProjectPath, want[i]) + break + } + } + } +} diff --git a/server/internal/indexer/indexer.go b/server/internal/indexer/indexer.go index 03a17ad5..4608f01d 100644 --- a/server/internal/indexer/indexer.go +++ b/server/internal/indexer/indexer.go @@ -23,6 +23,7 @@ import ( "github.com/dvcdsys/code-index/server/internal/embeddings" "github.com/dvcdsys/code-index/server/internal/langdetect" "github.com/dvcdsys/code-index/server/internal/symbolindex" + "github.com/dvcdsys/code-index/server/internal/tokenizer" "github.com/dvcdsys/code-index/server/internal/vectorstore" ) @@ -112,6 +113,17 @@ type TokenAwareEmbedder interface { TokenizeAndEmbed(ctx context.Context, texts []string) ([][]float32, error) } +// TokenBudgetSource is the capability of telling the chunker what a chunk +// costs in the active model's tokens. Named rather than asserted inline so a +// rename of TokenBudget is a compile error somewhere instead of a silent +// return to byte-sized chunking everywhere. +// +// *embeddings.Service satisfies it; test fakes generally do not, and get the +// byte path. +type TokenBudgetSource interface { + TokenBudget() tokenizer.Budget +} + // Service owns sessions and wires dependencies for the three-phase protocol. type Service struct { db *sql.DB @@ -141,6 +153,10 @@ type Service struct { // reindexed under the new format. embedIncludePath bool + // maxChunkTokens is the per-chunk token target (CIX_MAX_CHUNK_TOKENS). + // 0 means the chunker's own default. + maxChunkTokens int + // embeddingModel is the active embedding model identifier persisted on // projects.indexed_with_model at FinishIndexing. Set via // SetEmbeddingModel from main; empty string keeps the column NULL so @@ -208,6 +224,12 @@ func (s *Service) SetEmbedIncludePath(v bool) { s.embedIncludePath = v } +// SetMaxChunkTokens sets the per-chunk token target used when the active +// embedding provider can count tokens exactly. +func (s *Service) SetMaxChunkTokens(n int) { + s.maxChunkTokens = n +} + // SetEmbeddingModel records the model identifier the indexer will write to // projects.indexed_with_model at FinishIndexing. Called from main once the // runtime config is resolved; empty string disables the write (the column @@ -665,6 +687,7 @@ func (s *Service) ProcessFilesStreaming( // is CPU-local and cheap, so it stays sequential to keep progress-event // order; the expensive embed work is parallelised in stage 2. prep := make([]*preparedFile, 0, len(files)) + budgetSrc, _ := s.emb.(TokenBudgetSource) for fi, fp := range files { // file_started — emit even for files we'll skip below, so the client // counter advances monotonically and rendering stays aligned with N. @@ -705,7 +728,14 @@ func (s *Service) ProcessFilesStreaming( language = "text" } - chunks, refs, err := chunker.ChunkFile(fp.Path, fp.Content, language, 0) + // The budget is re-read per file: a provider swap between files is + // legitimate, mixing two models' limits inside one file's chunks is + // not. The type assertion itself is hoisted out of the loop. + var budget tokenizer.Budget + if budgetSrc != nil { + budget = budgetSrc.TokenBudget() + } + chunks, refs, err := chunker.ChunkFileTokens(fp.Path, fp.Content, language, 0, budget, s.maxChunkTokens) if err != nil { s.logger.Warn("indexer: chunk file failed", "path", fp.Path, "err", err) progressSend(progress, ProgressEvent{ diff --git a/server/internal/maintenance/dirsize_test.go b/server/internal/maintenance/dirsize_test.go new file mode 100644 index 00000000..e5e34e3a --- /dev/null +++ b/server/internal/maintenance/dirsize_test.go @@ -0,0 +1,81 @@ +package maintenance + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" +) + +func TestDirSizeBytes_SumsRegularFiles(t *testing.T) { + dir := t.TempDir() + writeFileOfSize(t, filepath.Join(dir, "a"), 100) + writeFileOfSize(t, filepath.Join(dir, "sub", "b"), 50) + + n, ok := DirSizeBytes(context.Background(), dir) + if !ok { + t.Fatal("ok = false on a readable tree") + } + if n != 150 { + t.Errorf("total = %d, want 150", n) + } +} + +func TestDirSizeBytes_MissingRoot_ReportsNotOK(t *testing.T) { + n, ok := DirSizeBytes(context.Background(), filepath.Join(t.TempDir(), "nope")) + if ok { + t.Error("ok = true on a missing directory — 'unreadable' and 'empty' must stay distinguishable") + } + if n != 0 { + t.Errorf("total = %d, want 0", n) + } +} + +func TestDirSizeBytes_UnreadableSubtree_ReturnsPartial(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("root ignores directory permissions") + } + dir := t.TempDir() + writeFileOfSize(t, filepath.Join(dir, "a"), 100) + locked := filepath.Join(dir, "locked") + writeFileOfSize(t, filepath.Join(locked, "hidden"), 999) + if err := os.Chmod(locked, 0o000); err != nil { + t.Fatalf("chmod: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(locked, 0o755) }) + + n, skipped, ok := dirSizeDetail(context.Background(), dir) + if !ok { + t.Fatal("ok = false — one unreadable subtree must not throw the whole number away") + } + if n != 100 { + t.Errorf("total = %d, want 100 (the readable part)", n) + } + if skipped == 0 { + t.Error("skipped = 0 — the undercount must be visible so DiskUsage can flag it as partial") + } +} + +func TestDirSizeBytes_CancelledContext_ReportsNotOK(t *testing.T) { + dir := t.TempDir() + // Enough entries to guarantee the every-512-entries context check fires. + for i := range 600 { + writeFileOfSize(t, filepath.Join(dir, fmt.Sprintf("f%04d", i)), 1) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, ok := DirSizeBytes(ctx, dir); ok { + t.Error("ok = true on a cancelled context, want false so callers omit the number") + } +} + +func writeFileOfSize(t *testing.T, path string, size int) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, make([]byte, size), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} diff --git a/server/internal/maintenance/maintenance.go b/server/internal/maintenance/maintenance.go index f82c1220..26a8948a 100644 --- a/server/internal/maintenance/maintenance.go +++ b/server/internal/maintenance/maintenance.go @@ -217,9 +217,15 @@ type Analysis struct { Warnings []string `json:"warnings,omitempty"` } -// DirSizeBytes walks dir and sums regular-file sizes. Returns (0,false) on any -// error (missing dir, permission, cancelled context) so callers can omit the -// number rather than report a misleading 0. +// DirSizeBytes walks dir and sums regular-file sizes. An unreadable entry +// inside the tree is skipped and the rest still counts — a partial number +// beats no number on a tree of hundreds of thousands of git objects, where a +// single bad directory used to make the whole "Cloned repositories" row +// vanish. Returns (partial, false) only when nothing trustworthy could be +// produced: the root itself is missing/unreadable (so "unreadable" and +// "empty" stay distinguishable) or the context was cancelled mid-walk. +// Callers that need to tell a complete sum from an undercount use +// dirSizeDetail, which also reports how many entries were skipped. // // The context is checked every so many entries: on a vector store that is one // file per document these walks visit hundreds of thousands of entries, and a @@ -229,11 +235,23 @@ type Analysis struct { // Lives here rather than in httpapi because both the resource endpoints and // the project-detail card need it and there must be exactly one copy. func DirSizeBytes(ctx context.Context, dir string) (int64, bool) { - var total int64 + n, _, ok := dirSizeDetail(ctx, dir) + return n, ok +} + +func dirSizeDetail(ctx context.Context, dir string) (total int64, skipped int, ok bool) { var seen int - walkErr := filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error { + walkErr := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { if err != nil { - return err + // Root failure means there is nothing to report; anything + // deeper is one bad subtree — count it as skipped and keep + // walking. (WalkDir already skips the children of a directory + // it could not read.) + if path == dir { + return err + } + skipped++ + return nil } // Checking every entry would make ctx.Err() a meaningful share of the // walk's cost; every 512 keeps cancellation prompt for free. @@ -253,9 +271,9 @@ func DirSizeBytes(ctx context.Context, dir string) (int64, bool) { return nil }) if walkErr != nil { - return 0, false + return total, skipped, false } - return total, true + return total, skipped, true } // dirSizeOrZero is the convenience form for places that already know the diff --git a/server/internal/maintenance/usage.go b/server/internal/maintenance/usage.go index 07a621b0..af1b86ea 100644 --- a/server/internal/maintenance/usage.go +++ b/server/internal/maintenance/usage.go @@ -23,7 +23,12 @@ type DiskUsage struct { Exists bool `json:"exists"` // UsedBytes is omitted rather than zeroed when the tree could not be // walked, so "unreadable" and "empty" stay distinguishable. - UsedBytes *int64 `json:"used_bytes,omitempty"` + UsedBytes *int64 `json:"used_bytes,omitempty"` + // Partial marks a UsedBytes that undercounts: some entries could not be + // read and were skipped. Without this flag a single root-owned checkout + // makes the row show a confident wrong number — exactly what an + // operator chasing disk growth must not rule out. + Partial bool `json:"partial,omitempty"` FSTotalBytes *int64 `json:"fs_total_bytes,omitempty"` FSFreeBytes *int64 `json:"fs_free_bytes,omitempty"` } @@ -125,27 +130,32 @@ func (s *Service) computeUsage(ctx context.Context) Usage { // pre-migration gob files are reported (and reclaimed) through the // abandoned-namespace category instead. if cfg.VectorsDir != "" { - out.Disks = append(out.Disks, walkedDisk(ctx, DiskChroma, "Vector store", cfg.VectorsDir)) + out.Disks = append(out.Disks, s.walkedDisk(ctx, DiskChroma, "Vector store", cfg.VectorsDir)) } else if cfg.ChromaPersistDir != "" { - out.Disks = append(out.Disks, walkedDisk(ctx, DiskChroma, "Vector store", cfg.ChromaPersistDir)) + out.Disks = append(out.Disks, s.walkedDisk(ctx, DiskChroma, "Vector store", cfg.ChromaPersistDir)) } if root := s.reposRoot(); root != "" { - out.Disks = append(out.Disks, walkedDisk(ctx, DiskRepos, "Cloned repositories", root)) + out.Disks = append(out.Disks, s.walkedDisk(ctx, DiskRepos, "Cloned repositories", root)) } if dir := s.activeGGUFCacheDir(); dir != "" { - out.Disks = append(out.Disks, walkedDisk(ctx, DiskGGUF, "Model cache", dir)) + out.Disks = append(out.Disks, s.walkedDisk(ctx, DiskGGUF, "Model cache", dir)) } return out } -func walkedDisk(ctx context.Context, id, label, path string) DiskUsage { +func (s *Service) walkedDisk(ctx context.Context, id, label, path string) DiskUsage { d := DiskUsage{ID: id, Label: label, Path: path} if _, err := os.Stat(path); err == nil { d.Exists = true if !ctxDone(ctx) { - if n, ok := DirSizeBytes(ctx, path); ok { + if n, skipped, ok := dirSizeDetail(ctx, path); ok { d.UsedBytes = &n + if skipped > 0 { + d.Partial = true + s.d.Logger.Warn("maintenance: disk usage undercounts — entries were unreadable", + "disk", id, "path", path, "skipped_entries", skipped) + } } } } diff --git a/server/internal/repocloner/compact.go b/server/internal/repocloner/compact.go new file mode 100644 index 00000000..c2357ca0 --- /dev/null +++ b/server/internal/repocloner/compact.go @@ -0,0 +1,476 @@ +package repocloner + +// In-process object-store compaction for shallow checkouts. +// +// Why it exists: the update path is fetch(Depth:1) + hard reset. go-git +// persists ONE NEW PACKFILE per fetch — and each of those packs is a +// near-full snapshot of the tree, not a delta — while the reset makes the +// previously fetched snapshot unreachable. go-git has no gc and the +// distroless runtime has no git binary, so without intervention the object +// store grows with every upstream push, forever (this is what took a ~4.5 GB +// production fleet of checkouts to 76 GB). On top of that, go-git's clone +// default is Tags:AllTags, so a day-zero clone of a tag-rich repo carries a +// full shallow snapshot PER TAG (spring-boot: 391 tags → a 102 MB store for +// a 39 MB worktree) that cix, which indexes exactly one branch, never reads. +// +// Compaction rewrites the store down to what the server actually uses: +// it walks the objects reachable from the non-tag refs (honouring +// .git/shallow graft points exactly like git does) plus any explicitly +// protected commits, encodes that set into one new pack, drops refs/tags/*, +// deletes the old packs and loose objects, and rewrites .git/shallow to the +// entries that still exist. `git fsck --strict` is clean afterwards and the +// worktree is untouched — validated byte-for-byte against full-history +// canonical clones on 45 real checkouts (spring-boot, grafana, …) by the PoC +// on branch poc/gc-compaction (server/cmd/gc-poc). +// +// Cost model, measured on those 45 checkouts: time is linear — +// ~0.2–1.4 ms CPU per reachable object plus ~0.2 s per emitted GB (zlib); +// memory is linear in the SNAPSHOT size (not the store size) at roughly 3× +// the uncompressed content, because go-git's packfile encoder materialises +// object data. A typical 60 MB checkout compacts in single-digit seconds +// within a few hundred MB of transient heap; MaybeCompact's global gate +// keeps concurrent clone jobs from stacking those peaks. +// +// Crash-safety ordering inside compactCheckout: the new pack is durable +// before anything is deleted, and the tag refs are removed before the old +// packs go away (so a ref never dangles over a missing object). A crash at +// any point leaves either extra packs or dropped-tags-with-bloat — both +// states re-trigger needsCompaction (pack count and store/worktree ratio +// respectively) and heal on the next update. +// +// The delta window is 0 on purpose: after tags are dropped the reachable set +// is essentially a single snapshot, and the PoC measured window=10 at −17% +// pack size for 2.9× the CPU. + +import ( + "context" + "errors" + "fmt" + "io/fs" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/filemode" + "github.com/go-git/go-git/v5/plumbing/format/packfile" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/storer" + "github.com/go-git/go-git/v5/storage" +) + +// compactPackThreshold is how many packfiles a checkout may accumulate +// before the next update compacts the store. Each fetch adds one +// snapshot-sized pack, so the steady-state disk overhead between compactions +// is bounded by (threshold-1) worktree-sized packs, and the compaction cost +// is amortised over that many pushes. +const compactPackThreshold = 4 + +// Ratio backstop: a store this many times larger than the worktree it +// serves (and above the floor) is carrying dead weight regardless of pack +// count. This is what re-arms cleanup for a checkout whose tag refs were +// dropped by a compaction that crashed before deleting the old packs — pack +// count alone would never fire again on a quiet repo. A healthy compacted +// store is zlib-compressed and smaller than its worktree, so legitimate +// checkouts sit far below 2×. +const ( + compactRatioTrigger = 2 + compactRatioFloor = 1 << 20 // ignore ratio noise on tiny stores +) + +// compactMu serialises compactions across concurrent clone jobs. The +// transient heap of one compaction is ~3× the repo's uncompressed snapshot; +// letting several worker goroutines pay that simultaneously is how an 8 GB +// host gets OOM-killed. MaybeCompact acquires it BEFORE the caller-supplied +// per-repo write lock, so a job queued on this mutex never stalls another +// repo's readers. +var compactMu sync.Mutex + +const tagRefPrefix = "refs/tags/" + +// CompactStats reports what one compaction did. Purely informational — +// callers log it. +type CompactStats struct { + ObjectsBefore int64 // .git/objects bytes before + ObjectsAfter int64 // .git/objects bytes after + Reachable int // objects written to the new pack + PacksDeleted int + LoosePruned int + TagRefsDropped int + Duration time.Duration +} + +// MaybeCompact compacts dir's object store when it needs it (see +// needsCompaction) and reports what it did; (nil, nil) means "nothing to +// do". withWrite, when non-nil, must serialise the on-disk mutation against +// concurrent readers of this checkout (repojobs passes RepoLocks.WithWrite); +// it is acquired AFTER the global compaction gate, so waiting for another +// repo's compaction never happens while holding this repo's lock. +// +// A compaction error leaves the checkout exactly as the preceding update +// left it — valid — so callers must NOT treat it as reason to discard the +// checkout: log it and move on; the trigger re-fires on the next update. +// +// protect lists commit SHAs that must survive even though no ref points at +// them: git_repos.indexed_sha (the base of the next incremental tree-diff) +// and the pre-fetch HEAD (the target of a possibly still-queued index job). +// Empty strings and SHAs absent from the store are skipped. +func MaybeCompact(ctx context.Context, dir string, withWrite func(func() error) error, protect ...string) (*CompactStats, error) { + if ctx.Err() != nil || !needsCompaction(dir) { + return nil, nil + } + var hashes []plumbing.Hash + for _, s := range protect { + if s = strings.TrimSpace(s); s != "" { + hashes = append(hashes, plumbing.NewHash(s)) + } + } + + compactMu.Lock() + defer compactMu.Unlock() + // The queue on compactMu can be long on upgrade day; don't start work + // for a request that is already gone. + if err := ctx.Err(); err != nil { + return nil, err + } + + var st CompactStats + run := func() error { + var err error + st, err = compactCheckout(ctx, dir, hashes...) + return err + } + var err error + if withWrite != nil { + err = withWrite(run) + } else { + err = run() + } + if err != nil { + return nil, err + } + return &st, nil +} + +// needsCompaction reports whether the checkout's object store warrants a +// rewrite. Three triggers: +// +// - packfileCount ≥ compactPackThreshold: accumulated fetch packs. +// - tag refs present: snapshots left behind by pre-NoTags server versions. +// Their objects dominate the store, and with Tags:NoTags on every fetch +// they will not come back — this makes the first post-upgrade update of +// every existing checkout clean it, with no separate migration. +// - store ≥ compactRatioTrigger × worktree (packs ≥ 2 only): the backstop +// that re-arms cleanup after a crash mid-compaction dropped the tag refs +// without reclaiming their objects. Gated on pack count so the worktree +// walk is not paid on the common single-pack steady state. +func needsCompaction(dir string) bool { + packs := packfileCount(dir) + if packs >= compactPackThreshold { + return true + } + repo, err := git.PlainOpen(dir) + if err != nil { + return false + } + refs, err := repo.References() + if err != nil { + return false + } + found := false + _ = refs.ForEach(func(ref *plumbing.Reference) error { + if strings.HasPrefix(ref.Name().String(), tagRefPrefix) { + found = true + return storer.ErrStop + } + return nil + }) + refs.Close() + if found { + return true + } + if packs >= 2 { + if objects := objectsDirSize(dir); objects > compactRatioFloor { + return objects >= compactRatioTrigger*worktreeSize(dir) + } + } + return false +} + +// compactCheckout rewrites dir's object store down to the objects reachable +// from its non-tag references plus the protected commits, then drops the tag +// refs. See the package comment for the crash-safety ordering. The context +// is honoured between phases and inside the walk; cancellation before the +// deletion phase leaves the store untouched (bar an extra pack). +func compactCheckout(ctx context.Context, dir string, protect ...plumbing.Hash) (CompactStats, error) { + started := time.Now() + st := CompactStats{ObjectsBefore: objectsDirSize(dir)} + + repo, err := git.PlainOpen(dir) + if err != nil { + return st, fmt.Errorf("open: %w", err) + } + + // 1. One pass over the refs: non-tag hash refs seed the walk, tag refs + // are remembered for deletion later. cix serves exactly one branch; + // every tag is a whole retained snapshot the server never reads. + var roots []plumbing.Hash + var tagRefs []plumbing.ReferenceName + refs, err := repo.References() + if err != nil { + return st, err + } + err = refs.ForEach(func(ref *plumbing.Reference) error { + if strings.HasPrefix(ref.Name().String(), tagRefPrefix) { + tagRefs = append(tagRefs, ref.Name()) + return nil + } + if ref.Type() == plumbing.HashReference { + roots = append(roots, ref.Hash()) + } + return nil + }) + refs.Close() + if err != nil { + return st, err + } + for _, h := range protect { + if h.IsZero() { + continue + } + // Absent is fine (already gc'd away, or a bogus SHA — nothing to + // protect); any OTHER failure is a store problem the caller must + // hear about, not a silent loss of the diff base. + if _, gerr := object.GetObject(repo.Storer, h); gerr != nil { + if errors.Is(gerr, plumbing.ErrObjectNotFound) { + continue + } + return st, fmt.Errorf("probe protected %s: %w", h, gerr) + } + roots = append(roots, h) + } + + // 2. Reachability walk with git's shallow semantics. + shallowList, err := repo.Storer.Shallow() + if err != nil { + return st, fmt.Errorf("read shallow: %w", err) + } + shallowSet := make(map[plumbing.Hash]struct{}, len(shallowList)) + for _, h := range shallowList { + shallowSet[h] = struct{}{} + } + seen, err := walkReachable(ctx, repo.Storer, roots, shallowSet) + if err != nil { + return st, fmt.Errorf("reachability walk: %w", err) + } + st.Reachable = len(seen) + objs := make([]plumbing.Hash, 0, len(seen)) + for h := range seen { + objs = append(objs, h) + } + if err := ctx.Err(); err != nil { + return st, err + } + + // 3. Write the reachable set as one new pack. PackfileWriter lands it in + // objects/pack with a proper idx before we touch anything old. + pos, ok := repo.Storer.(storer.PackedObjectStorer) + if !ok { + return st, fmt.Errorf("storage does not support packed objects") + } + oldPacks, err := pos.ObjectPacks() + if err != nil { + return st, err + } + pfw, ok := repo.Storer.(storer.PackfileWriter) + if !ok { + return st, fmt.Errorf("storage does not support packfile writing") + } + wc, err := pfw.PackfileWriter() + if err != nil { + return st, err + } + enc := packfile.NewEncoder(wc, repo.Storer, false) + // Window 0: no delta search — see the package comment. + newPack, err := enc.Encode(objs, 0) + if cerr := wc.Close(); err == nil { + err = cerr + } + if err != nil { + return st, fmt.Errorf("encode pack: %w", err) + } + if err := ctx.Err(); err != nil { + return st, err + } + + // 4. The new pack is durable — now drop the tag refs, BEFORE the old + // packs: a tag ref must never outlive its objects (fetch negotiation + // advertises refs as haves), and the reverse crash window — tags + // gone, bloat still on disk — is re-armed by the ratio trigger. + for _, name := range tagRefs { + if err := repo.Storer.RemoveReference(name); err != nil { + return st, fmt.Errorf("remove tag ref %s: %w", name, err) + } + st.TagRefsDropped++ + } + + // 5. Delete the old packs. + for _, h := range oldPacks { + if h == newPack { + continue + } + if err := pos.DeleteOldObjectPackAndIndex(h, time.Time{}); err != nil { + return st, fmt.Errorf("delete pack %s: %w", h, err) + } + st.PacksDeleted++ + } + + // 6. Loose objects: everything reachable is in the new pack, so every + // loose object is redundant regardless of reachability. + if los, ok := repo.Storer.(storer.LooseObjectStorer); ok { + err = los.ForEachObjectHash(func(h plumbing.Hash) error { + if derr := los.DeleteLooseObject(h); derr != nil { + return derr + } + st.LoosePruned++ + return nil + }) + if err != nil { + return st, fmt.Errorf("prune loose: %w", err) + } + } + + // 7. .git/shallow gains one graft entry per fetch; entries whose commit + // was just dropped would make real git tooling error out ("did not + // find object for shallow …"), so keep only entries still present. + kept := shallowList[:0] + for _, h := range shallowList { + if _, reachable := seen[h]; reachable { + kept = append(kept, h) + } + } + if len(kept) != len(shallowList) { + if err := repo.Storer.SetShallow(kept); err != nil { + return st, fmt.Errorf("rewrite shallow: %w", err) + } + } + + st.ObjectsAfter = objectsDirSize(dir) + st.Duration = time.Since(started) + return st, nil +} + +// walkReachable collects every object reachable from roots. It is go-git's +// objectWalker (repository.go uses it for RepackObjects) reworked into an +// iterative worklist — recursion depth would otherwise equal the contiguous +// commit-chain length, and a full-history clone manually seeded into the +// repos dir must not be able to blow the goroutine stack — with three +// behavioural fixes, each of which real checkouts hit immediately: +// +// - commits listed in .git/shallow are graft points whose parents are +// never walked — git's own semantics. (Stock go-git follows ParentHashes +// unconditionally: it crashes on any multi-commit push fetched at +// Depth:1, and where the chain happens to be complete it retains every +// previously fetched snapshot forever.) +// - submodule (gitlink) tree entries are skipped — the hash is a commit in +// a different repository. (Stock go-git crashes.) +// - blobs reached as objects (via symlink and other non-regular-file tree +// entries) are accepted leaves. (Stock go-git errors "unknown object".) +func walkReachable(ctx context.Context, s storage.Storer, roots []plumbing.Hash, shallow map[plumbing.Hash]struct{}) (map[plumbing.Hash]struct{}, error) { + seen := make(map[plumbing.Hash]struct{}) + stack := make([]plumbing.Hash, len(roots)) + copy(stack, roots) + visited := 0 + for len(stack) > 0 { + hash := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if _, ok := seen[hash]; ok { + continue + } + // Keep cancellation prompt without paying ctx.Err() per object. + if visited++; visited%1024 == 0 { + if err := ctx.Err(); err != nil { + return nil, err + } + } + obj, err := object.GetObject(s, hash) + if err != nil { + return nil, fmt.Errorf("get object %s: %w", hash, err) + } + seen[hash] = struct{}{} + switch obj := obj.(type) { + case *object.Commit: + stack = append(stack, obj.TreeHash) + if _, grafted := shallow[obj.Hash]; grafted { + continue + } + for _, p := range obj.ParentHashes { + if _, ok := seen[p]; ok { + continue + } + // A parent this shallow store never fetched: boundary, + // not error. + if _, gerr := object.GetObject(s, p); gerr != nil { + if errors.Is(gerr, plumbing.ErrObjectNotFound) { + continue + } + return nil, fmt.Errorf("probe parent %s: %w", p, gerr) + } + stack = append(stack, p) + } + case *object.Tree: + for i := range obj.Entries { + e := obj.Entries[i] + if e.Mode == filemode.Submodule { + continue + } + if e.Mode|0o755 == filemode.Executable { // plain blob, any file mode + seen[e.Hash] = struct{}{} + continue + } + stack = append(stack, e.Hash) + } + case *object.Blob: + // Leaf. + case *object.Tag: + stack = append(stack, obj.Target) + default: + return nil, fmt.Errorf("unknown object type %T at %s", obj, hash) + } + } + return seen, nil +} + +// objectsDirSize sums .git/objects — a few packs plus loose files, so the +// walk is cheap. Best effort; 0 on error. +func objectsDirSize(dir string) int64 { + return treeSize(filepath.Join(dir, ".git", "objects"), false) +} + +// worktreeSize sums the checkout's payload, excluding .git. Only consulted +// by the ratio backstop, which is gated on pack count ≥ 2. +func worktreeSize(dir string) int64 { + return treeSize(dir, true) +} + +func treeSize(dir string, skipDotGit bool) int64 { + var total int64 + _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return nil + } + if d.IsDir() { + if skipDotGit && d.Name() == ".git" { + return filepath.SkipDir + } + return nil + } + if info, ierr := d.Info(); ierr == nil { + total += info.Size() + } + return nil + }) + return total +} diff --git a/server/internal/repocloner/repocloner.go b/server/internal/repocloner/repocloner.go index 870e9bea..bd35625e 100644 --- a/server/internal/repocloner/repocloner.go +++ b/server/internal/repocloner/repocloner.go @@ -12,8 +12,13 @@ // go-git into the binary keeps the runtime image untouched. // // What this package does: -// - Clone a branch (public OR PAT-authenticated) +// - Clone a branch (public OR PAT-authenticated), shallow and tag-free // - Fetch + reset to remote HEAD on subsequent runs +// - Compact the object store in place when fetch packs pile up or legacy +// tag snapshots are present (see compact.go) — go-git has no gc of its +// own and the distroless runtime has no git binary +// - Discard and re-clone a checkout whose local state is unusable +// (half-written clone, changed remote URL, failed compaction) // - Report the current HEAD SHA (for last_sha bookkeeping) // - Resolve a "github.com/owner/repo" + branch to a deterministic local // directory under DataDir/repos/{path_hash}/ @@ -42,6 +47,14 @@ import ( // short-circuit reindex on this. var ErrAlreadyUpToDate = errors.New("repo already up to date") +// errLocalState marks a reuse-path failure caused by the checkout on disk +// itself (half-written clone, missing refs, mismatched remote) rather than by +// the network or the remote. CloneOrFetch recovers from these by discarding +// the directory and cloning fresh; anything NOT wrapped with this sentinel +// (fetch/transport failures) is returned as-is so a network blip never costs +// an otherwise healthy clone. +var errLocalState = errors.New("local clone state unusable") + // CloneOptions parameterises a clone or fetch. type CloneOptions struct { // GitHubURL is the canonical HTTPS URL — "https://github.com/owner/repo" @@ -107,12 +120,38 @@ type Result struct { // matches the local HEAD before the fetch (i.e. nothing new). The // caller can skip enqueueing an index_repo job entirely. NoChanges bool + // RecloneReason is non-empty when an existing checkout was discarded + // and cloned fresh (unusable local state or a changed remote URL). + // Purely informational — callers log it so the operator can see why a + // fetch turned into a full clone. + RecloneReason string + // PrevHeadSHA is the commit that was on disk BEFORE this update moved + // the checkout (empty on a fresh clone). Callers pass it to + // MaybeCompact's protect list: it is the target of a possibly + // still-queued index job, and nothing else keeps it alive once the + // branch ref has moved on. + PrevHeadSHA string } // CloneOrFetch clones the repo when LocalDir is empty, otherwise fetches // + resets the local checkout to origin/{branch}. Returns the HEAD SHA // after the operation completes. // +// An existing checkout is discarded and cloned fresh (Result.RecloneReason +// says why) in two situations: its .git state is unusable — the half-clone +// a SIGKILL mid-clone leaves behind used to fail every retry forever — or +// its origin URL no longer matches the requested one (github_url changed). +// The local-state check is retried once first: transient filesystem pressure +// (EMFILE under concurrent jobs, a momentary EACCES) must not cost a +// multi-GB checkout, while a genuinely broken one fails the retry the same +// way. Fetch/transport failures are never grounds for a re-clone — a network +// blip must not cost a healthy clone (and force the full reindex that +// follows one). +// +// Compaction of the object store is NOT part of this call: callers run +// MaybeCompact afterwards, outside their per-repo write lock — see that +// function for why the locking is layered that way. +// // The caller is responsible for choosing a LocalDir that won't collide // across repos — typically `/repos//` keyed by // projects.path_hash (NOT the github URL, which can change with @@ -132,46 +171,94 @@ func CloneOrFetch(ctx context.Context, opts CloneOptions) (Result, error) { // First-time clone path: LocalDir is missing or empty. if needsClone(opts.LocalDir) { - if err := os.MkdirAll(opts.LocalDir, 0o755); err != nil { - return Result{}, fmt.Errorf("mkdir clone target: %w", err) - } - repo, err := git.PlainCloneContext(ctx, opts.LocalDir, false, &git.CloneOptions{ - URL: url, - Auth: auth, - ReferenceName: plumbing.NewBranchReferenceName(opts.Branch), - SingleBranch: true, - Depth: 1, // shallow — minimises bandwidth + disk - }) - if err != nil { - // Cleanup so the next retry isn't stuck with a half-clone. - _ = os.RemoveAll(opts.LocalDir) - return Result{}, fmt.Errorf("clone: %w", err) - } - head, err := repo.Head() - if err != nil { - return Result{}, fmt.Errorf("resolve HEAD: %w", err) - } - return Result{HeadSHA: head.Hash().String()}, nil + return cloneFresh(ctx, opts, url, auth) + } + + res, err := updateExisting(ctx, opts, url, auth) + if err != nil && errors.Is(err, errLocalState) && ctx.Err() == nil { + // One retry before concluding the state is structural: EMFILE/EIO + // class failures heal, a half-written clone fails identically. + res, err = updateExisting(ctx, opts, url, auth) + } + if err == nil { + return res, nil + } + // Only local-state failures are recoverable by re-cloning, and never on + // a dead context — a cancelled shutdown fetch is not evidence the + // checkout is bad. + if !errors.Is(err, errLocalState) || ctx.Err() != nil { + return Result{}, err + } + return reclone(ctx, opts, url, auth, err.Error()) +} + +// cloneFresh is the first-time clone into an empty or missing LocalDir. +func cloneFresh(ctx context.Context, opts CloneOptions, url string, auth *http.BasicAuth) (Result, error) { + if err := os.MkdirAll(opts.LocalDir, 0o755); err != nil { + return Result{}, fmt.Errorf("mkdir clone target: %w", err) + } + repo, err := git.PlainCloneContext(ctx, opts.LocalDir, false, &git.CloneOptions{ + URL: url, + Auth: auth, + ReferenceName: plumbing.NewBranchReferenceName(opts.Branch), + SingleBranch: true, + Depth: 1, // shallow — minimises bandwidth + disk + // go-git's clone default is AllTags, and on a shallow clone every + // tag arrives as a FULL tree snapshot (spring-boot's 391 tags cost + // a 102 MB store for a 39 MB worktree). cix indexes one branch and + // never reads tags. + Tags: git.NoTags, + }) + if err != nil { + // Cleanup so the next retry isn't stuck with a half-clone. + _ = os.RemoveAll(opts.LocalDir) + return Result{}, fmt.Errorf("clone: %w", err) + } + head, err := repo.Head() + if err != nil { + return Result{}, fmt.Errorf("resolve HEAD: %w", err) + } + return Result{HeadSHA: head.Hash().String()}, nil +} + +// reclone discards the existing checkout and clones fresh. The re-clone loses +// the old object store, so PrevIndexedSHA becomes unreachable and Changes +// stays nil — the caller lands in its reconcile path, which is the correct +// (and hash-gated, so cheap) recovery. +func reclone(ctx context.Context, opts CloneOptions, url string, auth *http.BasicAuth, reason string) (Result, error) { + if err := os.RemoveAll(opts.LocalDir); err != nil { + return Result{}, fmt.Errorf("remove stale clone at %s (%s): %w", opts.LocalDir, reason, err) + } + res, err := cloneFresh(ctx, opts, url, auth) + if err != nil { + return Result{}, fmt.Errorf("reclone (%s): %w", reason, err) } + res.RecloneReason = reason + return res, nil +} - // Reuse path: open the existing repo, ensure the remote matches, fetch, - // (optionally compute change set,) reset to origin/{branch}. +// updateExisting is the reuse path: open the existing repo, ensure the remote +// matches, fetch, (optionally compute change set,) reset to origin/{branch}. +// Failures rooted in the on-disk state are wrapped with errLocalState so the +// caller can recover by re-cloning; fetch failures are returned bare. +func updateExisting(ctx context.Context, opts CloneOptions, url string, auth *http.BasicAuth) (Result, error) { repo, err := git.PlainOpen(opts.LocalDir) if err != nil { - return Result{}, fmt.Errorf("open existing repo at %s: %w", opts.LocalDir, err) + return Result{}, fmt.Errorf("%w: open existing repo at %s: %w", errLocalState, opts.LocalDir, err) } if err := ensureRemote(repo, url); err != nil { - return Result{}, err + return Result{}, fmt.Errorf("%w: %w", errLocalState, err) } // Snapshot the pre-fetch HEAD so we can short-circuit on NoChanges // when the fetch reveals no new commits. This is the commit currently // on disk; it may or may not match opts.PrevIndexedSHA (mismatch // means a previous index job failed mid-way — the caller decides - // how to recover). + // how to recover). A failure here is the signature of a half-written + // clone (SIGKILL mid-PlainClone leaves .git without refs). prevHead, err := repo.Head() if err != nil { - return Result{}, fmt.Errorf("resolve pre-fetch HEAD: %w", err) + return Result{}, fmt.Errorf("%w: resolve pre-fetch HEAD: %w", errLocalState, err) } err = repo.FetchContext(ctx, &git.FetchOptions{ @@ -179,6 +266,9 @@ func CloneOrFetch(ctx context.Context, opts CloneOptions) (Result, error) { RefSpecs: []config.RefSpec{config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", opts.Branch, opts.Branch))}, Depth: 1, Force: true, + // Default is TagFollowing; without NoTags every fetch can drag in + // new tag snapshots. See the matching option in cloneFresh. + Tags: git.NoTags, }) if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { return Result{}, fmt.Errorf("fetch: %w", err) @@ -186,17 +276,19 @@ func CloneOrFetch(ctx context.Context, opts CloneOptions) (Result, error) { remoteRef, err := repo.Reference(plumbing.NewRemoteReferenceName("origin", opts.Branch), true) if err != nil { - return Result{}, fmt.Errorf("resolve remote ref: %w", err) + return Result{}, fmt.Errorf("%w: resolve remote ref: %w", errLocalState, err) } newSHA := remoteRef.Hash() - // No-op fetch: remote HEAD already matches what's on disk. Skip the - // reset (it would be a no-op anyway) and tell the caller there is - // nothing to reindex. NoChanges supersedes Changes — the caller - // should not enqueue an index job at all. - if prevHead.Hash() == newSHA { - return Result{HeadSHA: newSHA.String(), NoChanges: true}, nil - } + // No-op fetch: remote HEAD already matches what's on disk. Tell the + // caller there is nothing to reindex (NoChanges supersedes Changes — + // no index job should be enqueued) but still run the hard reset below: + // a crash mid-reset on a previous run leaves HEAD already pointing at + // newSHA over half-rewritten files, and this path is the only chance + // to repair that — go-git writes HEAD before it touches the worktree, + // so the torn state looks exactly like a completed update. On a clean + // worktree the reset writes nothing. + noChanges := prevHead.Hash() == newSHA // Best-effort change-set computation. Runs BEFORE the reset so // tree.Diff still sees both commits via their stored tree objects. @@ -204,7 +296,7 @@ func CloneOrFetch(ctx context.Context, opts CloneOptions) (Result, error) { // falls back to a full reindex. var changes *ChangeSet diffBase := strings.TrimSpace(opts.PrevIndexedSHA) - if diffBase != "" { + if diffBase != "" && !noChanges { cs, derr := computeChangeSet(repo, diffBase, newSHA.String()) if derr == nil { changes = cs @@ -217,22 +309,28 @@ func CloneOrFetch(ctx context.Context, opts CloneOptions) (Result, error) { wt, err := repo.Worktree() if err != nil { - return Result{}, fmt.Errorf("worktree: %w", err) + return Result{}, fmt.Errorf("%w: worktree: %w", errLocalState, err) } // Hard reset — discards any local mutation that crept in. Worker-managed - // checkouts have no human edits we'd want to preserve. + // checkouts have no human edits we'd want to preserve. The commit was + // just fetched, so a failure here means the local store is broken. if err := wt.Reset(&git.ResetOptions{ Commit: newSHA, Mode: git.HardReset, }); err != nil { - return Result{}, fmt.Errorf("reset: %w", err) + return Result{}, fmt.Errorf("%w: reset: %w", errLocalState, err) } head, err := repo.Head() if err != nil { - return Result{}, fmt.Errorf("resolve HEAD post-reset: %w", err) + return Result{}, fmt.Errorf("%w: resolve HEAD post-reset: %w", errLocalState, err) } - return Result{HeadSHA: head.Hash().String(), Changes: changes}, nil + return Result{ + HeadSHA: head.Hash().String(), + Changes: changes, + NoChanges: noChanges, + PrevHeadSHA: prevHead.Hash().String(), + }, nil } // computeChangeSet diffs the tree of oldSHA against the tree of newSHA @@ -332,6 +430,17 @@ func normaliseURL(u string) string { return u } +// packfileCount counts the .pack files in the checkout's object store. A +// fresh shallow clone has exactly one; each subsequent fetch adds one more. +// Best effort — 0 on any error keeps the caller on the ordinary reuse path. +func packfileCount(dir string) int { + matches, err := filepath.Glob(filepath.Join(dir, ".git", "objects", "pack", "*.pack")) + if err != nil { + return 0 + } + return len(matches) +} + func needsClone(dir string) bool { gitDir := filepath.Join(dir, ".git") if _, err := os.Stat(gitDir); err != nil { @@ -349,9 +458,9 @@ func ensureRemote(repo *git.Repository, wantURL string) error { urls := remote.Config().URLs if len(urls) == 0 || urls[0] != wantURL { // Repo on disk points at a different URL — likely the workspace - // admin changed the github_url. Easiest fix: nuke + reclone, but - // the caller can't see that from here. Surface as an error so the - // operator at least sees the mismatch in the failed job. + // admin changed the github_url. The old checkout is dead weight; + // the errLocalState wrap this gets in updateExisting is what turns + // it into a nuke + re-clone. return fmt.Errorf("local repo remote %v does not match expected %s", urls, wantURL) } return nil diff --git a/server/internal/repocloner/repocloner_test.go b/server/internal/repocloner/repocloner_test.go index 4a9f0ffd..2b5f5ee9 100644 --- a/server/internal/repocloner/repocloner_test.go +++ b/server/internal/repocloner/repocloner_test.go @@ -2,9 +2,12 @@ package repocloner import ( "context" + "fmt" + "math/rand" "os" "path/filepath" "sort" + "strings" "testing" "time" @@ -146,6 +149,98 @@ func (w *commitWriter) CommitFiles(t *testing.T, message string, files map[strin return sha.String() } +// Tag creates a lightweight tag at the current worktree HEAD and pushes it. +func (w *commitWriter) Tag(t *testing.T, name string) { + t.Helper() + w.ensureWorktree(t) + repo, err := git.PlainOpen(w.worktree) + if err != nil { + t.Fatalf("open worktree: %v", err) + } + head, err := repo.Head() + if err != nil { + t.Fatalf("head: %v", err) + } + if _, err := repo.CreateTag(name, head.Hash(), nil); err != nil { + t.Fatalf("create tag %s: %v", name, err) + } + if err := repo.Push(&git.PushOptions{ + RemoteName: "origin", + RefSpecs: []config.RefSpec{ + config.RefSpec("refs/tags/" + name + ":refs/tags/" + name), + }, + }); err != nil { + t.Fatalf("push tag %s: %v", name, err) + } +} + +// legacyClone reproduces what pre-NoTags server versions wrote to disk: +// go-git's clone default was Tags:AllTags, so a shallow clone carried a full +// snapshot per tag. +func legacyClone(t *testing.T, upstream, dir, branch string) { + t.Helper() + _, err := git.PlainClone(dir, false, &git.CloneOptions{ + URL: "file://" + upstream, + ReferenceName: plumbing.NewBranchReferenceName(branch), + SingleBranch: true, + Depth: 1, + Tags: git.AllTags, + }) + if err != nil { + t.Fatalf("legacy clone: %v", err) + } +} + +// legacyFetchReset reproduces the old update path: fetch(Depth:1)+hard reset +// without NoTags, persisting one more snapshot pack per call. +func legacyFetchReset(t *testing.T, dir, branch string) { + t.Helper() + repo, err := git.PlainOpen(dir) + if err != nil { + t.Fatalf("open %s: %v", dir, err) + } + err = repo.Fetch(&git.FetchOptions{ + RefSpecs: []config.RefSpec{config.RefSpec("+refs/heads/" + branch + ":refs/remotes/origin/" + branch)}, + Depth: 1, + Force: true, + }) + if err != nil && err != git.NoErrAlreadyUpToDate { + t.Fatalf("legacy fetch: %v", err) + } + ref, err := repo.Reference(plumbing.NewRemoteReferenceName("origin", branch), true) + if err != nil { + t.Fatalf("legacy resolve remote ref: %v", err) + } + wt, err := repo.Worktree() + if err != nil { + t.Fatalf("legacy worktree: %v", err) + } + if err := wt.Reset(&git.ResetOptions{Commit: ref.Hash(), Mode: git.HardReset}); err != nil { + t.Fatalf("legacy reset: %v", err) + } +} + +func tagRefCount(t *testing.T, dir string) int { + t.Helper() + repo, err := git.PlainOpen(dir) + if err != nil { + t.Fatalf("open %s: %v", dir, err) + } + refs, err := repo.References() + if err != nil { + t.Fatalf("references: %v", err) + } + defer refs.Close() + n := 0 + _ = refs.ForEach(func(ref *plumbing.Reference) error { + if strings.HasPrefix(ref.Name().String(), tagRefPrefix) { + n++ + } + return nil + }) + return n +} + // initialCloneFor runs a full CloneOrFetch (first-time clone path) so // subsequent calls go through the reuse/fetch branch. func initialCloneFor(t *testing.T, upstream, localDir, branch string) Result { @@ -338,6 +433,465 @@ func TestCloneOrFetch_EmptyPrevSHA_ReturnsNilChangeSet(t *testing.T) { } } +func TestCloneOrFetch_HalfWrittenClone_SelfHeals(t *testing.T) { + upstream, w := makeBareUpstream(t, "main") + headSHA := w.CommitFiles(t, "init", map[string]string{ + "a.go": "package a\n", + }) + + // Simulate what a SIGKILL mid-PlainClone leaves behind: .git exists + // (so needsClone says "reuse") but holds no usable repository state. + local := filepath.Join(t.TempDir(), "clone") + if err := os.MkdirAll(filepath.Join(local, ".git"), 0o755); err != nil { + t.Fatalf("mkdir fake .git: %v", err) + } + if err := os.WriteFile(filepath.Join(local, ".git", "HEAD"), []byte("ref: refs/heads/main\n"), 0o644); err != nil { + t.Fatalf("write fake HEAD: %v", err) + } + + res, err := CloneOrFetch(context.Background(), CloneOptions{ + GitHubURL: "file://" + upstream, + Branch: "main", + LocalDir: local, + }) + if err != nil { + t.Fatalf("CloneOrFetch on half-written clone: %v (want self-heal, got permanent failure)", err) + } + if res.HeadSHA != headSHA { + t.Errorf("HeadSHA = %s, want %s", res.HeadSHA, headSHA) + } + if res.RecloneReason == "" { + t.Error("RecloneReason empty, want the local-state failure that forced the re-clone") + } +} + +func TestCloneOrFetch_RemoteURLChanged_Reclones(t *testing.T) { + upstreamA, wa := makeBareUpstream(t, "main") + wa.CommitFiles(t, "init A", map[string]string{"a.go": "package a\n"}) + + local := filepath.Join(t.TempDir(), "clone") + initialCloneFor(t, upstreamA, local, "main") + + upstreamB, wb := makeBareUpstream(t, "main") + headB := wb.CommitFiles(t, "init B", map[string]string{"b.go": "package b\n"}) + + res, err := CloneOrFetch(context.Background(), CloneOptions{ + GitHubURL: "file://" + upstreamB, + Branch: "main", + LocalDir: local, + }) + if err != nil { + t.Fatalf("CloneOrFetch with changed URL: %v (want re-clone, got error)", err) + } + if res.HeadSHA != headB { + t.Errorf("HeadSHA = %s, want %s (upstream B)", res.HeadSHA, headB) + } + if res.RecloneReason == "" { + t.Error("RecloneReason empty, want the remote-mismatch reason") + } +} + +func TestCloneOrFetch_FreshClone_HasNoTags(t *testing.T) { + upstream, w := makeBareUpstream(t, "main") + w.CommitFiles(t, "v1", map[string]string{"a.go": "package a\n"}) + w.Tag(t, "v1.0.0") + w.CommitFiles(t, "v2", map[string]string{"a.go": "package a // v2\n"}) + w.Tag(t, "v2.0.0") + + local := filepath.Join(t.TempDir(), "clone") + initialCloneFor(t, upstream, local, "main") + + if n := tagRefCount(t, local); n != 0 { + t.Errorf("fresh clone carries %d tag refs, want 0 (Tags:NoTags)", n) + } +} + +// updateAndCompact drives one update the way repojobs.handleClone does: +// CloneOrFetch, then MaybeCompact with the indexed SHA and the pre-fetch +// HEAD in the protect set. +func updateAndCompact(t *testing.T, upstream, local, indexedSHA string) (Result, *CompactStats) { + t.Helper() + res, err := CloneOrFetch(context.Background(), CloneOptions{ + GitHubURL: "file://" + upstream, + Branch: "main", + LocalDir: local, + PrevIndexedSHA: indexedSHA, + }) + if err != nil { + t.Fatalf("CloneOrFetch: %v", err) + } + st, err := MaybeCompact(context.Background(), local, nil, indexedSHA, res.PrevHeadSHA) + if err != nil { + t.Fatalf("MaybeCompact: %v", err) + } + return res, st +} + +// TestUpgradeCompactsLegacyCheckout is the no-explicit-migration upgrade +// path: a checkout produced by a PRE-NoTags server (AllTags clone, +// accumulated fetch packs) must be cleaned by the FIRST update cycle the +// upgraded server runs on it — even one where the upstream has nothing new — +// and later updates must still get their incremental diffs. +func TestUpgradeCompactsLegacyCheckout(t *testing.T) { + upstream, w := makeBareUpstream(t, "main") + w.CommitFiles(t, "v1", map[string]string{ + "a.go": "package a\n", + "keep.go": "package keep\n", + "assets/big": strings.Repeat("payload ", 4096), + }) + w.Tag(t, "r1") + w.CommitFiles(t, "v2", map[string]string{"a.go": "package a // v2\n"}) + w.Tag(t, "r2") + + // What the OLD server left on disk: AllTags shallow clone plus two + // fetch+reset cycles, each of which persisted another snapshot pack. + local := filepath.Join(t.TempDir(), "clone") + legacyClone(t, upstream, local, "main") + w.CommitFiles(t, "v3", map[string]string{"b.go": "package b\n"}) + legacyFetchReset(t, local, "main") + indexedSHA := w.CommitFiles(t, "v4", map[string]string{"c.go": "package c\n"}) + legacyFetchReset(t, local, "main") + + if n := tagRefCount(t, local); n != 2 { + t.Fatalf("legacy checkout has %d tag refs, want 2 — test setup broken", n) + } + if n := packfileCount(local); n < 3 { + t.Fatalf("legacy checkout has %d packs, want >=3 — test setup broken", n) + } + objectsBefore := objectsDirSize(local) + + // Server upgrades. The first update cycle sees NOTHING new upstream — + // cleanup must not wait for the repo's next commit. + res, st := updateAndCompact(t, upstream, local, indexedSHA) + if !res.NoChanges { + t.Errorf("NoChanges = false on an unchanged upstream") + } + if res.RecloneReason != "" { + t.Errorf("RecloneReason = %q — the upgrade path must compact in place, not re-clone", res.RecloneReason) + } + if st == nil { + t.Fatal("no compaction on the first post-upgrade update of a legacy checkout") + } + if st.TagRefsDropped != 2 { + t.Errorf("TagRefsDropped = %d, want 2", st.TagRefsDropped) + } + if n := tagRefCount(t, local); n != 0 { + t.Errorf("%d tag refs survive the upgrade compaction, want 0", n) + } + if n := packfileCount(local); n != 1 { + t.Errorf("packfileCount = %d after compaction, want 1", n) + } + if after := objectsDirSize(local); after >= objectsBefore { + t.Errorf("objects dir did not shrink: %d -> %d bytes", objectsBefore, after) + } + + // The indexed commit survived (it is HEAD here) — the next real update + // must deliver its incremental diff across the compacted store. + newSHA := w.CommitFiles(t, "v5", map[string]string{"c.go": "package c // v5\n"}) + res2, st2 := updateAndCompact(t, upstream, local, indexedSHA) + if res2.HeadSHA != newSHA { + t.Errorf("HeadSHA = %s, want %s", res2.HeadSHA, newSHA) + } + if res2.Changes == nil { + t.Fatal("Changes nil after compaction, want incremental diff") + } + if got := sortedCopy(res2.Changes.Modified); !equalSlices(got, []string{"c.go"}) { + t.Errorf("Modified = %v, want [c.go]", got) + } + if st2 != nil { + t.Errorf("compaction ran again on a clean two-pack checkout: %+v", st2) + } + + // The protected diff base must survive future compactions too: pretend + // the index job never completed (indexed_sha still v4), push again, and + // demand a v4-based diff. + newestSHA := w.CommitFiles(t, "v6", map[string]string{"a.go": "package a // v6\n"}) + res3, _ := updateAndCompact(t, upstream, local, indexedSHA) + if res3.HeadSHA != newestSHA { + t.Errorf("HeadSHA = %s, want %s", res3.HeadSHA, newestSHA) + } + if res3.Changes == nil { + t.Error("Changes nil — protected indexed_sha did not survive") + } +} + +// TestMaybeCompact_ProtectsPendingIndexTarget covers the race the protect +// list exists for: clone cycle A fetched v2 and queued an index job for it, +// but before that job ran, cycle B fetched v3 and compacted. v2 is +// unreferenced by then — only PrevHeadSHA keeps it, and the diff from it +// must still compute afterwards. +func TestMaybeCompact_ProtectsPendingIndexTarget(t *testing.T) { + upstream, w := makeBareUpstream(t, "main") + w.CommitFiles(t, "v1", map[string]string{"a.go": "package a\n"}) + local := filepath.Join(t.TempDir(), "clone") + initialCloneFor(t, upstream, local, "main") + + pendingSHA := w.CommitFiles(t, "v2", map[string]string{"a.go": "package a // v2\n"}) + resA, err := CloneOrFetch(context.Background(), CloneOptions{ + GitHubURL: "file://" + upstream, Branch: "main", LocalDir: local, + }) + if err != nil { + t.Fatalf("cycle A: %v", err) + } + if resA.HeadSHA != pendingSHA { + t.Fatalf("cycle A HeadSHA = %s, want %s", resA.HeadSHA, pendingSHA) + } + + w.CommitFiles(t, "v3", map[string]string{"b.go": "package b\n"}) + resB, err := CloneOrFetch(context.Background(), CloneOptions{ + GitHubURL: "file://" + upstream, Branch: "main", LocalDir: local, + }) + if err != nil { + t.Fatalf("cycle B: %v", err) + } + if resB.PrevHeadSHA != pendingSHA { + t.Fatalf("PrevHeadSHA = %s, want the pending index target %s", resB.PrevHeadSHA, pendingSHA) + } + // Compact below threshold on purpose — call the internals directly the + // way MaybeCompact would once the trigger fires. + if _, err := compactCheckout(context.Background(), local, plumbing.NewHash(resB.PrevHeadSHA)); err != nil { + t.Fatalf("compactCheckout: %v", err) + } + + // The pending index job's diff base (v2) must still be usable. + finalSHA := w.CommitFiles(t, "v4", map[string]string{"c.go": "package c\n"}) + resC, err := CloneOrFetch(context.Background(), CloneOptions{ + GitHubURL: "file://" + upstream, Branch: "main", LocalDir: local, + PrevIndexedSHA: pendingSHA, + }) + if err != nil { + t.Fatalf("cycle C: %v", err) + } + if resC.HeadSHA != finalSHA { + t.Errorf("HeadSHA = %s, want %s", resC.HeadSHA, finalSHA) + } + if resC.Changes == nil { + t.Error("Changes nil — the pending index target was not protected across compaction") + } +} + +// TestCloneOrFetch_NoChangesStillRepairsWorktree: a crash mid-hard-reset +// leaves HEAD already moved over half-rewritten files, which a later cycle +// sees as "nothing to do". The NoChanges path must still reset so torn +// content cannot survive indefinitely. +func TestCloneOrFetch_NoChangesStillRepairsWorktree(t *testing.T) { + upstream, w := makeBareUpstream(t, "main") + headSHA := w.CommitFiles(t, "v1", map[string]string{"a.go": "package a\n"}) + local := filepath.Join(t.TempDir(), "clone") + initialCloneFor(t, upstream, local, "main") + + // Simulate the torn state: HEAD is right, a worktree file is not. + torn := filepath.Join(local, "a.go") + if err := os.WriteFile(torn, []byte("package torn\n"), 0o644); err != nil { + t.Fatalf("write torn file: %v", err) + } + + res, err := CloneOrFetch(context.Background(), CloneOptions{ + GitHubURL: "file://" + upstream, Branch: "main", LocalDir: local, + PrevIndexedSHA: headSHA, + }) + if err != nil { + t.Fatalf("CloneOrFetch: %v", err) + } + if !res.NoChanges { + t.Errorf("NoChanges = false, want true (upstream unchanged)") + } + got, err := os.ReadFile(torn) + if err != nil { + t.Fatalf("read repaired file: %v", err) + } + if string(got) != "package a\n" { + t.Errorf("worktree file = %q after NoChanges cycle, want the committed content", got) + } +} + +// TestMaybeCompact_ErrorKeepsCheckout: a compaction failure must leave the +// checkout exactly as the update left it — valid, tags intact (so the +// trigger re-fires) — and must not cascade into a delete or re-clone. +func TestMaybeCompact_ErrorKeepsCheckout(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("root ignores directory permissions") + } + upstream, w := makeBareUpstream(t, "main") + headSHA := w.CommitFiles(t, "v1", map[string]string{"a.go": "package a\n"}) + w.Tag(t, "r1") + local := filepath.Join(t.TempDir(), "clone") + legacyClone(t, upstream, local, "main") + + // Make the pack directory unwritable: the encoder cannot land the new + // pack, which is the earliest (and per the crash-ordering, the safest) + // failure point. + packDir := filepath.Join(local, ".git", "objects", "pack") + if err := os.Chmod(packDir, 0o555); err != nil { + t.Fatalf("chmod: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(packDir, 0o755) }) + + st, err := MaybeCompact(context.Background(), local, nil) + if err == nil { + t.Fatalf("MaybeCompact succeeded against a read-only pack dir, stats=%+v", st) + } + + // Checkout must be untouched and fully usable. + repo, oerr := git.PlainOpen(local) + if oerr != nil { + t.Fatalf("checkout destroyed by failed compaction: %v", oerr) + } + head, herr := repo.Head() + if herr != nil || head.Hash().String() != headSHA { + t.Fatalf("HEAD broken after failed compaction: %v (%v)", head, herr) + } + if n := tagRefCount(t, local); n != 1 { + t.Errorf("tag refs = %d after failed compaction, want 1 — the re-trigger must stay armed", n) + } +} + +// TestMaybeCompact_CancelledContext: shutdown must be able to skip +// compaction entirely; the store stays as-is and nothing is deleted. +func TestMaybeCompact_CancelledContext(t *testing.T) { + upstream, w := makeBareUpstream(t, "main") + w.CommitFiles(t, "v1", map[string]string{"a.go": "package a\n"}) + w.Tag(t, "r1") + local := filepath.Join(t.TempDir(), "clone") + legacyClone(t, upstream, local, "main") + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + st, err := MaybeCompact(ctx, local, nil) + if st != nil || err != nil { + t.Fatalf("MaybeCompact(cancelled) = (%+v, %v), want (nil, nil)", st, err) + } + if n := tagRefCount(t, local); n != 1 { + t.Errorf("tag refs = %d, want 1 — cancelled compaction must not touch the store", n) + } +} + +// TestNeedsCompaction_RatioBackstopRearms covers the crash window where a +// previous compaction dropped the tag refs but died before deleting the old +// packs: no tags, below the pack threshold, yet the store dwarfs the +// worktree. The size-ratio trigger must re-arm cleanup. +func TestNeedsCompaction_RatioBackstopRearms(t *testing.T) { + upstream, w := makeBareUpstream(t, "main") + // Incompressible payloads so zlib cannot hide the retained snapshots. + w.CommitFiles(t, "v1", map[string]string{"blob.bin": randomContent(t, 1, 1<<20)}) + w.Tag(t, "r1") + w.CommitFiles(t, "v2", map[string]string{"blob.bin": randomContent(t, 2, 1<<20)}) + w.Tag(t, "r2") + + local := filepath.Join(t.TempDir(), "clone") + legacyClone(t, upstream, local, "main") + w.CommitFiles(t, "v3", map[string]string{"blob.bin": randomContent(t, 3, 1<<20)}) + legacyFetchReset(t, local, "main") + + // Simulate the crashed compaction: tags durably gone, bloat still here. + repo, err := git.PlainOpen(local) + if err != nil { + t.Fatalf("open: %v", err) + } + for _, name := range []string{"r1", "r2"} { + if err := repo.Storer.RemoveReference(plumbing.ReferenceName("refs/tags/" + name)); err != nil { + t.Fatalf("drop tag %s: %v", name, err) + } + } + if n := packfileCount(local); n < 2 || n >= compactPackThreshold { + t.Fatalf("packfileCount = %d, want in [2, %d) — test setup broken", n, compactPackThreshold) + } + + if !needsCompaction(local) { + t.Fatal("needsCompaction = false on a tagless bloated checkout — the ratio backstop is dead") + } + before := objectsDirSize(local) + st, err := MaybeCompact(context.Background(), local, nil) + if err != nil { + t.Fatalf("MaybeCompact: %v", err) + } + if st == nil { + t.Fatal("MaybeCompact did nothing") + } + if after := objectsDirSize(local); after >= before/2 { + t.Errorf("objects %d -> %d, want the retained snapshots reclaimed", before, after) + } + if needsCompaction(local) { + t.Error("needsCompaction still true after compaction — would loop every update") + } +} + +// randomContent builds deterministic incompressible bytes. +func randomContent(t *testing.T, seed int64, n int) string { + t.Helper() + rnd := rand.New(rand.NewSource(seed)) + b := make([]byte, n) + if _, err := rnd.Read(b); err != nil { + t.Fatalf("rand: %v", err) + } + return string(b) +} + +func TestPackAccumulationStaysBounded(t *testing.T) { + upstream, w := makeBareUpstream(t, "main") + prev := w.CommitFiles(t, "init", map[string]string{"a.go": "package a\n"}) + + local := filepath.Join(t.TempDir(), "clone") + initialCloneFor(t, upstream, local, "main") + + compactions := 0 + for i := 1; i <= 2*compactPackThreshold; i++ { + sha := w.CommitFiles(t, fmt.Sprintf("push %d", i), map[string]string{ + "a.go": fmt.Sprintf("package a // rev %d\n", i), + }) + res, st := updateAndCompact(t, upstream, local, prev) + if res.HeadSHA != sha { + t.Fatalf("cycle %d: HeadSHA = %s, want %s", i, res.HeadSHA, sha) + } + if st != nil { + compactions++ + } + if n := packfileCount(local); n > compactPackThreshold { + t.Fatalf("cycle %d: %d packs on disk, bound is %d", i, n, compactPackThreshold) + } + prev = sha + } + if compactions == 0 { + t.Errorf("no compaction ran across %d fetch cycles", 2*compactPackThreshold) + } +} + +func TestCloneOrFetch_FetchFailure_KeepsClone(t *testing.T) { + upstream, w := makeBareUpstream(t, "main") + headSHA := w.CommitFiles(t, "init", map[string]string{"a.go": "package a\n"}) + + local := filepath.Join(t.TempDir(), "clone") + initialCloneFor(t, upstream, local, "main") + + // Kill the upstream. The URL still matches the checkout's origin, so + // this is indistinguishable from a network outage — the fetch must + // fail WITHOUT costing us the healthy local clone. + if err := os.RemoveAll(upstream); err != nil { + t.Fatalf("remove upstream: %v", err) + } + + _, err := CloneOrFetch(context.Background(), CloneOptions{ + GitHubURL: "file://" + upstream, + Branch: "main", + LocalDir: local, + }) + if err == nil { + t.Fatal("CloneOrFetch succeeded against a dead upstream, want error") + } + + repo, oerr := git.PlainOpen(local) + if oerr != nil { + t.Fatalf("local clone destroyed by a fetch failure: %v", oerr) + } + head, herr := repo.Head() + if herr != nil { + t.Fatalf("local clone HEAD unreadable after fetch failure: %v", herr) + } + if head.Hash().String() != headSHA { + t.Errorf("local HEAD = %s, want untouched %s", head.Hash().String(), headSHA) + } +} + func TestChangeSet_IsEmpty(t *testing.T) { if !(*ChangeSet)(nil).IsEmpty() { t.Error("nil ChangeSet should report IsEmpty=true") diff --git a/server/internal/repojobs/repojobs.go b/server/internal/repojobs/repojobs.go index 721fd586..153e07ac 100644 --- a/server/internal/repojobs/repojobs.go +++ b/server/internal/repojobs/repojobs.go @@ -241,6 +241,38 @@ func handleClone(ctx context.Context, d Deps, job jobs.Job) error { d.recordFailure(ctx, g, fmt.Errorf("clone: %w", err)) return err } + if result.RecloneReason != "" { + d.Logger.Info("repojobs: checkout discarded and re-cloned", + "project", g.ProjectPath, "reason", result.RecloneReason) + } + + // Compaction runs OUTSIDE the write-locked clone section above: + // MaybeCompact serialises all compactions on a global gate (their + // transient heap is ~3× the repo snapshot), and taking that gate while + // holding this repo's write lock would stall this repo's readers behind + // every other repo's compaction. MaybeCompact re-takes the write lock + // itself just for the store mutation. A compaction failure never fails + // the job — the checkout is still exactly what the update left behind, + // and the trigger re-fires on the next update. + // + // The protect list keeps two unreferenced commits alive across the + // rewrite: the indexed diff base, and the pre-fetch HEAD — the latter is + // the TargetSHA of an index job that may still be queued from a previous + // clone cycle. + compactLock := func(f func() error) error { return f() } + if d.RepoLocks != nil { + compactLock = func(f func() error) error { return d.RepoLocks.WithWrite(hash, f) } + } + if c, cerr := repocloner.MaybeCompact(ctx, cloneDir, compactLock, g.IndexedSHA, result.PrevHeadSHA); cerr != nil { + d.Logger.Warn("repojobs: compaction failed; checkout kept as-is", + "project", g.ProjectPath, "err", cerr) + } else if c != nil { + d.Logger.Info("repojobs: checkout object store compacted", + "project", g.ProjectPath, + "bytes_before", c.ObjectsBefore, "bytes_after", c.ObjectsAfter, + "packs_deleted", c.PacksDeleted, "tag_refs_dropped", c.TagRefsDropped, + "objects", c.Reachable, "ms", c.Duration.Milliseconds()) + } if err := d.GitRepos.SetClone(ctx, g.ProjectPath, result.HeadSHA, ""); err != nil { d.Logger.Warn("repojobs: set last_sha failed", "project", g.ProjectPath, "err", err) diff --git a/server/internal/tokenizer/bpecount/bpecount.go b/server/internal/tokenizer/bpecount/bpecount.go new file mode 100644 index 00000000..64c7387f --- /dev/null +++ b/server/internal/tokenizer/bpecount/bpecount.go @@ -0,0 +1,634 @@ +// Package bpecount is a minimal, count-only byte-level BPE tokenizer for +// GPT-2/Qwen2-style tokenizer.json files (voyage-code-3, Qwen2, GPT-4o…). +// +// It reproduces the HuggingFace pipeline: +// +// normalizer = NFC +// pretokenize = Split(Qwen2 regex, Isolated) + ByteLevel(add_prefix_space=false) +// model = BPE (greedy lowest-rank merge) +// +// The Split regex contains `\s+(?!\S)`, a negative lookahead Go's RE2 +// cannot express, so the splitter is hand-rolled rather than compiled. +// Only a COUNT is produced — no ids, no offsets. +package bpecount + +import ( + "container/heap" + "encoding/json" + "fmt" + "os" + "strings" + "sync" + "unicode" + "unicode/utf8" + + "golang.org/x/text/unicode/norm" +) + +// Counter holds the merge table and a memo of pre-token → token count. +type Counter struct { + merges map[string]int32 // "left right" -> rank + + mu sync.RWMutex + memo map[string]int +} + +type tokJSON struct { + Model struct { + Type string `json:"type"` + Merges json.RawMessage `json:"merges"` + } `json:"model"` + Normalizer struct { + Type string `json:"type"` + } `json:"normalizer"` + PreTokenizer struct { + Type string `json:"type"` + PreTokenizers []struct { + Type string `json:"type"` + Pattern struct { + Regex string `json:"Regex"` + } `json:"pattern"` + } `json:"pretokenizers"` + } `json:"pre_tokenizer"` +} + +// qwen2SplitPattern is the pre-tokenizer regex this package implements by +// hand. It is compared, not compiled: the point of the hand-rolled splitter is +// that Go's RE2 cannot express the `\s+(?!\S)` lookahead in it. +// +// The comparison is the load-time guard against a plausible and silent +// failure: GPT-2 and o200k tokenizer.json files parse fine, declare +// model.type "BPE", and would produce confidently wrong counts against this +// splitter — GPT-2 has no Split stage at all, o200k has a different pattern. +// Refusing them keeps ExactCounts() false and the caller on its estimate, +// which is wrong but knows it is. +const qwen2SplitPattern = `(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+` + +// checkPipeline refuses a tokenizer whose normalizer or pre-tokenizer is not +// the one this package reimplements. Counting a different pipeline with this +// splitter does not fail loudly — it returns plausible numbers that are wrong, +// and the caller then packs batches and sizes chunks against them. +func checkPipeline(tj tokJSON) error { + if tj.Normalizer.Type != "NFC" { + return fmt.Errorf("bpecount: normalizer is %q, this package implements NFC", + tj.Normalizer.Type) + } + if tj.PreTokenizer.Type != "Sequence" || len(tj.PreTokenizer.PreTokenizers) < 2 { + return fmt.Errorf("bpecount: pre_tokenizer is %q, expected Sequence[Split, ByteLevel]", + tj.PreTokenizer.Type) + } + split, byteLevel := tj.PreTokenizer.PreTokenizers[0], tj.PreTokenizer.PreTokenizers[1] + if split.Type != "Split" || byteLevel.Type != "ByteLevel" { + return fmt.Errorf("bpecount: pre_tokenizer is Sequence[%s, %s], expected Sequence[Split, ByteLevel]", + split.Type, byteLevel.Type) + } + if split.Pattern.Regex != qwen2SplitPattern { + return fmt.Errorf("bpecount: Split pattern is not the one implemented here " + + "(a GPT-2 or o200k tokenizer would count wrong rather than fail)") + } + return nil +} + +// Load reads a tokenizer.json and keeps only what a count needs: the merges. +func Load(path string) (*Counter, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return LoadBytes(b) +} + +func LoadBytes(b []byte) (*Counter, error) { + var tj tokJSON + if err := json.Unmarshal(b, &tj); err != nil { + return nil, err + } + if tj.Model.Type != "BPE" { + return nil, fmt.Errorf("bpecount: unsupported model type %q", tj.Model.Type) + } + if err := checkPipeline(tj); err != nil { + return nil, err + } + // merges is either ["a b", ...] (v1) or [["a","b"], ...] (v2). + var flat []string + m := make(map[string]int32) + if err := json.Unmarshal(tj.Model.Merges, &flat); err == nil { + for i, s := range flat { + m[s] = int32(i) + } + } else { + var pairs [][]string + if err := json.Unmarshal(tj.Model.Merges, &pairs); err != nil { + return nil, fmt.Errorf("bpecount: merges: %w", err) + } + for i, p := range pairs { + if len(p) == 2 { + m[p[0]+" "+p[1]] = int32(i) + } + } + } + return &Counter{merges: m, memo: make(map[string]int, 1<<16)}, nil +} + +// ---------- byte-level alphabet (GPT-2 bytes_to_unicode) ---------- + +var byteRune [256]rune + +func init() { + for b := 0; b < 256; b++ { + r := rune(b) + switch { + case r == 0xad: + r = 0x143 + case r <= 0x20: + r += 0x100 + case r >= 0x7f && r <= 0xa0: + r += 0xa2 + } + byteRune[b] = r + } +} + +// ---------- hand-rolled splitter ---------- +// +// Qwen2 pattern, alternation tried left to right (Perl leftmost-first): +// +// (?i:'s|'t|'re|'ve|'m|'ll|'d) +// [^\r\n\p{L}\p{N}]?\p{L}+ +// \p{N} +// ?[^\s\p{L}\p{N}]+[\r\n]* +// \s*[\r\n]+ +// \s+(?!\S) +// \s+ +// +// Every rune is covered by some branch, so Split(Isolated) yields no gaps. + +func isL(r rune) bool { return unicode.IsLetter(r) } +func isN(r rune) bool { return unicode.IsNumber(r) } +func isWS(r rune) bool { return unicode.IsSpace(r) } +func isNL(r rune) bool { return r == '\r' || r == '\n' } + +var contractions = []string{"s", "t", "re", "ve", "m", "ll", "d"} + +// nextToken returns the byte length of the pre-token starting at s[0]. +func nextToken(s string) int { + r0, w0 := decode(s, 0) + + // A: contraction + // + // Only the two bytes after the apostrophe can matter (the longest + // contraction is "re"/"ve"/"ll"), so lowercase just those. Lowercasing the + // whole remaining string here allocated a copy of the suffix for every + // apostrophe in the file: a 512 KB source with 5,000 quotes moved over a + // gigabyte through the allocator, on the indexing hot path. + if r0 == '\'' && len(s) > w0 { + tail := s[w0:] + if len(tail) > 2 { + tail = tail[:2] + } + low := strings.ToLower(tail) + for _, c := range contractions { + if strings.HasPrefix(low, c) { + return w0 + len(c) + } + } + } + + // B: [^\r\n\p{L}\p{N}]? \p{L}+ + { + i := 0 + if !isNL(r0) && !isL(r0) && !isN(r0) { + i = w0 + } + j := i + for j < len(s) { + r, w := decode(s, j) + if !isL(r) { + break + } + j += w + } + if j > i { // at least one letter followed + return j + } + } + + // C: single \p{N} + if isN(r0) { + return w0 + } + + // D: " ?" [^\s\p{L}\p{N}]+ [\r\n]* + { + i := 0 + if r0 == ' ' { + i = w0 + } + j := i + for j < len(s) { + r, w := decode(s, j) + if isWS(r) || isL(r) || isN(r) { + break + } + j += w + } + if j > i { + for j < len(s) { + r, w := decode(s, j) + if !isNL(r) { + break + } + j += w + } + return j + } + } + + // E/F/G: whitespace run. + if isWS(r0) { + // maximal whitespace run + end := 0 + lastNL := -1 + for end < len(s) { + r, w := decode(s, end) + if !isWS(r) { + break + } + if isNL(r) { + lastNL = end + w + } + end += w + } + // E: \s*[\r\n]+ — run truncated after its LAST \r or \n. + if lastNL >= 0 { + return lastNL + } + // F: \s+(?!\S) — whole run at EOF, else run minus its last rune. + if end == len(s) { + return end + } + _, lw := decodeLast(s[:end]) + if end-lw > 0 { + return end - lw + } + // G: \s+ (single whitespace rune followed by a non-space) + return end + } + + // Unreachable for well-formed input; make progress anyway. + return w0 +} + +func decode(s string, i int) (rune, int) { + if s[i] < utf8.RuneSelf { + return rune(s[i]), 1 + } + return utf8.DecodeRuneInString(s[i:]) +} + +func decodeLast(s string) (rune, int) { + return utf8.DecodeLastRuneInString(s) +} + +// ---------- BPE ---------- + +func (c *Counter) bpeLen(piece string) int { + c.mu.RLock() + n, ok := c.memo[piece] + c.mu.RUnlock() + if ok { + return n + } + n = c.bpe(piece) + c.mu.Lock() + if len(c.memo) < 1<<20 { + c.memo[piece] = n + } + c.mu.Unlock() + return n +} + +// node is one symbol in the doubly-linked list the merge loop walks. +type node struct { + prev, next int + s string + alive bool +} + +// cand is a candidate merge sitting in the priority queue. +type cand struct { + rank int32 + l, r int + // len of the two symbols when the candidate was pushed; a stale entry + // (one side already merged into something longer) is detected by comparing. + ll, rl int +} + +type candHeap []cand + +func (h candHeap) Len() int { return len(h) } +func (h candHeap) Less(i, j int) bool { + if h[i].rank != h[j].rank { + return h[i].rank < h[j].rank + } + return h[i].l < h[j].l // ties: leftmost first, matching HF +} +func (h candHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *candHeap) Push(x any) { *h = append(*h, x.(cand)) } +func (h *candHeap) Pop() any { + old := *h + n := len(old) + v := old[n-1] + *h = old[:n-1] + return v +} + +// bpe applies the greedy lowest-rank-first merge with a linked list + heap, +// so a pathological single pre-token (a 30 KB run of '-' or one enormous +// identifier) stays near-linear instead of the O(n^2) rescan a naive loop does. +func (c *Counter) bpe(piece string) int { + nodes := make([]node, 0, len(piece)) + for _, r := range piece { + i := len(nodes) + nodes = append(nodes, node{prev: i - 1, next: i + 1, s: string(r), alive: true}) + } + n := len(nodes) + if n < 2 { + return n + } + nodes[n-1].next = -1 + + h := make(candHeap, 0, n) + push := func(l, r int) { + if l < 0 || r < 0 || r >= n { + return + } + if rk, ok := c.merges[nodes[l].s+" "+nodes[r].s]; ok { + h = append(h, cand{rank: rk, l: l, r: r, ll: len(nodes[l].s), rl: len(nodes[r].s)}) + } + } + for i := 0; i+1 < n; i++ { + push(i, i+1) + } + heap.Init(&h) + + live := n + for h.Len() > 0 { + cd := heap.Pop(&h).(cand) + l, r := cd.l, cd.r + // Reject stale entries: either side merged away or grew since push. + if !nodes[l].alive || !nodes[r].alive || nodes[l].next != r || + len(nodes[l].s) != cd.ll || len(nodes[r].s) != cd.rl { + continue + } + nodes[l].s += nodes[r].s + nodes[r].alive = false + nodes[l].next = nodes[r].next + if nodes[r].next >= 0 { + nodes[nodes[r].next].prev = l + } + live-- + if live == 1 { + return 1 + } + if p := nodes[l].prev; p >= 0 { + if rk, ok := c.merges[nodes[p].s+" "+nodes[l].s]; ok { + heap.Push(&h, cand{rank: rk, l: p, r: l, ll: len(nodes[p].s), rl: len(nodes[l].s)}) + } + } + if nx := nodes[l].next; nx >= 0 { + if rk, ok := c.merges[nodes[l].s+" "+nodes[nx].s]; ok { + heap.Push(&h, cand{rank: rk, l: l, r: nx, ll: len(nodes[l].s), rl: len(nodes[nx].s)}) + } + } + } + return live +} + +// Count returns the number of tokens voyage/HF would produce for text. +func (c *Counter) Count(text string) int { + if text == "" { + return 0 + } + s := norm.NFC.String(text) + total := 0 + var sb strings.Builder + for len(s) > 0 { + n := nextToken(s) + if n <= 0 { + n = 1 + } + piece := s[:n] + s = s[n:] + // ByteLevel map + sb.Reset() + sb.Grow(len(piece) * 2) + for i := 0; i < len(piece); i++ { + sb.WriteRune(byteRune[piece[i]]) + } + total += c.bpeLen(sb.String()) + } + return total +} + +// SplitPoints returns the byte offsets at which text must be cut so that no +// piece exceeds budget tokens, plus the total token count of the whole text. +// +// Offsets index the CALLER's string, which is not the same thing as indexing +// the normalised copy counting works on. NFC can both shrink the text (a +// decomposed "e"+U+0301 becomes one code point) and grow it (the composition +// exclusions at U+0958..U+095F decompose under NFC), so an offset taken from +// the normalised copy can land mid-rune or past the end of the original — the +// latter panics the caller's slice expression. Already-normalised input, which +// is nearly all source code, takes the fast path where the two coincide. +// +// The cuts are exact, not estimated, and they need no search. BPE merges never +// cross a pre-token boundary in this pipeline (Split runs with Isolated +// behaviour, and ByteLevel+BPE are applied per pre-token), so a text's token +// count is the SUM of its pre-tokens' counts. Cutting on a pre-token boundary +// therefore leaves both sides tokenising exactly as they did inside the whole: +// the parts always add up to the total, with no drift to correct for. +// +// A single pre-token larger than budget cannot be honoured on a boundary (its +// merges DO interact internally), so splitInside falls back to a binary search +// on bytes within that one pre-token. That path is for base64 blobs and +// minified lines with no whitespace; on a 45-repo corpus it fires for 5 chunks +// in 1.9M. +// +// Offsets are cut points only: nil means the text already fits. +func (c *Counter) SplitPoints(text string, budget int) (offsets []int, total int) { + if text == "" { + return nil, 0 + } + if budget <= 0 { + return nil, c.Count(text) + } + if norm.NFC.IsNormalString(text) { + return c.splitNormalized(text, budget) + } + return c.splitDenormalized(text, budget) +} + +// splitDenormalized handles input that is not already NFC. +// +// It cuts one piece at a time: the first cut is computed on the normalised +// form, mapped back to a raw offset on a normalisation boundary, and the +// remainder is then processed from its real start. Recomputing per piece +// rather than translating a whole offsets slice is deliberate — mapping a cut +// backwards to a boundary shrinks the piece before it and grows the one after, +// so offsets computed against the old start would no longer hold. The cost is +// one pass per piece, paid only by input that is not already normalised, which +// source code essentially never is. +func (c *Counter) splitDenormalized(text string, budget int) ([]int, int) { + total := c.Count(text) + var offsets []int + base := 0 + for base < len(text) { + rest := text[base:] + cuts, _ := c.splitNormalized(norm.NFC.String(rest), budget) + if len(cuts) == 0 { + break + } + at := rawOffsetOf(rest, cuts[0]) + if at <= 0 || at >= len(rest) { + break + } + offsets = append(offsets, base+at) + base += at + } + return offsets, total +} + +// rawOffsetOf maps a byte offset in NFC(raw) back to a byte offset in raw, +// rounding DOWN to a normalisation boundary — rounding down can only shrink +// the piece that ends there, so the budget survives the rounding. +func rawOffsetOf(raw string, normOff int) int { + rawPos, normPos, lastRaw := 0, 0, 0 + for rawPos < len(raw) { + n := norm.NFC.NextBoundaryInString(raw[rawPos:], true) + if n <= 0 { + break + } + segNorm := len(norm.NFC.String(raw[rawPos : rawPos+n])) + if normPos+segNorm > normOff { + return lastRaw + } + normPos += segNorm + rawPos += n + lastRaw = rawPos + } + return lastRaw +} + +// splitNormalized is SplitPoints for input already known to be NFC, where a +// normalised offset IS a raw offset. +func (c *Counter) splitNormalized(s string, budget int) (offsets []int, total int) { + acc := 0 // tokens accumulated in the current piece + pos := 0 // byte offset into s + var sb strings.Builder + for pos < len(s) { + n := nextToken(s[pos:]) + if n <= 0 { + n = 1 + } + piece := s[pos : pos+n] + + sb.Reset() + sb.Grow(len(piece) * 2) + for i := 0; i < len(piece); i++ { + sb.WriteRune(byteRune[piece[i]]) + } + tk := c.bpeLen(sb.String()) + + switch { + case tk > budget: + // Does not fit even alone. Close the current piece, then cut + // inside this pre-token. + if acc > 0 { + offsets = append(offsets, pos) + acc = 0 + } + inner := c.splitInside(piece, budget) + for _, off := range inner { + offsets = append(offsets, pos+off) + } + // The tail after the last inner cut opens the next piece and + // must be CHARGED for: leaving acc at zero let the following + // pre-tokens add a full budget on top of it, producing pieces of + // up to twice the budget. + tailStart := 0 + if len(inner) > 0 { + tailStart = inner[len(inner)-1] + } + acc = c.Count(piece[tailStart:]) + case acc+tk > budget: + offsets = append(offsets, pos) + acc = tk + default: + acc += tk + } + total += tk + pos += n + } + return offsets, total +} + +// splitInside cuts one over-budget pre-token by binary search on its bytes. +// Inside a pre-token counts are not additive, so every candidate cut is +// re-counted — but the search converges in a handful of probes because +// bytes-per-token is near-constant within a homogeneous run. +func (c *Counter) splitInside(piece string, budget int) []int { + // Candidate cut positions are rune starts, enumerated once. The search + // then runs over INDICES into that list rather than over byte offsets. + // + // The byte-offset version of this loop deadlocked: it aligned a midpoint + // to a rune start by decrementing, and when alignment pulled the midpoint + // below lo, the next lo = mid+1 did not advance, so the (lo, hi) pair + // repeated forever. Any run of multi-byte runes long enough to exceed the + // budget reached it — a box-drawing comment separator is enough, and that + // hung the indexing worker with no error and no progress. Searching over + // rune indices removes the failure rather than guarding it: every + // candidate is a valid boundary by construction, so no alignment step + // exists to misbehave. + starts := make([]int, 0, len(piece)/2+2) + for i := 0; i < len(piece); { + starts = append(starts, i) + _, w := utf8.DecodeRuneInString(piece[i:]) + if w <= 0 { + w = 1 + } + i += w + } + starts = append(starts, len(piece)) + + var cuts []int + si := 0 + for si < len(starts)-1 { + if c.Count(piece[starts[si]:]) <= budget { + break + } + // Largest j > si whose prefix still fits. + lo, hi, best := si+1, len(starts)-1, -1 + for lo <= hi { + mid := (lo + hi) / 2 + if c.Count(piece[starts[si]:starts[mid]]) <= budget { + best = mid + lo = mid + 1 + } else { + hi = mid - 1 + } + } + if best < 0 { + // Even one rune exceeds the budget. Emit it anyway: refusing to + // advance is the deadlock this rewrite exists to remove, and a + // budget smaller than a single token is the caller's problem. + best = si + 1 + } + if starts[best] >= len(piece) { + break + } + cuts = append(cuts, starts[best]) + si = best + } + return cuts +} diff --git a/server/internal/tokenizer/bpecount/bpecount_test.go b/server/internal/tokenizer/bpecount/bpecount_test.go new file mode 100644 index 00000000..d41a6dfc --- /dev/null +++ b/server/internal/tokenizer/bpecount/bpecount_test.go @@ -0,0 +1,374 @@ +package bpecount + +import ( + "encoding/json" + "os" + "strings" + "testing" + "time" + "unicode/utf8" +) + +// tokenizerPath is the real voyage-code-3 tokenizer.json. The tests that need +// it skip when it is absent so a checkout without the 7 MB file still builds +// and tests clean. +const tokenizerPath = "../../../../loadtests/bench/voyage-code-3.tokenizer.json" + +func load(t *testing.T) *Counter { + t.Helper() + if _, err := os.Stat(tokenizerPath); err != nil { + t.Skip("tokenizer.json not present") + } + c, err := Load(tokenizerPath) + if err != nil { + t.Fatalf("load: %v", err) + } + return c +} + +// TestCountMatchesReference pins the counts that were verified against +// Voyage's own usage.total_tokens and the HuggingFace Rust tokenizer. The tab +// cases are the ones a RE2 rewrite of the pre-tokenizer regex gets wrong: the +// `\s+(?!\S)` lookahead cannot be expressed, and a naive rewrite absorbs a +// leading tab into the punctuation branch that may only absorb a space. +func TestCountMatchesReference(t *testing.T) { + c := load(t) + for _, tc := range []struct { + in string + want int + }{ + {"func main() {\n\tfmt.Println(\"hi\")\n}\n", 10}, + {"\t\t\"a\"", 4}, + {"\t\t\t\"end\": {", 6}, + {"a\t\t-b", 4}, + {"class A:\n def g(self):\n x = 1\n", 14}, + {"hello world", 2}, + {"#ifdef USE_THREADS", 3}, + {"", 0}, + } { + if got := c.Count(tc.in); got != tc.want { + t.Errorf("Count(%q) = %d, want %d", tc.in, got, tc.want) + } + } +} + +// TestNFCNormalisation covers the second gap in the ollama tokenizer: the +// tokenizer.json declares an NFC normalizer, and skipping it makes decomposed +// input cost an extra token. +func TestNFCNormalisation(t *testing.T) { + c := load(t) + nfc := "caf\u00e9" // é as one code point + nfd := "cafe\u0301" // e + combining acute + if a, b := c.Count(nfc), c.Count(nfd); a != b { + t.Errorf("NFC %d != NFD %d — normaliser not applied", a, b) + } +} + +// TestSplitPointsAreExact is the property the splitter exists for: because BPE +// merges never cross a pre-token boundary, the pieces must add up to the whole +// and none may exceed the budget. +func TestSplitPointsAreExact(t *testing.T) { + c := load(t) + src := "" + for i := 0; i < 400; i++ { + src += "func handler(w http.ResponseWriter, r *http.Request) {\n\tdefer r.Body.Close()\n}\n" + } + const budget = 500 + + offsets, total := c.SplitPoints(src, budget) + if total != c.Count(src) { + t.Fatalf("SplitPoints total %d != Count %d", total, c.Count(src)) + } + if len(offsets) == 0 { + t.Fatalf("expected cuts for %d tokens at budget %d", total, budget) + } + + sum, prev := 0, 0 + for _, off := range append(offsets, len(src)) { + n := c.Count(src[prev:off]) + if n > budget { + t.Errorf("piece [%d:%d] is %d tokens, over budget %d", prev, off, n, budget) + } + sum += n + prev = off + } + if sum != total { + t.Errorf("pieces sum to %d, whole is %d — merges leaked across a cut", sum, total) + } +} + +// TestSplitPointsFitsAlready — a text under budget must not be cut. +func TestSplitPointsFitsAlready(t *testing.T) { + c := load(t) + offsets, total := c.SplitPoints("package main\n", 1000) + if offsets != nil { + t.Errorf("expected no cuts, got %v", offsets) + } + if total == 0 { + t.Error("total should be counted even when no cut is needed") + } +} + +// TestSplitInsidePreToken covers the one case boundaries cannot serve: a +// single pre-token bigger than the budget (base64 blobs, minified lines). +func TestSplitPointsOversizePreToken(t *testing.T) { + c := load(t) + // One unbroken run of a single character class. "aB3" repeated would NOT + // do: the pre-tokenizer breaks letters from digits, so it yields 2-byte + // pre-tokens that never exceed the budget and the splitInside path this + // test exists for is never entered. + blob := strings.Repeat("a", 4000) + const budget = 100 + offsets, _ := c.SplitPoints(blob, budget) + if len(offsets) == 0 { + t.Fatal("expected the blob to be cut") + } + prev := 0 + for _, off := range append(offsets, len(blob)) { + if n := c.Count(blob[prev:off]); n > budget { + t.Errorf("piece [%d:%d] is %d tokens, over budget %d", prev, off, n, budget) + } + prev = off + } +} + +// --- CI coverage without the 7 MB file --- +// +// The golden-count tests above need the real voyage-code-3 tokenizer.json, +// which is not in the repo, so they skip on a clean checkout. The mechanics — +// pre-token splitting, merge application, the additivity SplitPoints relies on +// — do not need that vocabulary. A hand-built merge table exercises them, so +// CI still fails if the splitter or the merge loop regresses. +func syntheticCounter(t *testing.T) *Counter { + t.Helper() + // Merges are ranked: "a b" collapses first, then "ab c". + c, err := LoadBytes(syntheticJSON("a b", "ab c", "f u", "fu n")) + if err != nil { + t.Fatalf("LoadBytes: %v", err) + } + return c +} + +// TestSyntheticMergesApply — with only "a b" known, "abc" costs one merge plus +// the leftover byte; the second merge then folds that leftover in. +func TestSyntheticMergesApply(t *testing.T) { + c := syntheticCounter(t) + if got, want := c.Count("abc"), 1; got != want { + t.Errorf(`Count("abc") = %d, want %d (a+b -> ab, ab+c -> abc)`, got, want) + } + if got, want := c.Count("ab"), 1; got != want { + t.Errorf(`Count("ab") = %d, want %d`, got, want) + } + if got, want := c.Count("acb"), 3; got != want { + t.Errorf(`Count("acb") = %d, want %d (no merge applies)`, got, want) + } +} + +// TestSyntheticAdditivity is the property the whole splitter rests on: BPE +// never merges across a pre-token boundary, so counts add up. If a future +// change made merges span boundaries, cuts would silently produce over-budget +// pieces — this catches it without needing the real vocabulary. +func TestSyntheticAdditivity(t *testing.T) { + c := syntheticCounter(t) + const text = "abc abc\n\tabc fun fun" + whole := c.Count(text) + + offsets, total := c.SplitPoints(text, 3) + if total != whole { + t.Fatalf("SplitPoints total %d != Count %d", total, whole) + } + sum, prev := 0, 0 + for _, off := range append(offsets, len(text)) { + n := c.Count(text[prev:off]) + if n > 3 { + t.Errorf("piece %q is %d tokens, over budget 3", text[prev:off], n) + } + sum += n + prev = off + } + if sum != whole { + t.Errorf("pieces sum to %d, whole is %d", sum, whole) + } +} + +// TestPreTokenBoundaries pins the hand-rolled splitter against the branches of +// the Qwen2 pattern that a RE2 rewrite gets wrong — whitespace runs, and a tab +// that must NOT be absorbed into the punctuation branch. +func TestPreTokenBoundaries(t *testing.T) { + for _, tc := range []struct { + in string + want []string + }{ + {"a b", []string{"a", " b"}}, + {"a b", []string{"a", " ", " b"}}, + // A tab is a legal single-character prefix for the letter branch + // ([^\r\n\p{L}\p{N}]?\p{L}+), so it attaches to what follows. + {"x\n\ty", []string{"x", "\n", "\ty"}}, + // The lookahead branch \s+(?!\S) matches a whitespace run only when + // nothing non-space follows it. The first tab qualifies (a tab + // follows); the second does not (a quote follows) and falls through + // to plain \s+. Hence two separate pre-tokens, not one run — this is + // precisely what a RE2 rewrite of the pattern gets wrong. + {"\t\t\"a\"", []string{"\t", "\t", "\"a", "\""}}, + {"it's", []string{"it", "'s"}}, + {"a1", []string{"a", "1"}}, + } { + var got []string + s := tc.in + for len(s) > 0 { + n := nextToken(s) + if n <= 0 { + t.Fatalf("nextToken(%q) returned %d", s, n) + } + got = append(got, s[:n]) + s = s[n:] + } + if len(got) != len(tc.want) { + t.Errorf("split(%q) = %q, want %q", tc.in, got, tc.want) + continue + } + for i := range got { + if got[i] != tc.want[i] { + t.Errorf("split(%q) = %q, want %q", tc.in, got, tc.want) + break + } + } + } +} + +// TestNFDPiecesRespectBudget covers input that is not already NFC, where the +// normalised copy counting works on has different byte offsets from the +// caller's string. Before the fix, offsets computed against the normalised +// copy were returned as-is: decomposed text made them land mid-rune, and the +// composition exclusions at U+0958..U+095F (which DEcompose under NFC, making +// the normalised form longer) pushed them past the end of the original, so the +// caller's slice expression panicked. Every other fixture in this file is +// ASCII or already-NFC and could not see it. +func TestNFDPiecesRespectBudget(t *testing.T) { + c := load(t) + for _, raw := range []string{ + strings.Repeat("// café comment here\n", 200), + strings.Repeat("x क़ख़ग़ ", 400), + strings.Repeat("Ώ ", 900), + } { + offs, total := c.SplitPoints(raw, 50) + prev := 0 + for _, off := range append(offs, len(raw)) { + if off > len(raw) || off < prev { + t.Fatalf("bad offset %d (len %d, prev %d)", off, len(raw), prev) + } + if n := c.Count(raw[prev:off]); n > 50 { + t.Errorf("piece [%d:%d] is %d tokens, over budget 50", prev, off, n) + } + prev = off + } + if total != c.Count(raw) { + t.Errorf("total %d != Count %d", total, c.Count(raw)) + } + } +} + +// TestSplitInsideChargesTail — an over-budget pre-token used to leave its tail +// uncounted, letting the next piece stack a full budget on top of it. +func TestSplitInsideChargesTail(t *testing.T) { + c, err := LoadBytes(syntheticJSON("a b")) + if err != nil { + t.Fatal(err) + } + in := strings.Repeat("x", 10) + " " + strings.Repeat("y", 4) + offs, _ := c.SplitPoints(in, 5) + prev := 0 + for _, off := range append(offs, len(in)) { + if n := c.Count(in[prev:off]); n > 5 { + t.Errorf("piece %q is %d tokens, over budget 5", in[prev:off], n) + } + prev = off + } +} + +// syntheticJSON builds a tokenizer.json with a hand-picked merge table and the +// real pipeline sections, so LoadBytes's compatibility check sees what it +// expects. Tests that only exercise merging still have to declare the pipeline +// they are pretending to be — which is the point of the check. +func syntheticJSON(merges ...string) []byte { + doc := map[string]any{ + "model": map[string]any{"type": "BPE", "merges": merges}, + "normalizer": map[string]any{"type": "NFC"}, + "pre_tokenizer": map[string]any{ + "type": "Sequence", + "pretokenizers": []any{ + map[string]any{"type": "Split", "pattern": map[string]any{"Regex": qwen2SplitPattern}}, + map[string]any{"type": "ByteLevel"}, + }, + }, + } + b, err := json.Marshal(doc) + if err != nil { + panic(err) + } + return b +} + +// TestRejectsForeignPipeline — a GPT-2 or o200k tokenizer.json parses cleanly +// and declares BPE, but its pre-tokenizer is not the one implemented here. It +// must be refused rather than counted wrongly. +func TestRejectsForeignPipeline(t *testing.T) { + for name, doc := range map[string]string{ + "gpt2 (no Split stage)": `{"model":{"type":"BPE","merges":["a b"]}, + "normalizer":null,"pre_tokenizer":{"type":"ByteLevel"}}`, + "o200k (different pattern)": `{"model":{"type":"BPE","merges":["a b"]}, + "normalizer":{"type":"NFC"},"pre_tokenizer":{"type":"Sequence","pretokenizers":[ + {"type":"Split","pattern":{"Regex":"[^\\r\\n\\p{L}\\p{N}]?[\\p{L}]+"}}, + {"type":"ByteLevel"}]}}`, + } { + if _, err := LoadBytes([]byte(doc)); err == nil { + t.Errorf("%s: expected a load error, got none", name) + } + } +} + +// TestMultibyteRunsTerminate covers runs of multi-byte runes long enough to +// exceed the budget as a single pre-token — a box-drawing comment separator, +// an arrow run, a run of combining marks. +// +// The byte-offset binary search this replaced aligned its midpoint to a rune +// start by DECREMENTING, so when alignment pulled the midpoint below lo, the +// next lo = mid+1 did not advance and the search spun forever. It took no +// error path and produced no output: the indexing worker simply stopped. All +// three inputs below hung at budgets 5 and 50. +func TestMultibyteRunsTerminate(t *testing.T) { + c := load(t) + inputs := map[string]string{ + "box drawing separator": "// " + strings.Repeat("\u2500", 400) + "\n", + "arrow run": strings.Repeat("\u2192", 400), + "combining marks": strings.Repeat("\u0301", 50), + "composition exclusion": strings.Repeat("\u0958", 2000), + } + for _, budget := range []int{5, 50} { + for name, in := range inputs { + done := make(chan struct{}) + var offs []int + go func(s string, b int) { + offs, _ = c.SplitPoints(s, b) + close(done) + }(in, budget) + + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatalf("%s at budget %d: SplitPoints did not return", name, budget) + } + + prev := 0 + for _, off := range append(offs, len(in)) { + if off > len(in) || off < prev { + t.Fatalf("%s: offset %d out of range (len %d, prev %d)", name, off, len(in), prev) + } + if !utf8.ValidString(in[prev:off]) { + t.Errorf("%s: piece [%d:%d] is not valid UTF-8 — cut mid-rune", name, prev, off) + } + prev = off + } + } + } +} diff --git a/server/internal/tokenizer/budget.go b/server/internal/tokenizer/budget.go new file mode 100644 index 00000000..90d6406c --- /dev/null +++ b/server/internal/tokenizer/budget.go @@ -0,0 +1,49 @@ +// Package tokenizer carries the model-token knowledge the chunker needs and +// the embedding providers own. +// +// The chunker decides WHERE to cut; only the provider knows WHAT the model +// counts. Keeping the two apart is what lets a model change without teaching +// the chunker about byte-level BPE, SentencePiece, or llama-server's +// /tokenize endpoint. +package tokenizer + +// Budget is the whole surface the chunker sees. One interface rather than a +// mandatory one plus an optional splitter: a single constructor argument, and +// callers that need both never have to type-assert. +// +// Cost note, because the two methods look interchangeable and are not: +// CountTokens and SplitPoints do the same single left-to-right pass over the +// same memo, so neither is algorithmically cheaper. What differs is +// allocation. CountTokens returns an int and allocates nothing; SplitPoints +// must build a slice of offsets — for a 60 KB input that is on the order of +// 15k entries. CountTokens runs on every chunk (1.9M of them on the reference +// corpus) while SplitPoints runs only on inputs that exceed the model's +// context (5 of that same 1.9M). Call CountTokens on the hot path and reach +// for SplitPoints only once a text is known not to fit. +// +// A third shortcut avoids both: byte-level BPE cannot emit a token covering +// less than one byte, so len(text) <= budget PROVES the text fits, with no +// tokenisation at all. Use it before calling anything here. +type Budget interface { + // MaxInputTokens is the model's context window for a single input. + MaxInputTokens() int + + // ExactCounts reports whether CountTokens and SplitPoints are exact. + // False means the provider has no tokenizer and is estimating from + // byte length: counts may be wrong in both directions and split points + // are byte windows, not token boundaries. Callers that need a + // guarantee must widen their safety margin when this is false — + // silently trusting an estimate is what the byte-window splitter used + // to do, and it produced averaged vectors nobody could see was wrong. + ExactCounts() bool + + // CountTokens returns the number of tokens the model will charge for. + CountTokens(s string) int + + // SplitPoints returns byte offsets at which s must be cut so no piece + // exceeds budget tokens, plus the total token count of s. Offsets, not + // substrings, so the caller keeps ownership of the metadata that hangs + // off those positions — line numbers, symbol names, byte ranges. + // A nil offsets slice means s already fits. + SplitPoints(s string, budget int) (offsets []int, total int) +} diff --git a/server/internal/vectorstore/chromemimport.go b/server/internal/vectorstore/chromemimport.go index dceaa124..64bd5fb8 100644 --- a/server/internal/vectorstore/chromemimport.go +++ b/server/internal/vectorstore/chromemimport.go @@ -313,6 +313,25 @@ func (s *Store) importCollection(ctx context.Context, dir, name string) (int, er if err := tx.QueryRowContext(ctx, `SELECT id FROM collections WHERE name = ?`, name).Scan(&collID); err != nil { return 0, err } + // No compact scan copy is written here, deliberately: the import is already + // the slowest thing a boot can do, and the background backfill that runs + // right after it (see startQ8Backfill) converts the result at a duty cycle + // that leaves the server usable. Until then those collections search the + // float32 way, which is what they did before that table existed. + // + // But "the import creates the collection, so nothing marked it complete" + // is only true when the collection is NEW. INSERT OR IGNORE also succeeds + // against a collection this binary created and flagged earlier — an + // operator pointing CIX_CHROMA_PERSIST_DIR at a legacy tree after indexing + // the same project live reaches exactly that, because migration_state is + // keyed on the legacy collection name and has never seen it. Imported docs + // would then have no compact rows inside a collection whose flag says it + // is complete, and the backfill skips flagged collections: permanently + // invisible to search. So the flag comes off here, unconditionally. + if err := clearQ8Ready(ctx, tx, collID); err != nil { + return 0, err + } + s.forgetQ8(collID) vecStmt, err := tx.PrepareContext(ctx, upsertVectorSQL) if err != nil { return 0, err diff --git a/server/internal/vectorstore/layout_test.go b/server/internal/vectorstore/layout_test.go new file mode 100644 index 00000000..89eb09ae --- /dev/null +++ b/server/internal/vectorstore/layout_test.go @@ -0,0 +1,170 @@ +package vectorstore + +import ( + "context" + "fmt" + "math/rand" + "strings" + "testing" +) + +// --------------------------------------------------------------------------- +// Page layout — what a scan's cost is actually made of. +// +// Search latency on a developer's laptop is not a portable number: it is a +// statement about that machine's disk, page cache and core count. What IS +// portable is how many database pages a full-collection scan is obliged to +// touch, because SQLite's layout rules are the same everywhere. Multiply pages +// by the target machine's read throughput and you have its latency; assert on +// pages and the assertion means the same thing in CI, on a Mac with NVMe, and +// on the 2-vCPU production box whose page cache is smaller than its index. +// --------------------------------------------------------------------------- + +// scanPages reports the pages a full scan must read, from whichever table the +// scan actually walks. +// +// dbstat walks the b-tree page by page, which is the only way to see overflow +// chains: they show up in no COUNT, in no per-table file size, and a row that +// spills is indistinguishable from one that does not until you look at its +// pages. +func scanPages(t *testing.T, s *Store, table string) (leaf, overflow, rows int64) { + t.Helper() + err := s.db.QueryRow(`SELECT COALESCE(SUM(pagetype='leaf'), 0), + COALESCE(SUM(pagetype='overflow'), 0) + FROM dbstat WHERE name = ?`, table).Scan(&leaf, &overflow) + if err != nil { + t.Fatalf("dbstat(%s): %v", table, err) + } + if err := s.db.QueryRow(`SELECT COUNT(*) FROM ` + table).Scan(&rows); err != nil { + t.Fatalf("count(%s): %v", table, err) + } + return leaf, overflow, rows +} + +// scanTable is the table a search walks, and TestLayoutMeasuresTheScannedTable +// checks that scanQ8SQL still names it — otherwise a change of scan source +// would leave every measurement below pointed at a table nobody reads, and the +// numbers would keep passing while meaning nothing. +const scanTable = "vectors_q8" + +// fillDim writes n rows of dimension dim into one collection. +func fillDim(t *testing.T, s *Store, project string, n, dim int) { + t.Helper() + r := rand.New(rand.NewSource(7)) + chunks := make([]Chunk, n) + embs := make([][]float32, n) + for i := range chunks { + chunks[i] = Chunk{ + Content: "package main\n\nfunc main() {}\n", + FilePath: fmt.Sprintf("pkg/mod%03d/file%03d.go", i%16, i), + StartLine: i*10 + 1, + EndLine: i*10 + 9, + ChunkType: "function", + SymbolName: fmt.Sprintf("Handler%03d", i), + Language: "go", + } + embs[i] = randNorm(r, dim) + } + if err := s.UpsertChunks(context.Background(), project, chunks, embs); err != nil { + t.Fatalf("upsert: %v", err) + } +} + +// TestScanPackingEfficiency pins how much of what a scan reads is the data it +// came for. +// +// SQLite keeps a row inside its leaf page only while the payload fits +// usable-35 bytes (8157 on our 8 KiB pages). Past that it keeps about a +// kilobyte local and puts the rest in a chain of overflow pages. Neither the +// row size nor the crossing of that line is visible anywhere in the schema — +// and the row size is set by an operator choosing output_dimension in a config +// file. Measured on this schema, three dimensions behave in three different +// ways: +// +// 768 3.1 kB row two rows share a leaf page 4096 B/vector 1.33x +// 1024 4.1 kB row one row per leaf page 8192 B/vector 2.00x +// 2048 8.2 kB row leaf slice + one overflow page 9216 B/vector 1.12x +// +// The interesting line is 1024, not 2048. Overflow sounds like the pathology +// and is not: at 2048 the overflow page is nearly full, so the scan reads +// 9216 bytes to obtain 8192 useful ones. At 1024 nothing overflows and the +// scan still reads 8192 bytes to obtain 4096, because two 4.1 kB rows cannot +// share an 8 KiB page and the second half of every page is air. An operator +// who halves output_dimension to save disk and time gets half the vector +// quality for 89% of the I/O. +// +// The assertion is the ratio, so it survives a change of dimension, of page +// size, or of which columns live in this table. +func TestScanPackingEfficiency(t *testing.T) { + // A scan that reads more than 1.4x the bytes it needs is spending more on + // structure than any layout choice should cost. 1.33x — two rows to a + // page with the page header and cell pointers on top — is what a healthy + // packing looks like, and is what both 768-dim float32 and 2048-dim int8 + // achieve. + const maxRatio = 1.4 + + for _, dim := range []int{768, 1024, 2048} { + t.Run(fmt.Sprintf("dim%d", dim), func(t *testing.T) { + s := openStore(t) + fillDim(t, s, "/layout", 400, dim) + + leaf, overflow, rows := scanPages(t, s, scanTable) + perVec := float64((leaf+overflow)*pageSize) / float64(rows) + // One byte per component: what the scan reads, against what it + // reads it for. + payload := float64(dim) + ratio := perVec / payload + + t.Logf("dim=%d rows=%d leaf=%d overflow=%d %.0f B/vector %.2fx payload", + dim, rows, leaf, overflow, perVec, ratio) + + if ratio > maxRatio { + t.Errorf("scan reads %.0f B per %d-dim vector to obtain %.0f B of embedding "+ + "(%.2fx, limit %.2fx): leaf=%d overflow=%d over %d rows", + perVec, dim, payload, ratio, maxRatio, leaf, overflow, rows) + } + }) + } +} + +// TestScanBytesPerVectorBudget is the absolute number, and the one the search +// work exists to move. +// +// Packing efficiency says the scan wastes little; it says nothing about the +// scan being affordable. At 2048 dimensions a well-packed float32 scan still +// reads 9 kB per vector, and a workspace query scans every collection: on the +// 45-repo fixture that is 1.9M vectors, about 17 GB of reads for one search. +// No page cache on an 8 GB box holds that, so production pays it at disk +// speed, every query, per repo. +// +// The budget below is what the scan costs when it reads a compact +// representation instead of the float32 original — the float32 blob stays on +// disk for anything that needs exact scores, but the scan stops reading it. +func TestScanBytesPerVectorBudget(t *testing.T) { + const dim = 2048 + // 2048 int8 components + row overhead, three rows to a page. + const maxBytesPerVector = 3072 + + s := openStore(t) + fillDim(t, s, "/budget", 400, dim) + + leaf, overflow, rows := scanPages(t, s, scanTable) + perVec := float64((leaf+overflow)*pageSize) / float64(rows) + t.Logf("dim=%d rows=%d leaf=%d overflow=%d %.0f B/vector", dim, rows, leaf, overflow, perVec) + + if perVec > maxBytesPerVector { + t.Errorf("scan reads %.0f B per vector at %d dims, budget %d B: "+ + "a 1.9M-vector workspace query moves %.1f GB instead of %.1f GB", + perVec, dim, maxBytesPerVector, + perVec*1.9e6/1e9, float64(maxBytesPerVector)*1.9e6/1e9) + } +} + +// TestLayoutMeasuresTheScannedTable keeps the two constants honest. dbstat is +// asked about a table by name, and a name is exactly the kind of thing that +// survives a refactor that moved the data somewhere else. +func TestLayoutMeasuresTheScannedTable(t *testing.T) { + if !strings.Contains(scanQ8SQL, " "+scanTable+" ") { + t.Fatalf("the scan reads a table other than %q:\n%s", scanTable, scanQ8SQL) + } +} diff --git a/server/internal/vectorstore/maintenance.go b/server/internal/vectorstore/maintenance.go index 9503bea4..3c798916 100644 --- a/server/internal/vectorstore/maintenance.go +++ b/server/internal/vectorstore/maintenance.go @@ -88,6 +88,26 @@ SELECT c.name, COALESCE(SUM(LENGTH(vc.content) + LENGTH(vc.doc_id)), 0) LEFT JOIN vector_contents vc ON vc.collection_id = c.id GROUP BY c.id` +// q8SizeSQL accounts for the compact scan copy. It is a third of the float32 +// bytes and it is real disk, so leaving it out would make the Resources screen +// under-report the store by ~25% — the same kind of quiet mismatch the WAL +// high-water mark used to cause. +// sizeExprQ8 is the compact copy's per-row logical byte count, in the same +// shape as sizeExprVectors and pasted in no more places than it is. +// +// Reading it walks the compact table's leaf pages — about a fifth of what the +// same question costs over `vectors`, and behind the maintenance service's TTL +// cache either way, so it is answered rarely. If that stops being true, the +// cheap replacement is recording the total at backfill completion rather than +// making the aggregate faster. +const sizeExprQ8 = `LENGTH(q.embedding) + LENGTH(q.doc_id) + LENGTH(q.language) + 16` + +const q8SizeSQL = ` +SELECT c.name, COALESCE(SUM(` + sizeExprQ8 + `), 0) + FROM collections c + LEFT JOIN vectors_q8 q ON q.collection_id = c.id + GROUP BY c.id` + // ListCollections implements Maintainer. Results are sorted by name so callers // (and their tests) see a stable order. func (s *Store) ListCollections() []CollectionInfo { @@ -118,29 +138,45 @@ func (s *Store) ListCollections() []CollectionInfo { return nil } - // Chunk text lives in its own table (see schemaSQL); fold it into the - // reported size with a second aggregate rather than a join that would - // multiply the row counts. - crows, err := s.db.QueryContext(ctx, contentSizeSQL) + // Chunk text and the compact scan copy live in their own tables (see + // schemaSQL); fold each into the reported size with its own aggregate + // rather than joins that would multiply the row counts. + for _, q := range []struct { + what string + sql string + }{ + {"contents", contentSizeSQL}, + {"scan copy", q8SizeSQL}, + } { + sizes, err := s.sizesByCollection(ctx, q.sql) + if err != nil { + s.logger.Error("vectorstore: list collection "+q.what, "err", err) + return out + } + for i := range out { + out[i].SizeBytes += sizes[out[i].Name] + } + } + return out +} + +// sizesByCollection runs one name -> bytes aggregate. +func (s *Store) sizesByCollection(ctx context.Context, query string) (map[string]int64, error) { + rows, err := s.db.QueryContext(ctx, query) if err != nil { - s.logger.Error("vectorstore: list collection contents", "err", err) - return out + return nil, err } - defer crows.Close() - sizes := make(map[string]int64, len(out)) - for crows.Next() { + defer rows.Close() + sizes := map[string]int64{} + for rows.Next() { var name string var n int64 - if err := crows.Scan(&name, &n); err != nil { - s.logger.Error("vectorstore: list collection contents", "err", err) - return out + if err := rows.Scan(&name, &n); err != nil { + return nil, err } sizes[name] = n } - for i := range out { - out[i].SizeBytes += sizes[out[i].Name] - } - return out + return sizes, rows.Err() } // CollectionSizeBytes implements Maintainer. @@ -155,7 +191,7 @@ func (s *Store) CollectionSizeBytes(projectPath string) (int64, bool) { if err != nil || !ok { return 0, false } - var vecBytes, contentBytes int64 + var vecBytes, contentBytes, q8Bytes int64 if err := s.db.QueryRowContext(ctx, ` SELECT COALESCE(SUM(`+sizeExprVectors+`), 0) FROM vectors v WHERE v.collection_id = ?`, collID).Scan(&vecBytes); err != nil { @@ -166,7 +202,12 @@ func (s *Store) CollectionSizeBytes(projectPath string) (int64, bool) { FROM vector_contents WHERE collection_id = ?`, collID).Scan(&contentBytes); err != nil { return 0, false } - return vecBytes + contentBytes, true + if err := s.db.QueryRowContext(ctx, ` + SELECT COALESCE(SUM(`+sizeExprQ8+`), 0) + FROM vectors_q8 q WHERE q.collection_id = ?`, collID).Scan(&q8Bytes); err != nil { + return 0, false + } + return vecBytes + contentBytes + q8Bytes, true } // DeleteCollectionByName implements Maintainer. @@ -196,6 +237,8 @@ func (s *Store) DeleteCollectionByName(name string) error { for _, stmt := range []string{ `DELETE FROM vector_contents WHERE collection_id = ?`, + `DELETE FROM vectors_q8 WHERE collection_id = ?`, + `DELETE FROM q8_state WHERE collection_id = ?`, `DELETE FROM vectors WHERE collection_id = ?`, `DELETE FROM collections WHERE id = ?`, } { @@ -207,6 +250,7 @@ func (s *Store) DeleteCollectionByName(name string) error { return fmt.Errorf("vectorstore delete collection %q: %w", name, err) } s.forgetCollection(name) + s.forgetQ8(collID) return nil } diff --git a/server/internal/vectorstore/parity_test.go b/server/internal/vectorstore/parity_test.go index e140083c..915970d5 100644 --- a/server/internal/vectorstore/parity_test.go +++ b/server/internal/vectorstore/parity_test.go @@ -207,6 +207,19 @@ func TestSearchWhereFilterMirrorsChromemSemantics(t *testing.T) { t.Errorf("start_line filter returned %+v, want the single chunk starting at 11", got) } + // A KNOWN key with an empty value is a filter, not the absence of one: + // chromem compared metadata["language"] to "", which matches only rows + // whose language is empty. The compact scan supports exactly this one + // column, so it is the one place the two scan paths could disagree about + // what an empty value means. + got, err = s.Search(ctx, project, embs[0], 5, map[string]string{"language": ""}) + if err != nil { + t.Fatalf("empty language filter: %v", err) + } + if len(got) != 0 { + t.Errorf("language=\"\" returned %d results, want 0 — every chunk here is Go", len(got)) + } + // Several keys must all match. got, err = s.Search(ctx, project, embs[0], 5, map[string]string{"language": "go", "file_path": "b.go"}) diff --git a/server/internal/vectorstore/plan_test.go b/server/internal/vectorstore/plan_test.go index e2e5069b..9b9b8daa 100644 --- a/server/internal/vectorstore/plan_test.go +++ b/server/internal/vectorstore/plan_test.go @@ -74,3 +74,33 @@ func TestScanUsesCollectionIndex(t *testing.T) { t.Errorf("delete-by-file plan does not use idx_vec_coll_file:\n%s", plan) } } + +// TestQ8ScanUsesCollectionIndex is TestScanUsesCollectionIndex for the table a +// search actually walks. Same guarantee for the same reason — idx_q8_coll's +// keys are (collection_id, rowid), so the walk stays proportional to the +// collection and yields its rows in table order — and it matters more here, +// because this is the plan every search takes. +func TestQ8ScanUsesCollectionIndex(t *testing.T) { + s := openStore(t) + ctx := context.Background() + chunks, embs := makeChunks(20, "a.go", "go") + if err := s.UpsertChunks(ctx, "/q8plan", chunks, embs); err != nil { + t.Fatalf("upsert: %v", err) + } + + plan := queryPlan(t, s, scanQ8SQL, 1) + if !strings.Contains(plan, "idx_q8_coll") { + t.Errorf("compact scan plan does not use idx_q8_coll:\n%s", plan) + } + if strings.Contains(plan, "SCAN vectors_q8\n") || strings.HasSuffix(plan, "SCAN vectors_q8") { + t.Errorf("compact scan plan falls back to a full table scan:\n%s", plan) + } + + // The language filter must not change the driving index: it is an extra + // test on rows the index already visits, not a reason to pick a different + // one. + plan = queryPlan(t, s, scanQ8SQL+" AND language = ?", 1, "go") + if !strings.Contains(plan, "idx_q8_coll") { + t.Errorf("filtered compact scan plan does not use idx_q8_coll:\n%s", plan) + } +} diff --git a/server/internal/vectorstore/q8.go b/server/internal/vectorstore/q8.go new file mode 100644 index 00000000..93b4ca08 --- /dev/null +++ b/server/internal/vectorstore/q8.go @@ -0,0 +1,448 @@ +package vectorstore + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" +) + +// --------------------------------------------------------------------------- +// The compact scan copy. +// +// vectors_q8 holds every vector at one byte per component. A search scans it +// instead of the float32 table (3.4x fewer bytes, measured — see schemaSQL), +// takes a shortlist, and rescores that shortlist against the float32 +// originals, which is what keeps the answer exact. +// +// Everything here exists to answer one question cheaply and correctly: does +// this collection have a q8 row for every vector it has? The answer must not +// cost a COUNT per query, and it must never be "yes" when it is not — a +// half-built q8 table would silently hide documents from search, and a store +// that quietly returns fewer results is worse than a slow one. +// +// The invariant is maintained from both ends: +// +// - A collection created by this code is born complete: it has no rows, so +// the empty q8 side matches it, and ensureCollection records that. Every +// upsert afterwards writes both tables in one transaction, so the property +// is preserved by construction and never has to be re-checked. +// - A collection that predates this table has rows and no q8_state row. It +// stays on the float32 scan — correct, just slower — until the backfill +// has quantised all of it and records completion in the same transaction +// as the last batch. +// +// Nothing sets the flag optimistically, and nothing reads q8 without it. +// --------------------------------------------------------------------------- + +// backfillQ8SQL writes one converted vector, and is deliberately weaker than +// the upsert the write path uses. +// +// The backfill reads a batch, quantises it, and writes it in a separate +// transaction. Anything can commit in that gap — the file watcher reindexing a +// saved file, an admin deleting a project — so the write has to be harmless +// against both, without taking a lock that would make the watcher wait on a +// background job. Two clauses do that: +// +// - WHERE EXISTS: a doc deleted in the gap is not resurrected. Without it the +// backfill would reinsert compact rows whose float32 originals are gone — +// rows that DeleteByFile can never clean up afterwards, because its +// subquery finds doc_ids through `vectors`. Every later scan would +// shortlist them and every rescore would silently drop them, which reads +// as a search returning fewer results for no stated reason. +// - DO NOTHING: a doc re-embedded in the gap keeps the compact row upsertBatch +// wrote from its NEW embedding, instead of being overwritten by this batch's +// quantisation of the old one. The backfill only ever fills gaps; it is +// never the more recent writer. +const backfillQ8SQL = `INSERT INTO vectors_q8 (collection_id, doc_id, language, scale, embedding) + SELECT ?,?,?,?,? WHERE EXISTS (SELECT 1 FROM vectors WHERE collection_id = ? AND doc_id = ?) + ON CONFLICT(collection_id, doc_id) DO NOTHING` + +// q8BackfillBatch is how many vectors one backfill transaction converts. +// +// At 2048 dimensions this reads ~16 MB and writes ~4 MB per batch. Small +// enough that a writer (the indexer, the file watcher) never waits long for +// the write lock, large enough that the per-transaction overhead is noise. +const q8BackfillBatch = 2000 + +// q8BackfillDuty is the fraction of wall-clock the backfill is allowed to +// spend working. It sleeps for the rest. +// +// The backfill competes with live searches for the same disk and the same two +// vCPUs on the production box, and it is never urgent: until it finishes, the +// affected collections simply search the way they did before. Yielding half +// the time turns "the server is unusable for ten minutes after an upgrade" +// into "it is a bit slower for twenty". +const q8BackfillDuty = 0.5 + +// markQ8Ready records that a collection's q8 rows are complete. +// +// INSERT OR IGNORE, so calling it for a collection already marked is free and +// keeps the original timestamp — which is the one that says when the data was +// actually built. +func markQ8Ready(ctx context.Context, tx *sql.Tx, collID int64) error { + _, err := tx.ExecContext(ctx, + `INSERT OR IGNORE INTO q8_state(collection_id, built_at) VALUES(?, ?)`, + collID, time.Now().UTC().Format(time.RFC3339)) + if err != nil { + return fmt.Errorf("vectorstore: mark q8 ready for collection %d: %w", collID, err) + } + return nil +} + +// markCollectionQ8Ready records completion outside a caller-owned +// transaction, and updates the in-memory cache so the very next search on a +// freshly created collection takes the fast path. +func (s *Store) markCollectionQ8Ready(ctx context.Context, collID int64) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck // no-op after a successful Commit + if err := markQ8Ready(ctx, tx, collID); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + s.q8Mu.Lock() + s.q8State[collID] = true + s.q8Mu.Unlock() + return nil +} + +// clearQ8Ready withdraws a collection's completion flag. +func clearQ8Ready(ctx context.Context, tx *sql.Tx, collID int64) error { + if _, err := tx.ExecContext(ctx, `DELETE FROM q8_state WHERE collection_id = ?`, collID); err != nil { + return fmt.Errorf("vectorstore: clear q8 state for collection %d: %w", collID, err) + } + return nil +} + +// clearCollectionQ8Ready withdraws a collection's completion flag and forgets +// the cached answer, so the very next search falls back to the exact scan. +func (s *Store) clearCollectionQ8Ready(ctx context.Context, collID int64) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck // no-op after a successful Commit + if err := clearQ8Ready(ctx, tx, collID); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + s.forgetQ8(collID) + return nil +} + +// q8Ready reports whether the scan may read vectors_q8 for this collection. +// +// Cached in memory because it is consulted on every search and the answer only +// ever changes in one direction (not ready -> ready, once, when the backfill +// finishes). A false answer is therefore worth re-checking; a true one is not. +func (s *Store) q8Ready(ctx context.Context, collID int64) bool { + if !s.scanQuant { + return false + } + s.q8Mu.Lock() + _, ready := s.q8State[collID] + s.q8Mu.Unlock() + if ready { + return true + } + + // Only positives are cached, and presence IS the answer — there is no + // stored false to accidentally honour. A collection that is not ready yet + // may become ready at any moment (the backfill is running), so a negative + // has to be re-asked; a positive never reverts without going through + // forgetQ8. + var one int + err := s.db.QueryRowContext(ctx, + `SELECT 1 FROM q8_state WHERE collection_id = ?`, collID).Scan(&one) + switch { + case err == nil: + case errors.Is(err, sql.ErrNoRows): + return false + default: + // A probe that errors must not upgrade the scan: falling back to the + // float32 path answers the query correctly. + s.logger.Warn("vectorstore: q8 readiness probe failed", "collection_id", collID, "err", err) + return false + } + s.q8Mu.Lock() + s.q8State[collID] = true + s.q8Mu.Unlock() + return true +} + +// forgetQ8 drops a collection's cached readiness (after a delete). The next +// collection to be handed this id — which AUTOINCREMENT guarantees is never +// this one — must not inherit its answer. +func (s *Store) forgetQ8(collID int64) { + s.q8Mu.Lock() + delete(s.q8State, collID) + s.q8Mu.Unlock() +} + +// startQ8Backfill converts collections written before vectors_q8 existed. +// +// Runs in the background and returns immediately: the store is fully usable +// while it works, because an unconverted collection is not broken, only slow. +// That is the whole reason this is a background job and not part of Open — the +// alternative is a server that answers nothing for the minutes it takes to +// rewrite a multi-gigabyte store, which is exactly the failure mode the schema +// rebuild already has and which took three false "the server is down" reports +// to diagnose. +func (s *Store) startQ8Backfill(ctx context.Context) { + go func() { + if err := s.backfillQ8(ctx); err != nil && ctx.Err() == nil { + // Warn, not fatal: every collection it failed to convert keeps + // searching the float32 way. + s.logger.Warn("vectorstore: building the compact scan index stopped early", "err", err) + } + }() +} + +// pendingQ8Collections lists collections that have vectors but no completed q8 +// copy, largest first — so the collection whose searches hurt most is the +// first one to get faster. +func (s *Store) pendingQ8Collections(ctx context.Context) ([]int64, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT v.collection_id, COUNT(*) n + FROM vectors v + WHERE v.collection_id NOT IN (SELECT collection_id FROM q8_state) + GROUP BY v.collection_id + ORDER BY n DESC`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []int64 + for rows.Next() { + var id int64 + var n int64 + if err := rows.Scan(&id, &n); err != nil { + return nil, err + } + out = append(out, id) + } + return out, rows.Err() +} + +// backfillQ8 quantises every pending collection. +func (s *Store) backfillQ8(ctx context.Context) error { + if !s.acquire() { + return nil + } + pending, err := s.pendingQ8Collections(ctx) + s.release() + if err != nil { + return fmt.Errorf("list collections needing a scan copy: %w", err) + } + if len(pending) == 0 { + return nil + } + + // The scan copy is a quarter of the float32 bytes it is derived from. + // Refusing up front beats discovering it a gigabyte in: a failed backfill + // leaves the store fully working, but it also leaves the disk fuller than + // it needs to be and the failure buried in a log line. + if need, err := s.pendingQ8Bytes(ctx); err == nil { + if err := checkFreeSpace(s.dir, need); err != nil { + s.logger.Warn("vectorstore: skipping the compact scan index — not enough free disk", + "db", s.dbPath, "need_mb", need/(1<<20), "err", err) + return nil + } + } + + // WARN for the same reason the schema rebuild and the legacy import log at + // warn: production runs at warn level, and unexplained background I/O on a + // box that was just restarted is indistinguishable from a problem. + started := time.Now() + s.logger.Warn("vectorstore: building the compact scan index in the background", + "db", s.dbPath, "collections", len(pending)) + + var converted int64 + var failed int + for _, collID := range pending { + n, err := s.backfillCollection(ctx, collID) + converted += n + if err != nil { + if ctx.Err() != nil { + return nil + } + // One collection's failure must not cost the other forty-two. + // The realistic cause is a collection deleted out from under the + // walk — an admin removing a project, or the orphan sweep — which + // makes the next insert fail its foreign key. Aborting there would + // leave every collection after it on the float32 scan until + // somebody restarted the server, and the only trace would be one + // warn line. + failed++ + s.logger.Warn("vectorstore: could not build the compact scan index for a collection", + "db", s.dbPath, "collection_id", collID, "err", err) + continue + } + } + s.logger.Warn("vectorstore: compact scan index built", + "db", s.dbPath, "collections", len(pending)-failed, "failed", failed, + "vectors", converted, "took", time.Since(started).Round(time.Second)) + return nil +} + +// pendingQ8Bytes estimates the disk the backfill will add: one byte per +// component of every vector it has to convert, plus the row overhead the size +// accounting already uses for the compact table. +func (s *Store) pendingQ8Bytes(ctx context.Context) (int64, error) { + if !s.acquire() { + return 0, ErrClosed + } + defer s.release() + var n int64 + // LENGTH(embedding)/4 is the compact row's blob length derived from the + // float32 one — same arithmetic as sizeExprQ8, from the only table that + // has the rows yet. + err := s.db.QueryRowContext(ctx, ` + SELECT COALESCE(SUM(LENGTH(embedding)/4 + LENGTH(doc_id) + LENGTH(language) + 16), 0) + FROM vectors + WHERE collection_id NOT IN (SELECT collection_id FROM q8_state)`).Scan(&n) + return n, err +} + +// backfillCollection walks one collection in doc_id order, quantising as it +// goes, and marks the collection ready in the same transaction as its last +// batch — so a kill at any point leaves a collection that is unmarked and +// therefore still searchable the float32 way, never one that is marked and +// incomplete. +func (s *Store) backfillCollection(ctx context.Context, collID int64) (int64, error) { + var ( + after int64 + converted int64 + ) + for { + if ctx.Err() != nil { + return converted, ctx.Err() + } + batchStart := time.Now() + n, last, err := s.backfillBatch(ctx, collID, after) + if err != nil { + return converted, err + } + converted += n + if n == 0 { + return converted, nil + } + after = last + + // Yield. Sleeping proportionally to the work just done keeps the duty + // cycle honest whether a batch took 40 ms on an NVMe laptop or four + // seconds on a network disk. + select { + case <-ctx.Done(): + return converted, ctx.Err() + case <-time.After(time.Duration(float64(time.Since(batchStart)) * (1 - q8BackfillDuty) / q8BackfillDuty)): + } + } +} + +// backfillBatch converts up to q8BackfillBatch vectors whose rowid is above +// `after`, and returns how many it converted and the last rowid it saw. +// +// Keyset pagination rather than OFFSET, so resuming does not re-read +// everything before the cursor. On ROWID and idx_vec_coll rather than on +// doc_id: `vectors` is a rowid table whose composite primary key is a separate +// index, so paging in doc_id order would look up ~9 kB rows scattered across +// the collection's whole rowid span — the same 1.8x that made scanSQL pick +// idx_vec_coll over idx_vec_coll_file (see sqlite.go). Rows inserted ahead of +// the cursor while the walk runs need no special handling: upsertBatch writes +// their compact copy itself. +// +// The batch does NOT hold a lock across its read and its write, and does not +// need to. Two statement-level rules make the gap between them harmless: +// the insert is conditional on the vectors row still existing, so a delete +// that commits in the gap cannot be undone; and it never overwrites an +// existing compact row, so an upsert that commits in the gap keeps its own +// fresher quantisation instead of being clobbered by this one's stale copy. +func (s *Store) backfillBatch(ctx context.Context, collID int64, after int64) (int64, int64, error) { + if !s.acquire() { + return 0, 0, ErrClosed + } + defer s.release() + + type q8Row struct { + docID string + language string + scale float32 + blob []byte + } + batch := make([]q8Row, 0, q8BackfillBatch) + last := after + + // Read and quantise first, write second. Holding a write transaction open + // across a streaming read would keep the write lock for the whole batch, + // and the thing most likely to want that lock is the file watcher + // reindexing a file someone just saved. + rows, err := s.db.QueryContext(ctx, ` + SELECT rowid, doc_id, language, embedding FROM vectors INDEXED BY idx_vec_coll + WHERE collection_id = ? AND rowid > ? + ORDER BY rowid LIMIT ?`, collID, after, q8BackfillBatch) + if err != nil { + return 0, 0, fmt.Errorf("read vectors: %w", err) + } + var scratch []float32 + for rows.Next() { + var ( + rowID int64 + docID, language string + raw sql.RawBytes + ) + if err := rows.Scan(&rowID, &docID, &language, &raw); err != nil { + rows.Close() + return 0, 0, fmt.Errorf("scan vector: %w", err) + } + var vec []float32 + vec, scratch = blobFloats(raw, scratch) + // quantizeInt8 allocates its own output, so nothing here outlives the + // RawBytes it was derived from. + blob, scale := quantizeInt8(vec) + batch = append(batch, q8Row{docID: docID, language: language, scale: scale, blob: blob}) + last = rowID + } + if err := rows.Err(); err != nil { + rows.Close() + return 0, 0, fmt.Errorf("read vectors: %w", err) + } + rows.Close() + if len(batch) == 0 { + // The cursor ran off the end: every row this collection had when the + // walk started now has a compact copy, and every row written since got + // one from upsertBatch. Completeness does not rest on this statement + // being in the same transaction as anything — it rests on the two + // insert rules above, which hold whatever commits in between. + return 0, last, s.markCollectionQ8Ready(ctx, collID) + } + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, 0, err + } + defer tx.Rollback() //nolint:errcheck // no-op after a successful Commit + stmt, err := tx.PrepareContext(ctx, backfillQ8SQL) + if err != nil { + return 0, 0, err + } + defer stmt.Close() + for _, r := range batch { + if _, err := stmt.ExecContext(ctx, collID, r.docID, r.language, r.scale, r.blob, + collID, r.docID); err != nil { + return 0, 0, fmt.Errorf("write q8 row: %w", err) + } + } + if err := tx.Commit(); err != nil { + return 0, 0, err + } + return int64(len(batch)), last, nil +} diff --git a/server/internal/vectorstore/q8_test.go b/server/internal/vectorstore/q8_test.go new file mode 100644 index 00000000..7f3983c8 --- /dev/null +++ b/server/internal/vectorstore/q8_test.go @@ -0,0 +1,782 @@ +package vectorstore + +import ( + "context" + "fmt" + "math" + "math/rand" + "strings" + "sync" + "testing" + "time" +) + +// --------------------------------------------------------------------------- +// The compact scan copy. What has to be true of it: +// +// - the scores it reports are the exact ones, because the shortlist is +// rescored on the float32 originals; +// - it returns the same documents an exact scan would, which is an empirical +// property of the shortlist width and is therefore measured, not assumed; +// - a collection it has not covered yet still answers correctly; +// - deleting data deletes it here too, in both directions. +// --------------------------------------------------------------------------- + +// q8Corpus builds a collection that is hostile to quantisation: half the +// vectors are random, and the other half are near-duplicates of a few cluster +// centres, differing by less than the quantisation step. Random unit vectors +// in 2048 dimensions are almost orthogonal to each other and to any query, so +// a corpus of only those has no near-ties to misorder and would let any +// approximation look perfect. Real code corpora are the opposite: boilerplate, +// generated files and copied blocks produce exactly these clusters. +func q8Corpus(t *testing.T, s *Store, project string, n, dim int) ([]Chunk, [][]float32) { + t.Helper() + r := rand.New(rand.NewSource(11)) + centres := make([][]float32, 8) + for i := range centres { + centres[i] = randNorm(r, dim) + } + + chunks := make([]Chunk, n) + embs := make([][]float32, n) + langs := []string{"go", "python", "rust"} + for i := range chunks { + var v []float32 + if i%2 == 0 { + v = randNorm(r, dim) + } else { + base := centres[i%len(centres)] + v = make([]float32, dim) + for j := range v { + v[j] = base[j] + float32(r.NormFloat64())*1e-4 + } + v = normalizeVector(v) + } + chunks[i] = Chunk{ + Content: fmt.Sprintf("chunk %d", i), + FilePath: fmt.Sprintf("src/pkg%02d/f%04d.go", i%20, i), + StartLine: i*10 + 1, + EndLine: i*10 + 9, + ChunkType: "function", + SymbolName: fmt.Sprintf("Fn%04d", i), + Language: langs[i%len(langs)], + } + embs[i] = v + } + if err := s.UpsertChunks(context.Background(), project, chunks, embs); err != nil { + t.Fatalf("upsert: %v", err) + } + return chunks, embs +} + +// exactTopK is the oracle: the ranking a full float32 scan produces, computed +// in Go from the embeddings the test itself wrote. Deliberately not computed +// by asking the store to scan the other way — an oracle that shares code with +// the thing under test can agree with it about a shared mistake. +func exactTopK(chunks []Chunk, embs [][]float32, q []float32, k int, language string) []string { + type sc struct { + key string + score float32 + } + var all []sc + for i, e := range embs { + if language != "" && chunks[i].Language != language { + continue + } + all = append(all, sc{locKey(chunks[i]), dot(q, e)}) + } + for i := 1; i < len(all); i++ { + for j := i; j > 0 && all[j].score > all[j-1].score; j-- { + all[j], all[j-1] = all[j-1], all[j] + } + } + out := make([]string, 0, k) + for i := 0; i < k && i < len(all); i++ { + out = append(out, all[i].key) + } + return out +} + +// locKey identifies a chunk the way a caller sees it — the doc_id is internal. +func locKey(c Chunk) string { return fmt.Sprintf("%s:%d-%d", c.FilePath, c.StartLine, c.EndLine) } + +func resultKeys(rs []SearchResult) []string { + out := make([]string, len(rs)) + for i, r := range rs { + out[i] = fmt.Sprintf("%s:%d-%d", r.FilePath, r.StartLine, r.EndLine) + } + return out +} + +// TestSearchScoresAreExact is the property that does not depend on the corpus, +// the query or the shortlist width: whatever documents come back, the number +// attached to each is the true cosine against the stored float32 vector, not +// the int8 approximation that selected it. +// +// It matters because the score is not decoration. Callers threshold on it +// (min_score), the workspace fan-out normalises across projects with it, and +// hybrid search blends it with BM25. An approximate score would move results +// between projects in ways no per-project test would catch. +func TestSearchScoresAreExact(t *testing.T) { + const dim = 512 + s := openStore(t) + ctx := context.Background() + chunks, embs := q8Corpus(t, s, "/exact", 1500, dim) + + r := rand.New(rand.NewSource(99)) + for qi := 0; qi < 10; qi++ { + q := randNorm(r, dim) + got, err := s.Search(ctx, "/exact", q, 10, nil) + if err != nil { + t.Fatalf("search: %v", err) + } + if len(got) == 0 { + t.Fatal("no results") + } + byKey := map[string]float32{} + for i := range chunks { + byKey[locKey(chunks[i])] = dot(q, embs[i]) + } + for _, res := range got { + key := fmt.Sprintf("%s:%d-%d", res.FilePath, res.StartLine, res.EndLine) + want := round4(byKey[key]) + if math.Abs(float64(res.Score-want)) > 1e-4 { + t.Errorf("query %d: %s scored %v, exact cosine is %v — "+ + "the reported score came from the int8 approximation, not the rescore", + qi, key, res.Score, want) + } + } + } +} + +// TestSearchMatchesExactRanking measures what the shortlist width buys. +// +// The compact scan is an approximation and could in principle drop a document +// that belongs in the top K. q8Shortlist picks its width from a measurement on +// real data (see its comment); this is the same measurement in miniature, on a +// corpus built to contain the near-ties that quantisation actually confuses, +// and it fails loudly if a change to the width, the quantisation or the +// rescore starts losing documents. +func TestSearchMatchesExactRanking(t *testing.T) { + const ( + dim = 512 + k = 10 + ) + s := openStore(t) + ctx := context.Background() + chunks, embs := q8Corpus(t, s, "/rank", 2000, dim) + + collID, ok, err := s.collectionID(ctx, collectionName("/rank")) + if err != nil || !ok { + t.Fatalf("collection id: %v ok=%v", err, ok) + } + if !s.q8Ready(ctx, collID) { + t.Fatal("collection is not on the compact scan — this test would be measuring the float32 path") + } + + r := rand.New(rand.NewSource(7)) + for qi := 0; qi < 25; qi++ { + q := randNorm(r, dim) + got, err := s.Search(ctx, "/rank", q, k, nil) + if err != nil { + t.Fatalf("search: %v", err) + } + want := exactTopK(chunks, embs, q, k, "") + if strings.Join(resultKeys(got), ",") != strings.Join(want, ",") { + t.Errorf("query %d: compact scan returned a different top-%d than an exact scan\n got: %v\nwant: %v", + qi, k, resultKeys(got), want) + } + } +} + +// TestSearchLanguageFilterOnCompactScan pins the one metadata column the +// compact copy carries. It is duplicated from `vectors`, so it can drift; a +// filter that silently matched nothing would look like "no results for that +// language", which is a plausible answer and therefore an invisible bug. +func TestSearchLanguageFilterOnCompactScan(t *testing.T) { + const dim = 512 + s := openStore(t) + ctx := context.Background() + chunks, embs := q8Corpus(t, s, "/lang", 1200, dim) + + r := rand.New(rand.NewSource(3)) + q := randNorm(r, dim) + got, err := s.Search(ctx, "/lang", q, 10, map[string]string{"language": "rust"}) + if err != nil { + t.Fatalf("search: %v", err) + } + if len(got) == 0 { + t.Fatal("language filter returned nothing") + } + for _, res := range got { + if res.Language != "rust" { + t.Fatalf("language filter leaked a %q result", res.Language) + } + } + if want := exactTopK(chunks, embs, q, 10, "rust"); strings.Join(resultKeys(got), ",") != strings.Join(want, ",") { + t.Errorf("filtered compact scan disagrees with an exact filtered scan\n got: %v\nwant: %v", + resultKeys(got), want) + } +} + +// TestSearchUnsupportedFilterFallsBack covers the other half of q8Filterable: +// a filter the compact copy cannot express must take the float32 scan, which +// has every column, rather than be ignored. +func TestSearchUnsupportedFilterFallsBack(t *testing.T) { + const dim = 256 + s := openStore(t) + ctx := context.Background() + chunks, _ := q8Corpus(t, s, "/filter", 600, dim) + + r := rand.New(rand.NewSource(5)) + q := randNorm(r, dim) + target := chunks[123].FilePath + got, err := s.Search(ctx, "/filter", q, 10, map[string]string{"file_path": target}) + if err != nil { + t.Fatalf("search: %v", err) + } + if len(got) == 0 { + t.Fatal("file_path filter returned nothing — the fallback did not run") + } + for _, res := range got { + if res.FilePath != target { + t.Fatalf("file_path filter leaked %q", res.FilePath) + } + } +} + +// stripQ8 turns a store back into what an older binary would have left behind: +// float32 vectors, no compact copy, no completion flag. +func stripQ8(t *testing.T, s *Store) { + t.Helper() + for _, stmt := range []string{`DELETE FROM vectors_q8`, `DELETE FROM q8_state`} { + if _, err := s.db.Exec(stmt); err != nil { + t.Fatalf("%s: %v", stmt, err) + } + } + s.q8Mu.Lock() + s.q8State = map[int64]bool{} + s.q8Mu.Unlock() +} + +// TestSearchWithoutCompactCopy is the guarantee that makes the backfill safe +// to run in the background: a collection with no compact copy answers the same +// queries, correctly, the slow way. +func TestSearchWithoutCompactCopy(t *testing.T) { + const dim = 512 + s := openStore(t) + ctx := context.Background() + chunks, embs := q8Corpus(t, s, "/nocopy", 800, dim) + + r := rand.New(rand.NewSource(21)) + q := randNorm(r, dim) + before, err := s.Search(ctx, "/nocopy", q, 10, nil) + if err != nil { + t.Fatalf("search: %v", err) + } + + stripQ8(t, s) + after, err := s.Search(ctx, "/nocopy", q, 10, nil) + if err != nil { + t.Fatalf("search after strip: %v", err) + } + if len(after) == 0 { + t.Fatal("no results without the compact copy") + } + if strings.Join(resultKeys(after), ",") != strings.Join(exactTopK(chunks, embs, q, 10, ""), ",") { + t.Errorf("fallback scan disagrees with an exact scan: %v", resultKeys(after)) + } + if strings.Join(resultKeys(before), ",") != strings.Join(resultKeys(after), ",") { + t.Errorf("compact and fallback scans disagree\ncompact: %v\nfallback: %v", + resultKeys(before), resultKeys(after)) + } +} + +// TestBackfillConvertsAnOldStore walks the upgrade path end to end: a store +// with no compact copy is opened, the background pass converts it, and +// searches then take the fast path and still agree with an exact scan. +func TestBackfillConvertsAnOldStore(t *testing.T) { + const dim = 512 + dir := t.TempDir() + ctx := context.Background() + + s, err := Open(dir) + if err != nil { + t.Fatalf("open: %v", err) + } + chunks, embs := q8Corpus(t, s, "/old", 900, dim) + stripQ8(t, s) + if err := s.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + s2, err := Open(dir) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer s2.Close() + + collID, ok, err := s2.collectionID(ctx, collectionName("/old")) + if err != nil || !ok { + t.Fatalf("collection id: %v ok=%v", err, ok) + } + deadline := time.Now().Add(30 * time.Second) + for !s2.q8Ready(ctx, collID) { + if time.Now().After(deadline) { + t.Fatal("backfill did not finish within 30s") + } + time.Sleep(20 * time.Millisecond) + } + + var nVec, nQ8 int + if err := s2.db.QueryRow(`SELECT COUNT(*) FROM vectors`).Scan(&nVec); err != nil { + t.Fatal(err) + } + if err := s2.db.QueryRow(`SELECT COUNT(*) FROM vectors_q8`).Scan(&nQ8); err != nil { + t.Fatal(err) + } + if nVec != nQ8 { + t.Fatalf("backfill left %d of %d vectors unconverted", nVec-nQ8, nVec) + } + + r := rand.New(rand.NewSource(31)) + q := randNorm(r, dim) + got, err := s2.Search(ctx, "/old", q, 10, nil) + if err != nil { + t.Fatalf("search: %v", err) + } + if want := exactTopK(chunks, embs, q, 10, ""); strings.Join(resultKeys(got), ",") != strings.Join(want, ",") { + t.Errorf("backfilled scan disagrees with an exact scan\n got: %v\nwant: %v", resultKeys(got), want) + } +} + +// TestDeletesReachTheCompactCopy checks the direction that fails silently. +// +// A leftover q8 row is a document the scan keeps shortlisting and the rescore +// can no longer score, so it vanishes from results without anything logging a +// word — and it also keeps occupying disk that the "reclaimed" number says was +// freed. +func TestDeletesReachTheCompactCopy(t *testing.T) { + const dim = 256 + s := openStore(t) + ctx := context.Background() + chunks, _ := q8Corpus(t, s, "/del", 400, dim) + + victim := chunks[7].FilePath + if err := s.DeleteByFile(ctx, "/del", victim); err != nil { + t.Fatalf("delete by file: %v", err) + } + var orphans int + if err := s.db.QueryRow(` + SELECT COUNT(*) FROM vectors_q8 q + WHERE NOT EXISTS (SELECT 1 FROM vectors v + WHERE v.collection_id = q.collection_id AND v.doc_id = q.doc_id)`). + Scan(&orphans); err != nil { + t.Fatal(err) + } + if orphans != 0 { + t.Errorf("delete-by-file left %d orphaned rows in the compact copy", orphans) + } + + if err := s.DeleteCollection("/del"); err != nil { + t.Fatalf("delete collection: %v", err) + } + var left, states int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM vectors_q8`).Scan(&left); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow(`SELECT COUNT(*) FROM q8_state`).Scan(&states); err != nil { + t.Fatal(err) + } + if left != 0 || states != 0 { + t.Errorf("delete-collection left %d compact rows and %d state rows", left, states) + } +} + +// TestQuantizeRoundTrip pins the encoding itself, away from any database. +func TestQuantizeRoundTrip(t *testing.T) { + r := rand.New(rand.NewSource(4)) + for _, dim := range []int{1, 7, 256, 2048} { + v := randNorm(r, dim) + blob, scale := quantizeInt8(v) + if len(blob) != dim { + t.Fatalf("dim %d: blob is %d bytes, want one per component", dim, len(blob)) + } + var maxErr float64 + for i, b := range blob { + got := float64(int8(b)) * float64(scale) + if e := math.Abs(got - float64(v[i])); e > maxErr { + maxErr = e + } + } + // Half a quantisation step is the theoretical bound for round-to- + // nearest; anything above it means the scale or the rounding is wrong. + if bound := float64(scale)/2 + 1e-9; maxErr > bound { + t.Errorf("dim %d: worst component error %g exceeds half a step (%g)", dim, maxErr, bound) + } + } + + // The zero vector must not produce a NaN scale or a panic: it scores 0 + // against everything, which is what the float32 dot product also gives it. + blob, scale := quantizeInt8(make([]float32, 16)) + if scale != 0 || len(blob) != 16 { + t.Errorf("zero vector: scale=%v len=%d, want 0 and 16", scale, len(blob)) + } +} + +// TestDotInt8MatchesFloat checks the integer dot product against the float one +// on the same quantised values, so a mistake in the unrolled loop cannot hide +// behind quantisation error. +func TestDotInt8MatchesFloat(t *testing.T) { + r := rand.New(rand.NewSource(8)) + for _, dim := range []int{3, 4, 5, 64, 2048} { + a, _ := quantizeInt8(randNorm(r, dim)) + b, _ := quantizeInt8(randNorm(r, dim)) + var want int32 + for i := range a { + want += int32(int8(a[i])) * int32(int8(b[i])) + } + if got := dotInt8(a, b); got != want { + t.Errorf("dim %d: dotInt8 = %d, want %d", dim, got, want) + } + } + if got := dotInt8(make([]byte, 4), make([]byte, 5)); got != 0 { + t.Errorf("length mismatch returned %d, want 0", got) + } +} + +// TestScanQuantOffThenOn is the toggle nobody tests until it corrupts +// something. +// +// Turning the compact copy off has to be more than "stop reading it": a store +// that keeps its completion flag while writing rows the copy never sees is a +// store that, once the knob comes back on, answers searches from a copy +// missing everything written in between. Nothing errors, nothing logs — the +// results are just quietly incomplete, which is the failure mode that survives +// review. +func TestScanQuantOffThenOn(t *testing.T) { + const dim = 256 + dir := t.TempDir() + ctx := context.Background() + + on, err := OpenWith(Options{Dir: dir, ScanQuant: true}) + if err != nil { + t.Fatalf("open: %v", err) + } + firstHalf, embs1 := q8Corpus(t, on, "/toggle", 200, dim) + if err := on.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + // Second half written with the copy disabled. + off, err := OpenWith(Options{Dir: dir, ScanQuant: false}) + if err != nil { + t.Fatalf("reopen off: %v", err) + } + r := rand.New(rand.NewSource(77)) + secondHalf := make([]Chunk, 200) + embs2 := make([][]float32, 200) + for i := range secondHalf { + secondHalf[i] = Chunk{ + Content: fmt.Sprintf("late %d", i), + FilePath: fmt.Sprintf("late/f%04d.go", i), + StartLine: i*10 + 1, + EndLine: i*10 + 9, + ChunkType: "function", + SymbolName: fmt.Sprintf("Late%04d", i), + Language: "go", + } + embs2[i] = randNorm(r, dim) + } + if err := off.UpsertChunks(ctx, "/toggle", secondHalf, embs2); err != nil { + t.Fatalf("upsert while off: %v", err) + } + if err := off.Close(); err != nil { + t.Fatalf("close off: %v", err) + } + + back, err := OpenWith(Options{Dir: dir, ScanQuant: true}) + if err != nil { + t.Fatalf("reopen on: %v", err) + } + defer back.Close() + + collID, ok, err := back.collectionID(ctx, collectionName("/toggle")) + if err != nil || !ok { + t.Fatalf("collection id: %v ok=%v", err, ok) + } + deadline := time.Now().Add(30 * time.Second) + for !back.q8Ready(ctx, collID) { + if time.Now().After(deadline) { + t.Fatal("backfill did not re-cover the collection within 30s") + } + time.Sleep(20 * time.Millisecond) + } + + allChunks := append(append([]Chunk{}, firstHalf...), secondHalf...) + allEmbs := append(append([][]float32{}, embs1...), embs2...) + + // Query at one of the vectors written while the copy was off: if that + // window were lost, this is what would go missing. + q := allEmbs[len(embs1)+5] + got, err := back.Search(ctx, "/toggle", q, 10, nil) + if err != nil { + t.Fatalf("search: %v", err) + } + if want := exactTopK(allChunks, allEmbs, q, 10, ""); strings.Join(resultKeys(got), ",") != strings.Join(want, ",") { + t.Errorf("rows written while the compact copy was off are missing from search\n got: %v\nwant: %v", + resultKeys(got), want) + } +} + +// TestQ8FilterableCoversEveryFilter is a canary, not an invariant. +// +// The compact copy carries one metadata column, so exactly one `where` key can +// be answered from it and every other key silently costs 3.4x more bytes per +// vector. That trade is fine as long as somebody chose it. What this catches is +// nobody choosing: a new filterable column added to whereColumns, wired through +// the HTTP layer, and never considered here — after which large collections +// quietly go back to the float32 scan whenever that filter is used. +// +// If this fails, the fix is a decision, not a rubber stamp: either add the +// column to vectors_q8 and to scanQ8, or add it to the list below with a note +// saying the fallback is acceptable for it. +func TestQ8FilterableCoversEveryFilter(t *testing.T) { + fast := map[string]bool{"language": true} + + for key := range whereColumns { + got := q8Filterable(map[string]string{key: "x"}) + if got != fast[key] { + t.Errorf("q8Filterable(%q) = %v, want %v — a filter column changed and "+ + "nobody decided whether the compact scan should carry it", key, got, fast[key]) + } + } + + // An unknown key with an empty value is dropped by buildWhere (chromem + // parity: "" == ""), so it must not disqualify the fast path either. + if !q8Filterable(map[string]string{"nonsense": ""}) { + t.Error("an unknown key with an empty value should not force the exact scan") + } + // An unknown key with a value matches nothing at all; Search short-circuits + // before either scan, so which path it would have taken is moot — but it + // must not be reported as fast-path-able. + if q8Filterable(map[string]string{"nonsense": "x"}) { + t.Error("an unknown key with a value should not be treated as compact-scannable") + } +} + +// currentQ8Mismatches returns the doc_ids whose compact row disagrees with the +// float32 vector it is supposed to be derived from, plus the count of compact +// rows with no float32 row at all. +// +// Both are invisible in normal operation, which is why they are worth asserting +// directly. An orphan is shortlisted by every scan and dropped by every +// rescore, so it costs a result slot and says nothing. A stale row scores its +// document with a vector the document no longer has — it does not disappear, +// it ranks wrong. +func currentQ8Mismatches(t *testing.T, s *Store) (stale []string, orphans int) { + t.Helper() + rows, err := s.db.Query(` + SELECT q.doc_id, q.scale, q.embedding, v.embedding + FROM vectors_q8 q + LEFT JOIN vectors v ON v.collection_id = q.collection_id AND v.doc_id = q.doc_id`) + if err != nil { + t.Fatalf("join: %v", err) + } + defer rows.Close() + for rows.Next() { + var ( + docID string + scale float64 + q8, f32b []byte + ) + if err := rows.Scan(&docID, &scale, &q8, &f32b); err != nil { + t.Fatalf("scan: %v", err) + } + if f32b == nil { + orphans++ + continue + } + vec, _ := blobFloats(f32b, nil) + wantBlob, wantScale := quantizeInt8(vec) + if float32(scale) != wantScale || string(q8) != string(wantBlob) { + stale = append(stale, docID) + } + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + return stale, orphans +} + +// TestBackfillSurvivesConcurrentWrites is the regression for the gap between +// the backfill's read and its write. +// +// The backfill reads a batch of float32 vectors, quantises them in Go, and +// writes the results in a separate transaction. It deliberately holds no lock +// across that gap — the file watcher reindexing a file somebody just saved must +// not queue behind a background job — so anything can commit in between, and on +// the load-test fixture the gap is open for 245 seconds of live server. +// +// Two things go wrong without the WHERE EXISTS and DO NOTHING clauses in +// backfillQ8SQL, and neither surfaces as an error: +// +// - a document deleted in the gap gets its compact row reinserted, and +// nothing can ever remove it again — DeleteByFile finds doc_ids through +// `vectors`, where the row no longer is; +// - a document re-embedded in the gap has its fresh compact row overwritten +// by this batch's quantisation of the embedding it just replaced. +// +// The assertions are invariants rather than an expected interleaving, so this +// test can only fail for a real reason, whatever the scheduler does. +func TestBackfillSurvivesConcurrentWrites(t *testing.T) { + const ( + dim = 256 + files = 40 + per = 50 + ) + ctx := context.Background() + s := openStore(t) + + r := rand.New(rand.NewSource(17)) + writeFile := func(file string, seed int64) { + fr := rand.New(rand.NewSource(seed)) + chunks := make([]Chunk, per) + embs := make([][]float32, per) + for i := range chunks { + chunks[i] = Chunk{ + Content: fmt.Sprintf("%s chunk %d", file, i), FilePath: file, + StartLine: i*10 + 1, EndLine: i*10 + 9, + ChunkType: "function", SymbolName: fmt.Sprintf("S%03d", i), Language: "go", + } + embs[i] = randNorm(fr, dim) + } + if err := s.UpsertChunks(ctx, "/race", chunks, embs); err != nil { + t.Errorf("upsert %s: %v", file, err) + } + } + for i := 0; i < files; i++ { + writeFile(fmt.Sprintf("src/f%02d.go", i), int64(i)) + } + _ = r + + // Back to what an older binary would have left: float32 only. + stripQ8(t, s) + + collID, ok, err := s.collectionID(ctx, collectionName("/race")) + if err != nil || !ok { + t.Fatalf("collection id: %v ok=%v", err, ok) + } + + // The backfill walks the collection while the watcher churns it. + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + if _, err := s.backfillCollection(ctx, collID); err != nil { + t.Errorf("backfill: %v", err) + } + }() + go func() { + defer wg.Done() + for i := 0; i < files; i += 2 { + file := fmt.Sprintf("src/f%02d.go", i) + if i%4 == 0 { + // Deleted for good — the case that produces orphans. + if err := s.DeleteByFile(ctx, "/race", file); err != nil { + t.Errorf("delete %s: %v", file, err) + } + continue + } + // Re-embedded with different vectors — the case that produces + // stale rows. A different seed means every chunk's embedding, and + // therefore its quantisation, changes. + writeFile(file, int64(1000+i)) + } + }() + wg.Wait() + + stale, orphans := currentQ8Mismatches(t, s) + if orphans != 0 { + t.Errorf("%d compact rows survive with no vector behind them; "+ + "nothing can delete these — DeleteByFile looks up doc_ids through `vectors`", orphans) + } + if len(stale) != 0 { + t.Errorf("%d compact rows hold the quantisation of a replaced embedding, e.g. %q; "+ + "these documents are scored with vectors they no longer have", len(stale), stale[0]) + } + + // And the collection must still be searchable, by whichever path it ended + // up on — a race that leaves it correct but permanently unconverted would + // pass the assertions above and still be a bug worth seeing. + q := randNorm(rand.New(rand.NewSource(5)), dim) + if _, err := s.Search(ctx, "/race", q, 10, nil); err != nil { + t.Fatalf("search after the race: %v", err) + } +} + +// TestBackfillNeverResurrectsOrOverwrites pins the two SQL clauses the test +// above depends on, without needing a race to expose them. If backfillQ8SQL is +// ever simplified back to a plain upsert, this fails deterministically and says +// which half went missing. +func TestBackfillNeverResurrectsOrOverwrites(t *testing.T) { + ctx := context.Background() + s := openStore(t) + chunks, embs := q8Corpus(t, s, "/rules", 4, 64) + collID, ok, err := s.collectionID(ctx, collectionName("/rules")) + if err != nil || !ok { + t.Fatalf("collection id: %v ok=%v", err, ok) + } + var docID string + if err := s.db.QueryRow( + `SELECT doc_id FROM vectors WHERE collection_id = ? LIMIT 1`, collID).Scan(&docID); err != nil { + t.Fatal(err) + } + _ = chunks + _ = embs + + exec := func(docID string, scale float64, blob []byte) { + t.Helper() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() //nolint:errcheck // no-op after a successful Commit + if _, err := tx.ExecContext(ctx, backfillQ8SQL, + collID, docID, "go", scale, blob, collID, docID); err != nil { + t.Fatalf("backfill insert: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + } + + // DO NOTHING: an existing compact row is the fresher one and must survive. + var before float64 + if err := s.db.QueryRow( + `SELECT scale FROM vectors_q8 WHERE collection_id = ? AND doc_id = ?`, + collID, docID).Scan(&before); err != nil { + t.Fatal(err) + } + exec(docID, before+1, []byte{1, 2, 3}) + var after float64 + if err := s.db.QueryRow( + `SELECT scale FROM vectors_q8 WHERE collection_id = ? AND doc_id = ?`, + collID, docID).Scan(&after); err != nil { + t.Fatal(err) + } + if after != before { + t.Errorf("backfill overwrote a compact row written by the upsert path (scale %v -> %v)", before, after) + } + + // WHERE EXISTS: a doc with no vector must not gain one. + exec("ghost-doc-id", 0.5, []byte{9}) + var ghosts int + if err := s.db.QueryRow( + `SELECT COUNT(*) FROM vectors_q8 WHERE doc_id = 'ghost-doc-id'`).Scan(&ghosts); err != nil { + t.Fatal(err) + } + if ghosts != 0 { + t.Error("backfill inserted a compact row for a document that has no vector") + } +} diff --git a/server/internal/vectorstore/search.go b/server/internal/vectorstore/search.go index 38fad26e..507ffee8 100644 --- a/server/internal/vectorstore/search.go +++ b/server/internal/vectorstore/search.go @@ -19,6 +19,18 @@ import ( // of scanners rather than spawn a hundred threads and a hundred page caches. var scanSlots = make(chan struct{}, max(2, runtime.NumCPU())) +// acquireScanSlot takes one of the process-wide scan slots, returning the +// release function. A cancelled context gives up the wait rather than the +// query: a caller that has already gone away must not hold a scanner. +func acquireScanSlot(ctx context.Context) (func(), error) { + select { + case scanSlots <- struct{}{}: + return func() { <-scanSlots }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + // scanSQL streams one collection. // // INDEXED BY is not an optimisation hint, it is a guarantee, and the choice of @@ -36,16 +48,30 @@ var scanSlots = make(chan struct{}, max(2, runtime.NumCPU())) // index costs 1.8x because its keys are ordered by file_path, scattering the // lookups across the collection's whole rowid span. // TestScanUsesCollectionIndex pins the plan. -const scanSQL = `SELECT rowid, embedding FROM vectors INDEXED BY idx_vec_coll WHERE collection_id = ?` +const scanSQL = `SELECT doc_id, embedding FROM vectors INDEXED BY idx_vec_coll WHERE collection_id = ?` + +// scanQ8SQL is the same walk over the compact copy, and it is the one a search +// normally takes. Same INDEXED BY guarantee, same reason: idx_q8_coll's keys +// are (collection_id, rowid), so it visits only this collection's rows and +// yields them in table order. +// +// The row it reads is ~2.1 kB instead of ~8.2 kB, which is the whole point — +// three rows to a leaf page and no overflow chain, against one leaf slice plus +// a dedicated overflow page each. Measured with dbstat: 2731 vs 9216 bytes +// read per vector at 2048 dimensions. +const scanQ8SQL = `SELECT doc_id, scale, embedding FROM vectors_q8 INDEXED BY idx_q8_coll WHERE collection_id = ?` + +// rescoreSQL reads the exact vectors of the shortlist. +const rescoreSQL = `SELECT doc_id, embedding FROM vectors WHERE collection_id = ? AND doc_id IN (%s)` // hydrateSQL fetches the metadata and chunk text of the winners only. The // LEFT JOIN keeps a result whose content row is somehow missing (which should // be impossible — both are written in one transaction) instead of dropping it. -const hydrateSQL = `SELECT v.rowid, v.file_path, v.start_line, v.end_line, +const hydrateSQL = `SELECT v.doc_id, v.file_path, v.start_line, v.end_line, v.chunk_type, v.symbol_name, v.language, COALESCE(c.content, '') FROM vectors v LEFT JOIN vector_contents c ON c.collection_id = v.collection_id AND c.doc_id = v.doc_id - WHERE v.rowid IN (%s)` + WHERE v.collection_id = ? AND v.doc_id IN (%s)` // whereColumns maps chromem metadata keys to their SQL column. start_line and // end_line are integers in the schema but were strings in chromem's metadata, @@ -122,31 +148,120 @@ func (s *Store) Search(ctx context.Context, projectPath string, queryEmbedding [ q = normalizeVector(q) } + best, err := s.rank(ctx, collID, q, limit, where, clauses, args) + if err != nil { + return nil, fmt.Errorf("vectorstore search: %w", err) + } + if len(best) == 0 { + return nil, nil + } + return s.hydrate(ctx, collID, best) +} + +// q8Shortlist is how many candidates the compact scan hands to the rescorer. +// +// The int8 ranking is not the answer, it is a filter: it puts the right +// documents in the shortlist but misorders near-ties, so the shortlist has to +// be wide enough that everything belonging in the top K is inside it. Measured +// on 60k vectors of the fixture's largest collection (ziglang/zig, +// voyage-code-3 @2048) against 50 real query-side embeddings, recall of the +// exact float32 top-K after rescoring: +// +// shortlist k=10 k=20 +// 20 0.998 0.994 +// 40 0.998 0.999 +// 60 1.000 1.000 +// 200 1.000 1.000 +// +// (Without rescoring at all, the int8 ranking alone gives 0.994 at both k.) +// 60 is where both columns reach 1.000, so the floor is 64 and the multiple is +// 4x for larger k. The cost of a wider shortlist is one float32 row each — +// 9 kB — against a scan that just read thousands of times that, which is why +// the floor is generous rather than tight. +// +// A fixed width cannot be exact in the worst case, and it is worth stating +// which case that is: topK rejects boundary ties strictly, so a collection +// holding more than `shortlist` documents within one quantisation step of each +// other — a file vendored a hundred times, the near-duplicate clusters +// q8Corpus models — truncates the tie in scan order, and the rescore cannot +// recover a document that never reached it. Extending the shortlist to +// swallow ties at the boundary would close it. No corpus measured so far has +// needed that, and the documentation says "measured 1.000", not "exact", +// because of it. +func q8Shortlist(limit int) int { + if n := 4 * limit; n > 64 { + return n + } + return 64 +} + +// q8Filterable reports whether the compact scan can answer this filter. +// +// vectors_q8 carries one metadata column, language, because that is the only +// filter any caller actually produces (fetchVectorResults in the HTTP layer, +// from the `languages` query parameter). Anything else — a file_path or +// symbol_name filter, reachable through the Go API but not through HTTP — +// falls back to the float32 scan, which has every column. Slower and correct +// beats fast and wrong. +func q8Filterable(where map[string]string) bool { + for k, v := range where { + if k == "language" { + continue + } + // An unknown key with an empty value matches everything and is dropped + // by buildWhere, so it does not disqualify the fast path. + if _, known := whereColumns[k]; !known && v == "" { + continue + } + return false + } + return true +} + +// rank produces the final ordered candidates, by whichever route this +// collection supports. +func (s *Store) rank(ctx context.Context, collID int64, q []float32, limit int, + where map[string]string, clauses []string, args []any) ([]candidate, error) { + + if !q8Filterable(where) { + // Correct, and quietly ~3.4x more expensive per vector. Said out loud + // because the way this gets slow is a new filter key appearing in the + // HTTP layer: nothing breaks, nothing errors, large collections just + // go back to reading 9 kB per vector. TestQ8FilterableCoversEveryFilter + // is the compile-time half of the same guard. + s.logger.Debug("vectorstore: filter not supported by the compact scan, using the exact one", + "collection_id", collID, "filter_keys", len(where)) + } + if q8Filterable(where) && s.q8Ready(ctx, collID) { + shortlist, err := s.scanQ8(ctx, collID, q, q8Shortlist(limit), where) + if err != nil { + return nil, err + } + if len(shortlist) == 0 { + return nil, nil + } + return s.rescore(ctx, collID, q, shortlist, limit) + } + query := scanSQL queryArgs := append([]any{collID}, args...) if len(clauses) > 0 { query += " AND " + strings.Join(clauses, " AND ") } - top, err := s.scan(ctx, query, queryArgs, q, limit) if err != nil { - return nil, fmt.Errorf("vectorstore search: %w", err) - } - best := top.sorted() - if len(best) == 0 { - return nil, nil + return nil, err } - return s.hydrate(ctx, best) + return top.sorted(), nil } // scan streams the collection past the dot product, keeping the top K. func (s *Store) scan(ctx context.Context, query string, args []any, q []float32, k int) (*topK, error) { - select { - case scanSlots <- struct{}{}: - defer func() { <-scanSlots }() - case <-ctx.Done(): - return nil, ctx.Err() + release, err := acquireScanSlot(ctx) + if err != nil { + return nil, err } + defer release() rows, err := s.db.QueryContext(ctx, query, args...) if err != nil { @@ -155,18 +270,35 @@ func (s *Store) scan(ctx context.Context, query string, args []any, q []float32, defer rows.Close() top := newTopK(k) + if err := streamExact(rows, q, top); err != nil { + return nil, err + } + return top, nil +} + +// streamExact drives (doc_id, embedding) rows past the exact dot product and +// keeps the top K. Shared by the float32 scan and by the rescore, which are +// the same loop over the same columns with different WHERE clauses — and the +// float32 decoding protocol is exactly the kind of thing that rots when it +// exists in two places. +// +// Both columns are read as RawBytes, which is the whole point: the driver +// hands back a view of its own buffer, valid only until the next Next(). The +// embedding is consumed immediately by the dot product, and the doc_id becomes +// a Go string only for a row that actually enters the heap — K allocations +// over a scan instead of one per row, which at 1.9M rows per workspace query +// is the difference between a few thousand strings and sixty megabytes of +// garbage. +func streamExact(rows *sql.Rows, q []float32, top *topK) error { var ( - rowID int64 + docID sql.RawBytes raw sql.RawBytes scratch []float32 ) dim := len(q) for rows.Next() { - // RawBytes avoids a copy of every 3 kB embedding; it is only valid - // until the next Next(), which is fine because the dot product - // consumes it immediately. - if err := rows.Scan(&rowID, &raw); err != nil { - return nil, err + if err := rows.Scan(&docID, &raw); err != nil { + return err } if len(raw)/4 != dim { // A row from a different embedding model (namespaces are supposed @@ -179,10 +311,130 @@ func (s *Store) scan(ctx context.Context, query string, args []any, q []float32, vec, scratch = blobFloats(raw, scratch) score := dot(q, vec) if top.qualifies(score) { - top.add(candidate{rowID: rowID, score: score}) + top.add(candidate{docID: string(docID), score: score}) } } - return top, rows.Err() + return rows.Err() +} + +// scanQ8 streams the compact copy and returns the shortlist in approximate +// score order. +// +// The scores it produces are NOT returned to anyone: they rank the shortlist +// and are then thrown away by rescore, which recomputes them on the exact +// vectors. That is deliberate — an int8 dot product is a good enough ordering +// to choose 64 documents out of 350,000 and not good enough to be shown as a +// similarity. +func (s *Store) scanQ8(ctx context.Context, collID int64, q []float32, n int, where map[string]string) ([]candidate, error) { + release, err := acquireScanSlot(ctx) + if err != nil { + return nil, err + } + defer release() + + // The query is quantised the same way the stored vectors were, and its + // scale is constant across the scan, so it cancels out of every + // comparison. Only the per-row scale has to be applied. + // + // A zero query (scale 0, every component 0) is NOT short-circuited here. + // It scores every row 0, the heap fills with the first rows it sees, and + // the caller gets `limit` results at score 0 — which is exactly what the + // float32 scan does with the same input. Returning nothing instead would + // be more defensible in isolation and wrong in context: the two paths are + // chosen per collection, so a workspace fan-out would answer the same + // broken query with hits from the collections still on float32 and + // silence from the converted ones. + qq, _ := quantizeInt8(q) + + query := scanQ8SQL + args := []any{collID} + // Presence, not emptiness: chromem compared metadata["language"] against + // the filter value, so {"language": ""} asks for rows whose language is + // empty — a real query, not an absent filter. buildWhere gets this right + // for the float32 path by mapping the key to a column and binding + // whatever value came with it; treating "" as "no filter" here would make + // the two paths disagree on the one filter this one supports. + if language, ok := where["language"]; ok { + query += " AND language = ?" + args = append(args, language) + } + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + top := newTopK(n) + var ( + docID sql.RawBytes + scale float64 + raw sql.RawBytes + ) + dim := len(q) + for rows.Next() { + if err := rows.Scan(&docID, &scale, &raw); err != nil { + return nil, err + } + if len(raw) != dim { + // Same guard as the float32 scan: a row left by a different model. + continue + } + score := float32(scale) * float32(dotInt8(raw, qq)) + if top.qualifies(score) { + // Both columns alias the driver's buffer until the next Next(); + // the string is materialised only for a row that survives. See + // streamExact for why that matters at this row count. + top.add(candidate{docID: string(docID), score: score}) + } + } + if err := rows.Err(); err != nil { + return nil, err + } + return top.sorted(), nil +} + +// rescore recomputes the shortlist's scores on the exact float32 vectors and +// returns the true top `limit`. +// +// This is what makes the compact scan lossless in practice: measured against +// exact search over 50 real queries, the shortlist contained every document of +// the exact top-K, and rescoring restored the order the approximation blurred +// (see q8Shortlist for the table, and for the boundary case a fixed shortlist +// width cannot rule out). +func (s *Store) rescore(ctx context.Context, collID int64, q []float32, shortlist []candidate, limit int) ([]candidate, error) { + top := newTopK(limit) + for start := 0; start < len(shortlist); start += hydrateBatch { + batch := shortlist[start:min(start+hydrateBatch, len(shortlist))] + query, args := docIDInList(rescoreSQL, collID, batch) + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("rescore: %w", err) + } + err = streamExact(rows, q, top) + rows.Close() + if err != nil { + return nil, fmt.Errorf("rescore: %w", err) + } + } + return top.sorted(), nil +} + +// docIDInList fills a %s-templated IN-list with one placeholder per candidate +// and returns the statement with its arguments, collection first. +// +// Shared by the rescore and the hydrate because they ask the same question of +// the same key — "these doc_ids, in this collection" — and both are already +// chunked by hydrateBatch so neither can exceed SQLite's bound-parameter +// ceiling. +func docIDInList(tmpl string, collID int64, batch []candidate) (string, []any) { + placeholders := make([]string, len(batch)) + args := make([]any, 0, len(batch)+1) + args = append(args, collID) + for i, c := range batch { + placeholders[i] = "?" + args = append(args, c.docID) + } + return fmt.Sprintf(tmpl, strings.Join(placeholders, ",")), args } // hydrateBatch bounds the IN-list so a caller asking for an enormous limit @@ -191,18 +443,18 @@ const hydrateBatch = 500 // hydrate fetches metadata and chunk text for the winning rows and returns // them in score order. -func (s *Store) hydrate(ctx context.Context, best []candidate) ([]SearchResult, error) { - byRowID := make(map[int64]SearchResult, len(best)) +func (s *Store) hydrate(ctx context.Context, collID int64, best []candidate) ([]SearchResult, error) { + byDocID := make(map[string]SearchResult, len(best)) for start := 0; start < len(best); start += hydrateBatch { batch := best[start:min(start+hydrateBatch, len(best))] - if err := s.hydrateInto(ctx, batch, byRowID); err != nil { + if err := s.hydrateInto(ctx, collID, batch, byDocID); err != nil { return nil, err } } out := make([]SearchResult, 0, len(best)) for _, c := range best { - r, ok := byRowID[c.rowID] + r, ok := byDocID[c.docID] if !ok { // Deleted between the scan and the hydrate. Dropping it is the // honest answer — the chunk no longer exists. @@ -218,14 +470,9 @@ func (s *Store) hydrate(ctx context.Context, best []candidate) ([]SearchResult, } // hydrateInto reads one batch of winners into dst. -func (s *Store) hydrateInto(ctx context.Context, batch []candidate, dst map[int64]SearchResult) error { - placeholders := make([]string, len(batch)) - args := make([]any, len(batch)) - for i, c := range batch { - placeholders[i] = "?" - args[i] = c.rowID - } - rows, err := s.db.QueryContext(ctx, fmt.Sprintf(hydrateSQL, strings.Join(placeholders, ",")), args...) +func (s *Store) hydrateInto(ctx context.Context, collID int64, batch []candidate, dst map[string]SearchResult) error { + query, args := docIDInList(hydrateSQL, collID, batch) + rows, err := s.db.QueryContext(ctx, query, args...) if err != nil { return fmt.Errorf("vectorstore search hydrate: %w", err) } @@ -233,14 +480,14 @@ func (s *Store) hydrateInto(ctx context.Context, batch []candidate, dst map[int6 for rows.Next() { var ( - rowID int64 + docID string r SearchResult ) - if err := rows.Scan(&rowID, &r.FilePath, &r.StartLine, &r.EndLine, + if err := rows.Scan(&docID, &r.FilePath, &r.StartLine, &r.EndLine, &r.ChunkType, &r.SymbolName, &r.Language, &r.Content); err != nil { return fmt.Errorf("vectorstore search hydrate: %w", err) } - dst[rowID] = r + dst[docID] = r } if err := rows.Err(); err != nil { return fmt.Errorf("vectorstore search hydrate: %w", err) diff --git a/server/internal/vectorstore/sqlite.go b/server/internal/vectorstore/sqlite.go index 1521c20d..feff5e62 100644 --- a/server/internal/vectorstore/sqlite.go +++ b/server/internal/vectorstore/sqlite.go @@ -84,10 +84,11 @@ const idleConnTimeout = 30 * time.Second // schemaSQL is the whole schema. Two tables, deliberately. // -// `vectors` holds only what a scan reads: the metadata columns the `where` -// filter can constrain and the embedding itself. A row is ~3.2 kB, which fits -// inside an 8 KiB table-leaf cell (the local-payload limit is usable-35), so -// the scan reads two rows per page and never follows an overflow chain. +// `vectors` holds the authoritative float32 embedding and the metadata columns +// the `where` filter can constrain. At 768 dimensions a row is ~3.2 kB and fits +// inside an 8 KiB table-leaf cell (the local-payload limit is usable-35); at +// 2048 it is ~8.2 kB and does not, so every row spills into an overflow page. +// That is why the scan no longer reads this table — see `vectors_q8` below. // // `vector_contents` holds the chunk text. It is stored (duplicating chunks_fts // on disk) so SearchResult.Content behaves identically with no cross-database @@ -98,6 +99,35 @@ const idleConnTimeout = 30 * time.Second // chain. That would roughly double the pages a scan touches. Content is read // only for the K winners, so it costs one extra btree lookup per result. // +// `vectors_q8` is what a search actually scans: the same vectors at one byte +// per component, plus the per-vector scale that undoes the quantisation and +// the one metadata column a search can filter on in practice (language — see +// fetchVectorResults, the only caller that passes a filter). It exists because +// the paragraph above stopped being true once models grew past 1024 +// dimensions: a 2048-dim float32 row is 8.2 kB, which does NOT fit an 8 KiB +// leaf cell, so `vectors` is now exactly the overflow-chain layout that +// splitting out the content was meant to avoid. Measured with dbstat on 400 +// rows, per vector read by a full scan: +// +// 768 float32 4096 B two rows per leaf page +// 1024 float32 8192 B one row per leaf page, half of it air +// 2048 float32 9216 B leaf slice plus a whole overflow page +// 2048 int8 2731 B three rows per leaf page, no overflow +// +// The float32 blob stays in `vectors` and stays authoritative: the scan reads +// q8 to pick a shortlist, then rescores that shortlist against the exact +// vectors, which is what keeps the final ranking identical (see vector.go for +// the recall measurement). q8 rows are therefore derived data — losing them +// costs speed, never answers, which is what lets the backfill run in the +// background while searches fall back to the float32 scan. +// +// `q8_state` records that a collection's q8 rows are complete. Without it, +// "is this collection ready" would be a COUNT(*) over both tables on every +// query — the same mistake that made the stale-FTS probe cost 53 ms per +// workspace search. It carries no dimension: the scan compares each row's blob +// length against the query's own, so a row left by a different model is +// skipped per row rather than gated per collection. +// // Two indexes, and the difference between them matters: // // - idx_vec_coll is (collection_id, rowid) — SQLite appends the rowid to @@ -155,6 +185,19 @@ CREATE TABLE IF NOT EXISTS vectors ( ); CREATE INDEX IF NOT EXISTS idx_vec_coll ON vectors(collection_id); CREATE INDEX IF NOT EXISTS idx_vec_coll_file ON vectors(collection_id, file_path); +CREATE TABLE IF NOT EXISTS vectors_q8 ( + collection_id INTEGER NOT NULL REFERENCES collections(id) ON DELETE CASCADE, + doc_id TEXT NOT NULL, + language TEXT NOT NULL DEFAULT '', + scale REAL NOT NULL, + embedding BLOB NOT NULL, + PRIMARY KEY (collection_id, doc_id) +); +CREATE INDEX IF NOT EXISTS idx_q8_coll ON vectors_q8(collection_id); +CREATE TABLE IF NOT EXISTS q8_state ( + collection_id INTEGER PRIMARY KEY REFERENCES collections(id) ON DELETE CASCADE, + built_at TEXT NOT NULL +); CREATE TABLE IF NOT EXISTS vector_contents ( collection_id INTEGER NOT NULL REFERENCES collections(id) ON DELETE CASCADE, doc_id TEXT NOT NULL, diff --git a/server/internal/vectorstore/store.go b/server/internal/vectorstore/store.go index 30212027..76ec7e56 100644 --- a/server/internal/vectorstore/store.go +++ b/server/internal/vectorstore/store.go @@ -78,6 +78,20 @@ type Options struct { // mapped database pages are clean and reclaimable, but they count in RSS // and every connection maps the file. MMapBytes int64 + // ScanQuant enables the compact int8 copy that searches scan instead of + // the float32 originals (see q8.go). It governs the whole lifecycle, not + // just reading: with it off, the scan takes the float32 path, the backfill + // does not run, and writes DELETE the compact rows of the docs they touch + // and withdraw the collection's completion flag. That last part is what + // makes the switch safe to flip back — leaving rows behind under a live + // flag would mean searching a copy that no longer matches the vectors. + // + // The zero value is false so that a caller constructing Options by hand — + // every test, every tool — gets the plain float32 behaviour unless it asks + // otherwise. Open() is the exception: it is the convenience form and turns + // it on, because a store opened with defaults should behave like the + // server's. + ScanQuant bool // Logger receives migration progress. Defaults to a discarding logger. Logger *slog.Logger } @@ -105,6 +119,19 @@ type Store struct { // only invalidation needed is on delete. collMu sync.Mutex collIDs map[string]int64 + + // q8Mu guards q8State, a cache of collection id -> "the compact scan copy + // is complete". See q8.go; the entry only ever flips one way, so a cached + // true is permanent and a cached false is re-probed. + q8Mu sync.Mutex + q8State map[int64]bool + // scanQuant mirrors Options.ScanQuant. + scanQuant bool + + // stopBG cancels background work (the q8 backfill) on Close. The + // goroutines also go through acquire(), so cancelling is about not doing + // pointless work rather than about safety. + stopBG context.CancelFunc } // ErrClosed is returned by every method once Close has run. @@ -113,7 +140,7 @@ var ErrClosed = errors.New("vectorstore: store is closed") // Open opens (creating if needed) a vector store in the namespace directory // dir, with no legacy import. Kept as the simple form used by tests and tools. func Open(dir string) (*Store, error) { - return OpenWith(Options{Dir: dir}) + return OpenWith(Options{Dir: dir, ScanQuant: true}) } // OpenWith opens a vector store and, when o.LegacyChromaDir holds a chromem-go @@ -139,6 +166,8 @@ func OpenWith(o Options) (*Store, error) { legacyDir: strings.TrimSuffix(filepath.Clean(o.LegacyChromaDir), string(os.PathSeparator)), logger: logger, collIDs: map[string]int64{}, + q8State: map[int64]bool{}, + scanQuant: o.ScanQuant, } if o.LegacyChromaDir == "" { s.legacyDir = "" @@ -147,6 +176,11 @@ func OpenWith(o Options) (*Store, error) { db.Close() return nil, err } + bgCtx, stopBG := context.WithCancel(context.Background()) + s.stopBG = stopBG + if s.scanQuant { + s.startQ8Backfill(bgCtx) + } return s, nil } @@ -156,6 +190,9 @@ func (s *Store) Close() error { if s == nil { return nil } + if s.stopBG != nil { + s.stopBG() + } s.closeMu.Lock() defer s.closeMu.Unlock() if s.closed { @@ -230,7 +267,12 @@ func (s *Store) ensureCollection(ctx context.Context, name string) (int64, error if id, ok, err := s.collectionID(ctx, name); err != nil || ok { return id, err } - if _, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO collections(name) VALUES(?)`, name); err != nil { + res, err := s.db.ExecContext(ctx, `INSERT OR IGNORE INTO collections(name) VALUES(?)`, name) + if err != nil { + return 0, fmt.Errorf("vectorstore: create collection %q: %w", name, err) + } + created, err := res.RowsAffected() + if err != nil { return 0, fmt.Errorf("vectorstore: create collection %q: %w", name, err) } id, ok, err := s.collectionID(ctx, name) @@ -240,6 +282,23 @@ func (s *Store) ensureCollection(ctx context.Context, name string) (int64, error if !ok { return 0, fmt.Errorf("vectorstore: collection %q vanished after insert", name) } + if created > 0 && s.scanQuant { + // A collection that has just been created has no vectors, so its + // (empty) q8 side already matches it. Recording that here is what + // makes every collection this binary creates exempt from the backfill: + // upsertBatch writes both tables in one transaction from now on, so + // the property holds by construction. See q8.go. + // + // Logged, not returned: this flag is a performance hint, and its own + // transaction can lose a race for the write lock. Failing the caller + // would abort a whole indexing batch over a row whose absence costs + // nothing but a slower scan — and the absence self-heals, because the + // backfill sets the flag on the next open. + if err := s.markCollectionQ8Ready(ctx, id); err != nil { + s.logger.Warn("vectorstore: could not mark a new collection for the compact scan", + "collection", name, "err", err) + } + } return id, nil } @@ -262,6 +321,19 @@ const upsertContentSQL = `INSERT INTO vector_contents (collection_id, doc_id, co VALUES (?,?,?) ON CONFLICT(collection_id, doc_id) DO UPDATE SET content=excluded.content` +// upsertQ8SQL writes the scan copy in the same transaction as the vector it is +// derived from. Same transaction, not a later pass: a q8 row that disagrees +// with its float32 original would shortlist the wrong documents silently, and +// the only cheap way to guarantee they agree is to make them atomic. +const upsertQ8SQL = `INSERT INTO vectors_q8 (collection_id, doc_id, language, scale, embedding) + VALUES (?,?,?,?,?) + ON CONFLICT(collection_id, doc_id) DO UPDATE SET + language=excluded.language, scale=excluded.scale, embedding=excluded.embedding` + +// deleteQ8DocSQL removes one doc's compact copy. Used by the write path when +// the copy is switched off — see upsertBatch for why not writing is not enough. +const deleteQ8DocSQL = `DELETE FROM vectors_q8 WHERE collection_id = ? AND doc_id = ?` + // ErrCollectionDeleted reports that the collection an upsert was writing into // was deleted while the write was in flight — see UpsertChunks. var ErrCollectionDeleted = errors.New("vectorstore: collection was deleted while the upsert was in flight") @@ -324,6 +396,20 @@ func (s *Store) UpsertChunks(ctx context.Context, projectPath string, chunks []C return err } + // With the compact copy switched off, everything written below leaves it + // stale — so the completion flag comes off BEFORE the first byte lands, + // not after the last. Ordered that way because the failure it prevents is + // a crash mid-write with the flag still set: a collection that says it is + // complete while missing whatever the interrupted run had already written. + // Each batch also deletes the compact rows of the docs it touches, so a + // re-enable rebuilds from the float32 side rather than trusting a copy + // that was left behind. + if !s.scanQuant { + if err := s.clearCollectionQ8Ready(ctx, collID); err != nil { + return err + } + } + for start := 0; start < len(chunks); start += upsertBatchSize { end := min(start+upsertBatchSize, len(chunks)) if err := s.upsertBatch(ctx, collID, chunks[start:end], embeddings[start:end], start); err != nil { @@ -358,6 +444,23 @@ func (s *Store) upsertBatch(ctx context.Context, collID int64, chunks []Chunk, e return err } defer contentStmt.Close() + // One statement per doc either way: write the compact copy, or delete it. + // Deleting matters as much as writing. A doc re-embedded while the copy is + // off would otherwise keep the compact row of its PREVIOUS embedding, and + // the backfill deliberately never overwrites an existing compact row (see + // backfillQ8SQL) — so re-enabling the knob would seal that stale row in + // behind a completion flag, and searches would score the doc with a vector + // it no longer has. + var q8Stmt *sql.Stmt + if s.scanQuant { + q8Stmt, err = tx.PrepareContext(ctx, upsertQ8SQL) + } else { + q8Stmt, err = tx.PrepareContext(ctx, deleteQ8DocSQL) + } + if err != nil { + return err + } + defer q8Stmt.Close() for i, c := range chunks { emb := embeddings[i] @@ -372,6 +475,14 @@ func (s *Store) upsertBatch(ctx context.Context, collID int64, chunks []Chunk, e if _, err := contentStmt.ExecContext(ctx, collID, id, c.Content); err != nil { return err } + if s.scanQuant { + q8, scale := quantizeInt8(emb) + if _, err := q8Stmt.ExecContext(ctx, collID, id, c.Language, scale, q8); err != nil { + return err + } + } else if _, err := q8Stmt.ExecContext(ctx, collID, id); err != nil { + return err + } } return tx.Commit() } @@ -402,6 +513,16 @@ func (s *Store) DeleteByFile(ctx context.Context, projectPath, filePath string) collID, collID, filePath); err != nil { return fmt.Errorf("vectorstore delete contents for %q: %w", filePath, err) } + // Before the vectors themselves: the subquery reads file_path from + // `vectors`, so deleting there first would leave every q8 row of that file + // behind, and an orphan q8 row is a document the scan keeps shortlisting + // and the rescore can no longer score. + if _, err := tx.ExecContext(ctx, `DELETE FROM vectors_q8 + WHERE collection_id = ? AND doc_id IN ( + SELECT doc_id FROM vectors WHERE collection_id = ? AND file_path = ?)`, + collID, collID, filePath); err != nil { + return fmt.Errorf("vectorstore delete q8 for %q: %w", filePath, err) + } if _, err := tx.ExecContext(ctx, `DELETE FROM vectors WHERE collection_id = ? AND file_path = ?`, collID, filePath); err != nil { return fmt.Errorf("vectorstore delete by file %q: %w", filePath, err) diff --git a/server/internal/vectorstore/vector.go b/server/internal/vectorstore/vector.go index 9b823125..b6660cce 100644 --- a/server/internal/vectorstore/vector.go +++ b/server/internal/vectorstore/vector.go @@ -125,11 +125,19 @@ func dot(a, b []float32) float32 { // top-K // --------------------------------------------------------------------------- -// candidate is one scored row. Only the rowid is kept during the scan — the +// candidate is one scored row. Only the doc_id is kept during the scan — the // metadata and the chunk text of the K winners are fetched afterwards, so the // scan never materialises a string it is about to throw away. +// +// doc_id rather than rowid because the identity has to survive a VACUUM. +// `vectors` has a composite PRIMARY KEY, so its rowid is implicit, and SQLite +// only promises to preserve implicit rowids across a VACUUM for tables with an +// INTEGER PRIMARY KEY — everywhere else it may renumber them. That is +// survivable while the rowid never leaves a single query, and fatal once a +// second table (vectors_q8) keys off it: the pairing would silently shift and +// every search would score one document with another one's vector. type candidate struct { - rowID int64 + docID string score float32 } @@ -194,7 +202,7 @@ func (t *topK) down(i int) { } // sorted returns the candidates in descending score order. Ties break on the -// lower rowid so the ordering is deterministic across runs (SQLite may hand +// lower doc_id so the ordering is deterministic across runs (SQLite may hand // equal-scoring rows back in a different physical order after churn). func (t *topK) sorted() []candidate { out := append([]candidate(nil), t.h...) @@ -212,5 +220,106 @@ func less(a, b candidate) bool { if a.score != b.score { return a.score > b.score } - return a.rowID < b.rowID + return a.docID < b.docID +} + +// --------------------------------------------------------------------------- +// int8 quantisation +// +// The scan's cost is the bytes it streams, and at 2048 dimensions a float32 +// embedding is 8 KiB — more than an 8 KiB page can hold beside its own header, +// so every row also drags an overflow page behind it. Measured on the 45-repo +// fixture, one workspace query moves 17.6 GB. +// +// int8 makes that 2 KiB, which packs three rows to a page and reads 2.7 kB per +// vector: 3.4x less I/O and, measured, 3.0x less CPU in the dot product. +// +// The quantisation is per vector, not global. A single outlier component +// anywhere in a corpus would otherwise set the scale for every vector in it +// and crush the resolution of all the ordinary ones. Per-vector costs 8 bytes +// of REAL and makes each vector's own dynamic range the thing being spent. +// +// What this loses, measured against exact float32 ranking over 50 real +// query-side embeddings on the fixture's largest collection (60k vectors of +// ziglang/zig, voyage-code-3 @2048): recall@10 = 0.994 from the int8 ranking +// alone. Rescoring the shortlist on the float32 originals brought it to 1.000 +// at k=10 and k=20 — the approximation misorders near-ties, it does not lose +// the documents, so re-reading a few dozen exact vectors recovered every one. +// That is why the float32 blob stays on disk: it is no longer read by the +// scan, only by the rescore. +// +// Note what that does and does not promise. The SCORES are exact — they come +// from the float32 vectors either way. The SET is an approximation measured at +// zero error here, not proved at zero in general; see q8Shortlist for the +// boundary case it cannot rule out. +// --------------------------------------------------------------------------- + +// int8Max is the quantisation range. -128 is deliberately excluded: keeping +// the range symmetric means scale*q reconstructs -maxAbs and +maxAbs alike, so +// no component's sign carries a different error than its opposite. +const int8Max = 127 + +// quantizeInt8 encodes v as one signed byte per component plus the scale that +// undoes it: v[i] ~= scale * int8(blob[i]). +// +// A zero vector (or one of zero length) yields scale 0, which scores 0 against +// every query — the same answer the float32 dot product gives it. +func quantizeInt8(v []float32) (blob []byte, scale float32) { + if len(v) == 0 { + return nil, 0 + } + var maxAbs float32 + for _, x := range v { + if x < 0 { + x = -x + } + if x > maxAbs { + maxAbs = x + } + } + if maxAbs == 0 { + return make([]byte, len(v)), 0 + } + scale = maxAbs / int8Max + blob = make([]byte, len(v)) + for i, x := range v { + r := float64(x) / float64(scale) + q := math.Round(r) + if q > int8Max { + q = int8Max + } else if q < -int8Max { + q = -int8Max + } + blob[i] = byte(int8(q)) + } + return blob, scale +} + +// dotInt8 is the integer dot product of two quantised vectors. +// +// int32 cannot overflow here: every term is at most 127*127 = 16129, so it +// would take more than 133k dimensions to reach 2^31. Accumulating in int32 +// rather than float32 also means the sum is exact — all the error in this path +// is in the quantisation, none of it in the arithmetic. +// +// Returns 0 for a length mismatch, matching dot(). +func dotInt8(a, b []byte) int32 { + if len(a) != len(b) { + return 0 + } + var s0, s1, s2, s3 int32 + i := 0 + for ; i+4 <= len(a); i += 4 { + x := a[i : i+4 : i+4] + y := b[i : i+4 : i+4] + s0 += int32(int8(x[0])) * int32(int8(y[0])) + s1 += int32(int8(x[1])) * int32(int8(y[1])) + s2 += int32(int8(x[2])) * int32(int8(y[2])) + s3 += int32(int8(x[3])) * int32(int8(y[3])) + } + sum := s0 + s1 + s2 + s3 + for ; i < len(a); i++ { + sum += int32(int8(a[i])) * int32(int8(b[i])) + } + return sum }