diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d127eeb --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,58 @@ +# CI for the cgo C++ SIMD core and pure-Go fallback. +# +# The matrix runs natively on amd64 (portable scalar C++ path) and arm64 +# (NEON kernels), each with cgo on and off. Because the hosted runners +# already cover both Linux architectures natively, the Makefile's +# docker-verify target is intentionally NOT run here. +name: CI + +on: + push: + branches: [main, banchmark] + pull_request: + +jobs: + test: + name: test (${{ matrix.os }}, cgo=${{ matrix.cgo }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] # amd64 + arm64 (NEON) + cgo: [1, 0] + env: + CGO_ENABLED: ${{ matrix.cgo }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: go vet + run: go vet ./... + + - name: go build + run: go build ./... + + - name: go test + run: go test ./... + + gofmt: + name: gofmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: gofmt + run: | + files=$(gofmt -l .) + if [ -n "$files" ]; then + echo "The following files are not gofmt-formatted:" + echo "$files" + exit 1 + fi diff --git a/CLAUDE.md b/CLAUDE.md index 2ad27c6..b555d92 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,10 +86,6 @@ make clean **API (`internal/api/`)** - Gin HTTP server (`cmd/vectordb-server`) -**Legacy (`db/`, `storage/`)** -- Older brute-force engine (linear scan + top-k heap), kept as scaffold; - its Search uses the batched `CosineSimilarityMany` kernel - ### Conventions for the SIMD core - Never change `pkg/vectormath` public signatures; callers must not care which kernel is active @@ -101,7 +97,9 @@ make clean locally and via `make docker-verify` ### Known quirks -- `config.yaml` contains absolute macOS paths; tests use `t.TempDir()` +- `config.yaml` uses relative paths (`data/`, `logs/`) resolved against the + process working directory — run binaries from the repo root (or pass an + absolute `-config` path and adjust the paths); tests use `t.TempDir()` ### Configuration System - Main config file: `config.yaml` (server host/port, storage path, index diff --git a/README.md b/README.md index f0d7df2..d83c0e1 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,6 @@ vectorDB/ │ ├── scalar/ # pure-Go kernels (CGO_ENABLED=0 + parity reference) │ └── simd/ # C++17 SIMD core via cgo (NEON on aarch64) ├── persistence/ # BadgerDB store -├── db/, storage/ # Legacy brute-force engine + storage interfaces ├── proto/ # Protocol buffer definitions ├── docs/ # SIMD benchmark report ├── test/ # Integration tests @@ -142,9 +141,9 @@ A Go→C call costs roughly 40–50 ns. That fact shaped the API: per-pair call has to be cheap by itself. Fusion is the mitigation there. - *Paths we control do batch*: `CosineSimilarityBatch`/`CosineSimilarityMany` score one query against N vectors in a **single** crossing (flattened - row-major buffer, query norm computed once). The brute-force scan in - `db/engine.go` uses this — 10,000 vectors cost one crossing instead of - 10,000, which is the 561 µs → 114 µs row in the tables. + row-major buffer, query norm computed once) — 10,000 vectors cost one + crossing instead of 10,000, which is the 561 µs → 114 µs row in the + tables. - *Zero allocations*: the fused kernel reports zero-norm inputs via a NaN sentinel instead of an out-pointer, because any Go pointer passed to C escapes to the heap — the sentinel keeps the hot path allocation-free. @@ -158,7 +157,7 @@ reassociates float32 additions. ### Key design points -- **Frozen public API** — every caller (`internal/index`, `db`, tests) is +- **Frozen public API** — every caller (`internal/index`, tests) is untouched; the kernel swap is invisible above `pkg/vectormath`. - **Pure-Go fallback** — `CGO_ENABLED=0` builds and passes the full test suite anywhere Go runs, no C++ toolchain required. @@ -287,8 +286,23 @@ index: database: max_vectors: 1000000 + +logging: + level: info + encoding: json + output_paths: + - stdout + - logs/vectordb.log + dev_mode: false + +badger: + path: data ``` +Paths (`storage.path`, `badger.path`, file entries in +`logging.output_paths`) are relative to the process working directory; +missing directories are created on startup. + ## Development ```bash make build diff --git a/cmd/main.go b/cmd/main.go deleted file mode 100644 index 37736c4..0000000 --- a/cmd/main.go +++ /dev/null @@ -1,31 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/ishaan29/vectorDB/db" - "github.com/ishaan29/vectorDB/storage" -) - -func main() { - // Create a new vector engine - engine := db.NewEngine() - - // Create a new vector - vector := storage.Vector{ - ID: "1", - Embedding: []float32{0.1, 0.2, 0.3}, - Metadata: map[string]interface{}{"name": "test"}, - } - - // Insert the vector into the engine - engine.Insert(vector) - - // Retrieve the vector from the engine - retrievedVector, ok := engine.Get("1") - if ok { - fmt.Println("Retrieved Vector:", retrievedVector) - } else { - fmt.Println("Vector not found") - } -} diff --git a/config.yaml b/config.yaml index a6367fb..b99789c 100644 --- a/config.yaml +++ b/config.yaml @@ -3,7 +3,7 @@ server: port: 8080 storage: - path: /Users/ishaanbajpai/Desktop/bitCamp/vectorDB/data + path: data index: type: hnsw @@ -18,7 +18,7 @@ logging: output_paths: - stdout - logs/vectordb.log - dev_mode: false + dev_mode: false badger: - path: /Users/ishaanbajpai/Desktop/bitCamp/vectorDB/data \ No newline at end of file + path: data diff --git a/db/badger_test.go b/db/badger_test.go deleted file mode 100644 index 4cad066..0000000 --- a/db/badger_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package db - -import ( - "encoding/json" - "testing" - - "github.com/ishaan29/vectorDB/internal/config" - "github.com/ishaan29/vectorDB/internal/logger" - "github.com/ishaan29/vectorDB/persistence" - "github.com/ishaan29/vectorDB/pkg/types" -) - -// TestBadgerDBConnection tests basic BadgerDB connectivity - -func TestBadgerDBConnection(t *testing.T) { - // Create a temporary directory for the test database - cfg, err := config.Load("../config.yaml") - if err != nil { - t.Fatalf("Failed to load config: %v", err) - } - cfg.Logging.Level = "debug" - // Keep the test hermetic: config.yaml's file sink would resolve - // relative to this package's directory during `go test`. - cfg.Logging.OutputPaths = []string{"stdout"} - log, err := logger.New(&cfg.Logging) - if err != nil { - t.Fatalf("failed to init logger: %v", err) - } - defer log.Sync() - - store, err := persistence.NewBadgerStore(cfg.Badger.Path, log) - if err != nil { - log.Error("Failed to create Badger store", logger.Error("error", err)) - t.Fatalf("Failed to create Badger store: %v", err) - } - - vector := types.Vector{ - ID: "1", - Embedding: []float32{0.1, 0.2, 0.3}, - Metadata: map[string]interface{}{"name": "test"}, - } - - // test put - - err = store.Put(vector) - if err != nil { - log.Error("Failed to put vector", logger.Error("error", err)) - t.Fatalf("Failed to put vector: %v", err) - } - - vector, err = store.Get("1") - if err != nil { - log.Error("Failed to get vector", logger.Error("error", err)) - t.Fatalf("Failed to get vector: %v", err) - } - - if b, mErr := json.Marshal(vector); mErr != nil { - log.Error("Failed to marshal vector", logger.Error("error", mErr)) - } else { - log.Info("Vector", logger.String("vector", string(b))) - } -} diff --git a/db/engine.go b/db/engine.go deleted file mode 100644 index 23c4f94..0000000 --- a/db/engine.go +++ /dev/null @@ -1,123 +0,0 @@ -package db - -import ( - "container/heap" - "log" - "math" - "sync" - - "github.com/ishaan29/vectorDB/pkg/types" - "github.com/ishaan29/vectorDB/pkg/vectormath" - "github.com/ishaan29/vectorDB/storage" -) - -type Engine struct { - mu sync.RWMutex - store map[string]storage.Vector - vectorStore storage.VectorStore - // index -} - -func NewEngine() *Engine { - return &Engine{ - store: make(map[string]storage.Vector), - vectorStore: nil, - } -} - -func (engine *Engine) Insert(vector storage.Vector) { - engine.mu.Lock() - defer engine.mu.Unlock() - // engine.store[vector.ID] = vector - vector.ID = engine.vectorStore.GetVectorKey(vector.ID) - engine.vectorStore.Put(types.Vector{ID: vector.ID, Embedding: vector.Embedding, Metadata: vector.Metadata}) -} - -func (engine *Engine) Get(id string) (storage.Vector, bool) { - engine.mu.RLock() - defer engine.mu.RUnlock() - vector, err := engine.vectorStore.Get(id) - if err != nil { - return storage.Vector{}, false - } - return storage.Vector{ID: vector.ID, Embedding: vector.Embedding, Metadata: vector.Metadata}, true -} - -// SearchResult represents a single search result with its similarity score -type SearchResult struct { - Vector storage.Vector - Similarity float32 -} - -// ResultHeap is a min-heap of SearchResults -type ResultHeap []SearchResult - -func (h ResultHeap) Len() int { return len(h) } -func (h ResultHeap) Less(i, j int) bool { return h[i].Similarity < h[j].Similarity } -func (h ResultHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h *ResultHeap) Push(x interface{}) { *h = append(*h, x.(SearchResult)) } -func (h *ResultHeap) Pop() interface{} { - old := *h - n := len(old) - x := old[n-1] - *h = old[0 : n-1] - return x -} - -// Search performs a K-nearest neighbor search using cosine similarity -func (engine *Engine) Search(query []float32, k int) ([]SearchResult, error) { - engine.mu.RLock() - defer engine.mu.RUnlock() - - // Initialize a min-heap to store top K results - h := &ResultHeap{} - heap.Init(h) - - // Compare query vector with all vectors in store - vectors, err := engine.vectorStore.GetAllVectors() - if err != nil { - log.Printf("Failed to get all vectors: %v", err) - return nil, err - } - - // Score every stored vector in a single batched kernel invocation (one - // cgo crossing when the SIMD core is active). Mismatched-dimension and - // zero-norm vectors come back as NaN and are skipped, matching the old - // per-vector error behavior. - embeddings := make([][]float32, len(vectors)) - for i, v := range vectors { - embeddings[i] = v.Embedding - } - similarities, err := vectormath.CosineSimilarityMany(query, embeddings) - if err != nil { - log.Printf("Failed to score vectors: %v", err) - return nil, err - } - - for i, v := range vectors { - similarity := similarities[i] - if math.IsNaN(float64(similarity)) { - continue // Skip vectors that can't be compared - } - - // If we haven't found K vectors yet, just add to heap - if h.Len() < k { - heap.Push(h, SearchResult{Vector: storage.Vector{ID: v.ID, Embedding: v.Embedding, Metadata: v.Metadata}, Similarity: similarity}) - continue - } - - // If this vector is more similar than the least similar in our heap - if similarity > (*h)[0].Similarity { - heap.Pop(h) - heap.Push(h, SearchResult{Vector: storage.Vector{ID: v.ID, Embedding: v.Embedding, Metadata: v.Metadata}, Similarity: similarity}) - } - } - - // Convert heap to sorted slice (most similar first) - results := make([]SearchResult, h.Len()) - for i := len(results) - 1; i >= 0; i-- { - results[i] = heap.Pop(h).(SearchResult) - } - - return results, nil -} diff --git a/docs/simd-benchmark-report.md b/docs/simd-benchmark-report.md index 33cca9b..26a92a4 100644 --- a/docs/simd-benchmark-report.md +++ b/docs/simd-benchmark-report.md @@ -42,7 +42,8 @@ Notes: | pure Go fused, batched | 1093 µs | 109 ns | The batched kernel amortizes the cgo crossing to nothing and lets the NEON -loop stream; it is used by the brute-force scan in `db/engine.go`. +loop stream; it is exposed as the public +`CosineSimilarityBatch`/`CosineSimilarityMany` API. ## End-to-end HNSW (dim=128, M=16, efConstruction=200; count=3, benchtime=1000x) diff --git a/internal/config/config.go b/internal/config/config.go index 41f8c26..0a8eb1d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -75,5 +75,14 @@ func DefaultConfig() *Config { Database: DatabaseConfig{ MaxVectors: 1000000, }, + Logging: logger.Config{ + Level: "info", + Encoding: "json", + OutputPaths: []string{"stdout"}, + DevMode: false, + }, + Badger: BadgerConfig{ + Path: "data", + }, } } diff --git a/mempool/pool.go b/mempool/pool.go index 3d4767e..0751bbd 100644 --- a/mempool/pool.go +++ b/mempool/pool.go @@ -2,8 +2,6 @@ package mempool import ( "fmt" - "math" - "sort" "time" ) @@ -17,24 +15,6 @@ const ( HybridWarming // Combination of frequency and size ) -// SearchResult represents a search result with similarity score -type SearchResult struct { - Block *MemBlock - Similarity float32 -} - -// Error definitions -var ( - ErrInvalidK = fmt.Errorf("k must be greater than 0") - ErrInvalidDataType = fmt.Errorf("invalid data type: expected []byte") -) - -// Block represents a data block in memory -type Block struct { - Header BlockHeader - Data interface{} -} - func NewMemPool(config PoolConfig) (*MemPool, error) { if config.InitialSize == 0 { return nil, ErrInvalidSize @@ -401,78 +381,3 @@ func (p *MemPool) TrackAllocation(size uint64) { } p.warmupStats.frequency[size]++ } - -// Search finds the k most similar blocks to the query vector -func (p *MemPool) Search(query interface{}, k int) []Block { - p.mu.RLock() - defer p.mu.RUnlock() - - queryBytes, ok := query.([]byte) - if !ok { - return nil - } - - type Result struct { - block Block - similarity float32 - } - - var results []Result - - // Calculate similarities for all blocks - for _, memBlock := range p.block { - if !memBlock.Header.IsAllocated { - continue - } - - blockData, ok := memBlock.Data.([]byte) - if !ok { - continue - } - - similarity := p.calculateSimilarity(queryBytes, blockData) - block := Block{ - Header: memBlock.Header, - Data: memBlock.Data, - } - results = append(results, Result{block: block, similarity: similarity}) - } - - // Sort results by similarity in descending order - sort.Slice(results, func(i, j int) bool { - return results[i].similarity > results[j].similarity - }) - - // Return top k results - k = min(k, len(results)) - topK := make([]Block, k) - for i := 0; i < k; i++ { - topK[i] = results[i].block - } - - return topK -} - -// calculateSimilarity computes the similarity between two byte slices -func (p *MemPool) calculateSimilarity(a, b []byte) float32 { - // Simple cosine similarity implementation - if len(a) != len(b) { - return 0.0 - } - - var dotProduct float32 - var normA float32 - var normB float32 - - for i := 0; i < len(a); i++ { - dotProduct += float32(a[i]) * float32(b[i]) - normA += float32(a[i]) * float32(a[i]) - normB += float32(b[i]) * float32(b[i]) - } - - if normA == 0 || normB == 0 { - return 0.0 - } - - return dotProduct / (float32(math.Sqrt(float64(normA))) * float32(math.Sqrt(float64(normB)))) -} diff --git a/pkg/types/vector.go b/pkg/types/vector.go index f1848f3..487894e 100644 --- a/pkg/types/vector.go +++ b/pkg/types/vector.go @@ -1,13 +1,5 @@ package types -import ( - "math" - - "github.com/ishaan29/vectorDB/internal/logger" -) - -var Log logger.Logger - // In mem representation of a vector database type Vector struct { ID string `json:"id"` @@ -27,57 +19,3 @@ type SearchOptions struct { IncludeVecs bool `json:"include_vecs"` // Include vectors in results IncludeMeta bool `json:"include_meta"` // Include metadata in results } - -// MathVector represents a mathematical vector of float64 values -type MathVector struct { - Values []float64 -} - -// NewMathVector creates a new vector from a slice of float64 values -func NewMathVector(values []float64) *MathVector { - v := make([]float64, len(values)) - copy(v, values) - return &MathVector{Values: v} -} - -// Dot computes the dot product with another vector -func (v *MathVector) Dot(other *MathVector) float64 { - if len(v.Values) != len(other.Values) { - if Log != nil { - Log.Warn("Dimension mismatch in Dot") - } - return 0.0 - } - - sum := 0.0 - for i := 0; i < len(v.Values); i++ { - sum += v.Values[i] * other.Values[i] - } - return sum -} - -// Magnitude returns the L2 norm (Euclidean norm) of the vector -func (v *MathVector) Magnitude() float64 { - sum := 0.0 - for _, val := range v.Values { - sum += val * val - } - return math.Sqrt(sum) -} - -// CosineSimilarity calculates the cosine similarity with another vector -func (v *MathVector) CosineSimilarity(other *MathVector) float64 { - if len(v.Values) != len(other.Values) { - return 0.0 - } - - dot := v.Dot(other) - magV := v.Magnitude() - magOther := other.Magnitude() - - if magV == 0 || magOther == 0 { - return 0.0 - } - - return dot / (magV * magOther) -} diff --git a/scratchpad.go b/scratchpad.go deleted file mode 100644 index c492f8d..0000000 --- a/scratchpad.go +++ /dev/null @@ -1,17 +0,0 @@ -package main - -import ( - "fmt" -) - -// Scratch pad for testing and experimenting with Go code. - -type Vector struct { - ID string - Embedding []float32 - Metadata map[string]interface{} -} - -func main() { - fmt.Println("Hello, New Vector!") -} diff --git a/storage/simpleVectorStore.go b/storage/simpleVectorStore.go deleted file mode 100644 index 2cdbe1e..0000000 --- a/storage/simpleVectorStore.go +++ /dev/null @@ -1,6 +0,0 @@ -package storage - -type SimpleVectorStore interface { - Insert(vector Vector) - Get(id string) (Vector, bool) -} diff --git a/storage/vector.go b/storage/vector.go deleted file mode 100644 index b0fa14e..0000000 --- a/storage/vector.go +++ /dev/null @@ -1,7 +0,0 @@ -package storage - -type Vector struct { - ID string - Embedding []float32 - Metadata map[string]interface{} -} diff --git a/storage/vectorStore.go b/storage/vectorStore.go deleted file mode 100644 index 1bed160..0000000 --- a/storage/vectorStore.go +++ /dev/null @@ -1,15 +0,0 @@ -package storage - -import "github.com/ishaan29/vectorDB/pkg/types" - -type VectorStore interface { - Put(vector types.Vector) error - Get(id string) (types.Vector, error) - Delete(id string) error - Search(query types.Vector, options types.SearchOptions) ([]types.SearchResult, error) - GetIndexKey(id string) string - GetMetadataKey(id string) string - GetVectorKey(id string) string - GetAllVectors() ([]types.Vector, error) - Close() error -}