From 5eba2b76e1903ea333d3ebc00b2edabb3df531da Mon Sep 17 00:00:00 2001 From: Leo Daidone Date: Thu, 30 Oct 2025 19:55:41 -0300 Subject: [PATCH] foundation --- .github/workflows/ci.yml | 42 ++++++++++++++ Makefile | 58 ++++++++++++++++++++ README.md | 60 +++++++++++++++++++- cmd/goembedx/main.go | 115 +++++++++++++++++++++++++++++++++++++++ examples/basic.go | 27 +++++++++ go.mod | 3 + goembedx.go | 40 ++++++++++++++ search/brute.go | 83 ++++++++++++++++++++++++++++ store/memory.go | 46 ++++++++++++++++ tests/brute_test.go | 38 +++++++++++++ tests/math_bench_test.go | 41 ++++++++++++++ tests/math_test.go | 45 +++++++++++++++ tests/search_test.go | 14 +++++ tests/store_test.go | 20 +++++++ vector/math.go | 41 ++++++++++++++ vector/opt.go | 1 + 16 files changed, 671 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 Makefile create mode 100644 cmd/goembedx/main.go create mode 100644 examples/basic.go create mode 100644 go.mod create mode 100644 goembedx.go create mode 100644 search/brute.go create mode 100644 store/memory.go create mode 100644 tests/brute_test.go create mode 100644 tests/math_bench_test.go create mode 100644 tests/math_test.go create mode 100644 tests/search_test.go create mode 100644 tests/store_test.go create mode 100644 vector/math.go create mode 100644 vector/opt.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2b7e83a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: CI + +on: + push: + branches: [ "main", "master" ] + pull_request: + branches: [ "main", "master" ] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: "1.22" + + - name: Verify gofmt + run: | + files=$(gofmt -l .) + if [ -n "$files" ]; then + echo "These files are not formatted:" + echo "$files" + exit 1 + fi + + - name: Go vet + run: go vet ./... + + - name: Test with race detector and coverage + run: | + go test ./... -race -coverprofile=coverage.out -covermode=atomic + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + files: coverage.out + token: e2791bc0-4d3b-47a1-b04f-ed6f1f5bff17 + flags: unittests + fail_ci_if_error: true diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..de5e268 --- /dev/null +++ b/Makefile @@ -0,0 +1,58 @@ +APP := goembedx +PKG := ./... +EXAMPLE := ./examples/basic.go +CLI := ./cmd/goembedx +COVER_FILE := coverage.out + +.PHONY: all fmt lint test bench cover build example clean + +all: fmt lint test + +## ---------- Dev Commands ---------- +fmt: + @echo "๐Ÿงน Formatting code..." + go fmt $(PKG) + +lint: + @echo "๐Ÿ” Running basic lint (go vet)..." + go vet $(PKG) + +test: + @echo "โœ… Running tests with race detector..." + go test -race -cover -coverprofile=$(COVER_FILE) $(PKG) + +bench: + @echo "๐ŸŽ๏ธ Benchmarking vector ops..." + go test -bench=. -benchmem ./vector + +cover: test + @echo "๐Ÿ“Š Coverage report at $(COVER_FILE)" + go tool cover -html=$(COVER_FILE) + +## ---------- Build ---------- +build: + @echo "๐Ÿ”ง Building CLI..." + go build -o bin/$(APP) $(CLI) + +example: + @echo "โ–ถ๏ธ Running example..." + go run $(EXAMPLE) + +## ---------- Utilities ---------- +clean: + @echo "๐Ÿงฝ Cleaning workspace..." + rm -rf bin/ + rm -f $(COVER_FILE) + +help: + @echo "Usage: make [target]" + @echo "" + @echo "Targets:" + @echo " fmt Format code" + @echo " lint Static analysis" + @echo " test Tests w/ race + coverage" + @echo " bench Run benchmarks" + @echo " cover Open coverage UI" + @echo " build Build CLI" + @echo " example Run example program" + @echo " clean Clean build artifacts" diff --git a/README.md b/README.md index 8bab896..ae18617 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ -# goembedx ๐Ÿง โšก +# goembedx ๐Ÿง โšก > Lightweight local embedding store for Go โ€” pure Go, zero dependencies, blazing fast nearest-vector search. [![Go Reference](https://pkg.go.dev/badge/github.com/ldaidone/goembedx.svg)](https://pkg.go.dev/github.com/ldaidone/goembedx) [![Go Report Card](https://goreportcard.com/badge/github.com/ldaidone/goembedx)](https://goreportcard.com/report/github.com/ldaidone/goembedx) ![Stars](https://img.shields.io/github/stars/ldaidone/goembedx?style=social) [![License](https://img.shields.io/badge/license-Apache_2.0-blue.svg)](LICENSE) -[![CI](https://github.com/ldaidone/goembedx/actions/workflows/ci.yml/badge.svg)](https://github.com/ldaidone/goembedx/actions/workflows/ci.yml) -[![Coverage](https://img.shields.io/badge/coverage-100%25-brightgreen)](#) +[![Build](https://github.com/ldaidone/goembedx/actions/workflows/ci.yml/badge.svg)](https://github.com/ldaidone/goembedx/actions/workflows/ci.yml) +[![codecov](https://codecov.io/gh/ldaidone/goembedx/branch/main/graph/badge.svg)](https://codecov.io/gh/ldaidone/goembedx) + > ๐Ÿ’ก **goembedx** is a tiny vector database for embeddings โ€” perfect for local LLM agents, RAG systems, and semantic search inside Go applications. @@ -21,3 +22,56 @@ --- +### ๐Ÿš€ Quick Start + +```go +import ( + "fmt" + + "github.com/ldaidone/goembedx" +) + +func main() { + store := goembedx.New(384) // 384-dim example (MiniLM, etc.) + + store.Add("doc1", []float32{ /* embedding */ }) + store.Add("doc2", []float32{ /* embedding */ }) + + query := []float32{ /* embedding */ } + results := store.Search(query, 3) + + for _, r := range results { + fmt.Println(r.ID, r.Score) + } +} +``` + +### ๐Ÿ“ฆ Install + +```bash +go get github.com/ldaidone/goembedx +``` + +### ๐Ÿ”ญ Roadmap + +- โœ… In-memory vector store +- โœ… Cosine similarity + Top-K +- ๐Ÿงฉ File-based persistence (.embedx) +- ๐Ÿง  Optional ANN index (HNSW lite) +- ๐Ÿค– Ollama & HF embedding helpers +- ๐Ÿ”Œ goembedx serve โ€” REST API mode + +### ๐Ÿงช Testing + +```bash +go test ./... +``` + +## License + +Apache 2.0 License - see the [LICENSE](LICENSE) file for details. + +## Support + +If this saves you time or helps your AI project, consider starring โญ +and consider [buying me a coffee](https://www.buymeacoffee.com/leodaido)! โ˜•๏ธ โ€” it keeps the ideas flowing! \ No newline at end of file diff --git a/cmd/goembedx/main.go b/cmd/goembedx/main.go new file mode 100644 index 0000000..dae00f4 --- /dev/null +++ b/cmd/goembedx/main.go @@ -0,0 +1,115 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + "strconv" + "strings" + + "github.com/ldaidone/goembedx" +) + +var ( + dim = flag.Int("dim", 3, "Dimension of vectors") + topK = flag.Int("k", 3, "Top-K results") + mode = flag.String("mode", "interactive", "Mode: add|query|interactive") + id = flag.String("id", "", "ID for add mode") + vecFlag = flag.String("vec", "", "Comma-separated vector") +) + +func parseVec(s string, dimension int) ([]float32, error) { + parts := strings.Split(s, ",") + if len(parts) != dimension { + return nil, fmt.Errorf("expected %d elements, got %d", dimension, len(parts)) + } + v := make([]float32, len(parts)) + for i, p := range parts { + f, err := strconv.ParseFloat(strings.TrimSpace(p), 32) + if err != nil { + return nil, err + } + v[i] = float32(f) + } + return v, nil +} + +func main() { + flag.Parse() + store := goembedx.MemoryStore(*dim) + + switch *mode { + case "add": + v, err := parseVec(*vecFlag, *dim) + if err != nil { + fmt.Println("parse error:", err) + os.Exit(1) + } + if err := goembedx.AddVector(store, *id, v); err != nil { + fmt.Println("add failed:", err) + os.Exit(1) + } + fmt.Println("โœ… Added:", *id) + return + + case "query": + v, err := parseVec(*vecFlag, *dim) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + results, err := goembedx.SearchTopK(store, v, *topK) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Println("๐Ÿ”Ž Results:") + for _, r := range results { + fmt.Printf(" %s => %.5f\n", r.ID, r.Score) + } + return + } + + // interactive REPL mode (phase-1 lightweight) + fmt.Println("goembedx interactive mode") + fmt.Println("commands:") + fmt.Println(" add 1,2,3") + fmt.Println(" query 1,2,3") + reader := bufio.NewScanner(os.Stdin) + + for { + fmt.Print("> ") + if !reader.Scan() { + break + } + line := strings.TrimSpace(reader.Text()) + + if strings.HasPrefix(line, "add ") { + fields := strings.Fields(line) + id := fields[1] + v, err := parseVec(fields[2], *dim) + if err != nil { + fmt.Println("โš ๏ธ parse:", err) + continue + } + _ = goembedx.AddVector(store, id, v) + fmt.Println("โœ… ok") + continue + } + + if strings.HasPrefix(line, "query ") { + fields := strings.Fields(line) + v, err := parseVec(fields[1], *dim) + if err != nil { + fmt.Println("โš ๏ธ parse:", err) + continue + } + results, _ := goembedx.SearchTopK(store, v, *topK) + for _, r := range results { + fmt.Printf(" %s => %.5f\n", r.ID, r.Score) + } + continue + } + } +} diff --git a/examples/basic.go b/examples/basic.go new file mode 100644 index 0000000..015b448 --- /dev/null +++ b/examples/basic.go @@ -0,0 +1,27 @@ +package main + +import ( + "fmt" + + "github.com/ldaidone/goembedx/search" + "github.com/ldaidone/goembedx/store" + "github.com/ldaidone/goembedx/vector" +) + +func main() { + // small demo showing add + search + s := store.NewMemoryStore(3) + _ = s.Add("doc1", []float32{1, 0, 0}) + _ = s.Add("doc2", []float32{0.9, 0.1, 0}) + _ = s.Add("doc3", []float32{0, 1, 0}) + + query := []float32{1, 0, 0} + results := search.SearchBrute(s, query, 2) + + fmt.Println("Top results:") + for i, r := range results { + fmt.Printf("%d) id=%s score=%.5f\n", i+1, r.ID, r.Score) + } + // show cosine computed directly + fmt.Println("Cosine(doc1,query) =", vector.Cosine([]float32{1, 0, 0}, query)) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..1e259eb --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/ldaidone/goembedx + +go 1.25.3 diff --git a/goembedx.go b/goembedx.go new file mode 100644 index 0000000..1a2a496 --- /dev/null +++ b/goembedx.go @@ -0,0 +1,40 @@ +package goembedx + +import ( + "errors" + + "github.com/ldaidone/goembedx/search" + "github.com/ldaidone/goembedx/store" +) + +// Store is the top-level interface to the vector storage engine. +// For now we only expose memory, but keep interface future-proof when sqlite/bolt arrive. +type Store interface { + Add(id string, vec []float32) error + Len() int + Dim() int + Data() []store.Vector +} + +// MemoryStore returns an in-memory vector store. +// dim = embedding dimensionality (e.g. 384, 512, 768, 1024) +func MemoryStore(dim int) Store { + return store.NewMemoryStore(dim) +} + +// AddVector adds a vector to any store (helper for fluent API). +func AddVector(s Store, id string, vec []float32) error { + return s.Add(id, vec) +} + +// SearchTopK performs brute-force cosine similarity search. +// k <= 0 means "return all". +func SearchTopK(s Store, query []float32, k int) ([]search.Result, error) { + if s == nil { + return nil, errors.New("nil store") + } + if len(query) != s.Dim() { + return nil, errors.New("query dimension mismatch") + } + return search.SearchBrute(s.(*store.MemoryStore), query, k), nil +} diff --git a/search/brute.go b/search/brute.go new file mode 100644 index 0000000..da8fb86 --- /dev/null +++ b/search/brute.go @@ -0,0 +1,83 @@ +package search + +import ( + "container/heap" + "sort" + + "github.com/ldaidone/goembedx/store" + "github.com/ldaidone/goembedx/vector" +) + +// Result is a single search result (ID + score). +type Result struct { + ID string + Score float32 +} + +// SearchBrute returns the top-k results from the provided MemoryStore for the given query vector. +// If k <= 0 it returns all results sorted by score descending. +// Complexity: O(n log k) using a min-heap of size k. +func SearchBrute(s *store.MemoryStore, query []float32, k int) []Result { + if s == nil { + return nil + } + n := s.Len() + if n == 0 { + return []Result{} + } + + // if k <= 0 or k >= n, compute all and sort + if k <= 0 || k >= n { + out := make([]Result, 0, n) + for _, v := range s.Data() { + score := vector.Cosine(query, v.Val) + out = append(out, Result{ID: v.ID, Score: score}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Score > out[j].Score }) + return out + } + + // use a min-heap to maintain top-k + h := &minHeap{} + heap.Init(h) + for _, v := range s.Data() { + score := vector.Cosine(query, v.Val) + if h.Len() < k { + heap.Push(h, &item{res: Result{ID: v.ID, Score: score}}) + } else if score > (*h)[0].res.Score { + heap.Pop(h) + heap.Push(h, &item{res: Result{ID: v.ID, Score: score}}) + } + } + + // pop heap into results (reverse order) + res := make([]Result, h.Len()) + for i := len(res) - 1; i >= 0; i-- { + it := heap.Pop(h).(*item) + res[i] = it.res + } + return res +} + +// min-heap implementation for top-K +type item struct { + res Result +} + +type minHeap []*item + +func (h minHeap) Len() int { return len(h) } +func (h minHeap) Less(i, j int) bool { return h[i].res.Score < h[j].res.Score } // min-heap +func (h minHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } + +func (h *minHeap) Push(x any) { + *h = append(*h, x.(*item)) +} + +func (h *minHeap) Pop() any { + old := *h + n := len(old) + it := old[n-1] + *h = old[:n-1] + return it +} diff --git a/store/memory.go b/store/memory.go new file mode 100644 index 0000000..e4278d0 --- /dev/null +++ b/store/memory.go @@ -0,0 +1,46 @@ +package store + +import ( + "errors" +) + +// Vector is a stored vector with an ID. +type Vector struct { + ID string + Val []float32 +} + +// MemoryStore is a very small in-memory vector container optimized for read-heavy workloads. +// It is not thread-safe; callers should synchronize if used concurrently. +type MemoryStore struct { + dim int + data []Vector +} + +// NewMemoryStore creates a store for vectors of dimension dim. dim must be > 0. +func NewMemoryStore(dim int) *MemoryStore { + return &MemoryStore{ + dim: dim, + data: make([]Vector, 0), + } +} + +// Dim returns the dimensionality of this store. +func (s *MemoryStore) Dim() int { return s.dim } + +// Add inserts a vector into the store. It returns an error if the vector length doesn't match store dim. +func (s *MemoryStore) Add(id string, vec []float32) error { + if len(vec) != s.dim { + return errors.New("store: vector dimension mismatch") + } + s.data = append(s.data, Vector{ID: id, Val: vec}) + return nil +} + +// Data returns the underlying slice of vectors (read-only semantics expected). +func (s *MemoryStore) Data() []Vector { + return s.data +} + +// Len returns number of vectors stored. +func (s *MemoryStore) Len() int { return len(s.data) } diff --git a/tests/brute_test.go b/tests/brute_test.go new file mode 100644 index 0000000..c1f0a45 --- /dev/null +++ b/tests/brute_test.go @@ -0,0 +1,38 @@ +package tests + +import ( + "github.com/ldaidone/goembedx/search" + "testing" + + "github.com/ldaidone/goembedx/store" +) + +func TestBruteSearchBasic(t *testing.T) { + s := store.NewMemoryStore(3) + _ = s.Add("a", []float32{1, 0, 0}) // similar to query + _ = s.Add("b", []float32{0, 1, 0}) + _ = s.Add("c", []float32{0, 0, 1}) + + query := []float32{1, 0, 0} + res := search.SearchBrute(s, query, 2) + if len(res) != 2 { + t.Fatalf("expected 2 results, got %d", len(res)) + } + if res[0].ID != "a" { + t.Fatalf("expected top result 'a', got %s", res[0].ID) + } +} + +func TestBruteSearchAll(t *testing.T) { + s := store.NewMemoryStore(2) + _ = s.Add("x", []float32{1, 0}) + _ = s.Add("y", []float32{0, 1}) + + res := search.SearchBrute(s, []float32{1, 0}, 0) + if len(res) != 2 { + t.Fatalf("expected 2 results when k=0, got %d", len(res)) + } + if res[0].ID != "x" { + t.Fatalf("expected x first, got %s", res[0].ID) + } +} diff --git a/tests/math_bench_test.go b/tests/math_bench_test.go new file mode 100644 index 0000000..c8d9190 --- /dev/null +++ b/tests/math_bench_test.go @@ -0,0 +1,41 @@ +package tests + +import ( + "github.com/ldaidone/goembedx/vector" + "math/rand" + "testing" + "time" +) + +func makeVec(n int) []float32 { + v := make([]float32, n) + for i := 0; i < n; i++ { + v[i] = rand.Float32()*2 - 1 // random in [-1,1] + } + return v +} + +func init() { + // seed deterministic-ish randomness for benchmarks + rand.Seed(time.Now().UnixNano()) +} + +func BenchmarkDot768(b *testing.B) { + const dim = 768 + a := makeVec(dim) + c := makeVec(dim) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = vector.Dot(a, c) + } +} + +func BenchmarkCosine768(b *testing.B) { + const dim = 768 + a := makeVec(dim) + c := makeVec(dim) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = vector.Cosine(a, c) + } +} diff --git a/tests/math_test.go b/tests/math_test.go new file mode 100644 index 0000000..564d64f --- /dev/null +++ b/tests/math_test.go @@ -0,0 +1,45 @@ +package tests + +import ( + "github.com/ldaidone/goembedx/vector" + "testing" +) + +func TestDotAndNorm(t *testing.T) { + a := []float32{1, 2, 3} + b := []float32{4, -5, 6} + + // manual dot = 1*4 + 2*(-5) + 3*6 = 4 -10 +18 = 12 + if got := vector.Dot(a, b); got != 12 { + t.Fatalf("Dot expected 12, got %v", got) + } + + // Norm(a)^2 = 1+4+9 = 14 -> Norm = sqrt(14) + normA := vector.Norm(a) + if normA <= 0 { + t.Fatalf("Norm expected >0, got %v", normA) + } +} + +func TestCosineIdentity(t *testing.T) { + a := []float32{1, 0, 0} + if got := vector.Cosine(a, a); got != 1 { + t.Fatalf("Cosine identity expected 1, got %v", got) + } +} + +func TestCosineOrthogonal(t *testing.T) { + a := []float32{1, 0, 0} + b := []float32{0, 1, 0} + if got := vector.Cosine(a, b); got != 0 { + t.Fatalf("Cosine orthogonal expected 0, got %v", got) + } +} + +func TestCosineNegative(t *testing.T) { + a := []float32{1} + b := []float32{-1} + if got := vector.Cosine(a, b); got != -1 { + t.Fatalf("Cosine negative expected -1, got %v", got) + } +} diff --git a/tests/search_test.go b/tests/search_test.go new file mode 100644 index 0000000..e2b97ba --- /dev/null +++ b/tests/search_test.go @@ -0,0 +1,14 @@ +package tests + +import ( + "github.com/ldaidone/goembedx/internal/vec" + "testing" +) + +func TestCosine(t *testing.T) { + a := []float32{1, 0} + b := []float32{1, 0} + if vec.Cosine(a, b) != 1 { + t.Fatal("expected 1") + } +} diff --git a/tests/store_test.go b/tests/store_test.go new file mode 100644 index 0000000..fdc8015 --- /dev/null +++ b/tests/store_test.go @@ -0,0 +1,20 @@ +package tests + +import ( + "github.com/ldaidone/goembedx/store" + "testing" +) + +func TestMemoryStoreAdd(t *testing.T) { + s := store.NewMemoryStore(2) + if err := s.Add("id1", []float32{1, 2}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if s.Len() != 1 { + t.Fatalf("expected length 1, got %d", s.Len()) + } + + if err := s.Add("bad", []float32{1}); err == nil { + t.Fatalf("expected dimension mismatch error") + } +} diff --git a/vector/math.go b/vector/math.go new file mode 100644 index 0000000..5f50e5e --- /dev/null +++ b/vector/math.go @@ -0,0 +1,41 @@ +package vector + +import "math" + +// Dot computes the dot product of two vectors. +// It panics if the input lengths differ. +func Dot(a, b []float32) float32 { + if len(a) != len(b) { + panic("vector: Dot requires vectors of equal length") + } + var sum float32 + // simple scalar loop; optimized variants (SIMD) will replace this later + for i := 0; i < len(a); i++ { + sum += a[i] * b[i] + } + return sum +} + +// Norm returns the L2 norm (magnitude) of a vector. +func Norm(a []float32) float32 { + var sum float32 + for i := 0; i < len(a); i++ { + sum += a[i] * a[i] + } + return float32(math.Sqrt(float64(sum))) +} + +// Cosine returns the cosine similarity between two vectors. +// Values range from -1.0 to 1.0. Panics on length mismatch or zero-length vectors. +func Cosine(a, b []float32) float32 { + if len(a) != len(b) { + panic("vector: Cosine requires vectors of equal length") + } + na := Norm(a) + nb := Norm(b) + if na == 0 || nb == 0 { + // avoid division by zero; treat as undefined โ€” panic for now + panic("vector: Cosine with zero-length vector") + } + return Dot(a, b) / (na * nb) +} diff --git a/vector/opt.go b/vector/opt.go new file mode 100644 index 0000000..5a16616 --- /dev/null +++ b/vector/opt.go @@ -0,0 +1 @@ +package vector