diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 95cad8f..264a835 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -21,12 +21,12 @@ jobs:
uses: actions/checkout@v4
- name: Set up Go
- uses: actions/setup-go@v4
+ uses: actions/setup-go@v5
with:
- go-version: '^1.22'
+ go-version-file: go.mod
- name: Install dependencies
- run: go mod tidy
+ run: go mod download
- name: Run unit tests
run: make test-unit
@@ -39,12 +39,12 @@ jobs:
uses: actions/checkout@v4
- name: Set up Go
- uses: actions/setup-go@v4
+ uses: actions/setup-go@v5
with:
- go-version: '^1.22'
+ go-version-file: go.mod
- name: Install dependencies
- run: go mod tidy
+ run: go mod download
- name: Run integration tests
env:
@@ -59,12 +59,12 @@ jobs:
uses: actions/checkout@v4
- name: Set up Go
- uses: actions/setup-go@v4
+ uses: actions/setup-go@v5
with:
- go-version: '^1.22'
+ go-version-file: go.mod
- name: Install dependencies
- run: go mod tidy
+ run: go mod download
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
diff --git a/Makefile b/Makefile
index 98a7d2f..5076d3d 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ run: build
.PHONY: test test-unit test-integration test-e2e test-all test-coverage
.PHONY: start-localai stop-localai wait-localai
-.PHONY: start-test-services stop-test-services
+.PHONY: start-test-services stop-test-services test-raw-image
# Start LocalAI service for unit/integration tests
start-localai:
@@ -109,9 +109,12 @@ stop-test-services:
test:
@go test -coverprofile=coverage.txt -covermode=atomic -v ./...
-test-unit: start-test-services
+test-raw-image:
+ @docker build -t localrecall-raw-task2-a5975c8 -f test/raw/Dockerfile.postgres .
+
+test-unit: start-test-services test-raw-image
@echo "Running unit tests..."
- @LOCALAI_ENDPOINT=http://localhost:8081 go test -v ./rag/... ./pkg/...; \
+ @LOCALAI_ENDPOINT=http://localhost:8081 sh test/raw/unit.sh; \
test_exit=$$?; \
$(MAKE) stop-test-services; \
exit $$test_exit
@@ -125,9 +128,9 @@ test-integration: start-test-services
test-e2e: prepare-e2e run-e2e clean-e2e
-test-all: start-test-services
+test-all: start-test-services test-raw-image
@echo "Running all tests..."
- @LOCALAI_ENDPOINT=http://localhost:8081 go test -v ./rag/... ./pkg/...; \
+ @LOCALAI_ENDPOINT=http://localhost:8081 sh test/raw/unit.sh; \
unit_exit=$$?; \
INTEGRATION=true LOCALAI_ENDPOINT=http://localhost:8081 go test -v ./test/integration/...; \
integration_exit=$$?; \
@@ -207,5 +210,6 @@ clean-test-services: stop-test-services
@echo "Cleaning up test services..."
@docker compose rm -f localai postgres || true
-run-e2e:
- @E2E=true LOCALAI_ENDPOINT=http://localhost:8081 LOCALRECALL_ENDPOINT=http://localhost:8080 go test -v -timeout 30m ./test/e2e/...
+run-e2e: test-raw-image
+ @E2E=true LOCALAI_ENDPOINT=http://localhost:8081 LOCALRECALL_ENDPOINT=http://localhost:8080 go test -v -count=1 -timeout 30m ./test/e2e/...
+ @LOCALAI_ENDPOINT=http://localhost:8081 EMBEDDING_MODEL=$(EMBEDDING_MODEL) sh test/raw/database.sh sh test/raw/legacy.sh --owned --e2e-only
diff --git a/README.md b/README.md
index dc37385..601ceca 100644
--- a/README.md
+++ b/README.md
@@ -21,6 +21,12 @@ Currently, LocalRecall is batteries included and supports multiple vector databa
It can easily integrate with LocalAI, LocalAGI, and other agent frameworks, offering an intuitive web UI for convenient file management, including support for raw text inputs.
+## Caller-supplied vectors
+
+The PostgreSQL-only `/api/raw/v1` API stores caller-supplied vectors without invoking an embedding model.
+It supports vector, BM25, and hybrid retrieval in a namespace separate from legacy collections.
+See the [raw vector API guide](docs/raw-vector-api.md) for deployment, request examples, consistency limits, and verification.
+
## 📚🆕 Local Stack Family
🆕 LocalAI is now part of a comprehensive suite of AI tools designed to work together:
diff --git a/docs/raw-vector-api.md b/docs/raw-vector-api.md
new file mode 100644
index 0000000..d7f2885
--- /dev/null
+++ b/docs/raw-vector-api.md
@@ -0,0 +1,137 @@
+# Raw vector API
+
+The `/api/raw/v1` API stores vectors supplied by your application. It requires PostgreSQL. Legacy collection routes and Go interfaces remain separate.
+
+LocalRecall does not embed raw records, fetch media, or manage source assets. Your application owns embedding models, chunking, asset storage, and publication.
+
+## Deployment
+
+Set `VECTOR_ENGINE=postgres` and `DATABASE_URL`. Install `vector` and `pg_textsearch` through your database administrator. The tested BM25 extension version is 1.3.1.
+
+The service role needs schema creation, table creation, index creation, and DML privileges for its raw registry and tables. Collection deletion requires ownership privileges.
+Raw startup does not repair unknown schemas or corrupt tables. Back up the registry and collection tables together.
+
+Raw uses one shared pool per service. The default maximum is four connections. `POSTGRES_POOL_MAX_CONNS` changes this maximum; a DSN `pool_max_conns` takes precedence.
+Legacy PostgreSQL collections retain their separate pools. Budget approximately `raw pool maximum + loaded legacy collections × legacy pool maximum` per process.
+Include administrative connections and replicas in your database budget.
+
+Raw operations have a 30-second deadline and a five-second database lock timeout. Server shutdown drains HTTP requests, cancels outstanding work, and closes the raw pool.
+
+## Authentication and readiness
+
+Raw routes use the existing `API_KEYS` authentication boundary. **When `API_KEYS` is unset, routes are unauthenticated. Use this mode only on a private, trusted network; do not expose it publicly.** Keys do not provide collection-level authorization or tenant isolation.
+Treat metadata filters as query predicates, not access controls. Use a trusted application gateway when users require different permissions.
+
+Read `GET /api/raw/v1/capabilities` before selecting a search mode. Implementation support and current readiness are separate fields.
+A core database outage produces unavailable responses. Missing BM25 or ANN support disables dependent operations rather than silently changing retrieval semantics.
+Non-PostgreSQL raw requests return HTTP 501 before mutation. Legacy backends remain available independently.
+
+Errors distinguish invalid requests (400), authentication failures (401), missing collections (404), conflicts (409), oversized bodies (413), unsupported operations (501), and unavailable dependencies (503).
+
+## Create and populate a collection
+
+Use your deployment address and authentication header in these requests:
+
+```http
+POST /api/raw/v1/collections
+Content-Type: application/json
+
+{"name":"catalog-v1","dimensions":3,"metric":"cosine","embedding_space_id":"example-v1"}
+```
+
+The response contains a UUID `collection_id`. Raw names occupy a separate namespace from legacy names.
+Dimensions, embedding space, metric, and lexical configuration are immutable. Dimensions range from 1 to 2000. Only cosine similarity is supported.
+Lexical indexing defaults to enabled. Its configuration captures `BM25_TEXT_CONFIG` at creation, or `english` when unset.
+
+```http
+POST /api/raw/v1/collections/COLLECTION_ID/records/upsert
+Content-Type: application/json
+
+{"embedding_space_id":"example-v1","records":[{"id":"item-1","vector":[1,0,0],"text":"An astronomy telescope","metadata":{"category":"science"}}]}
+```
+
+Vectors must contain finite values and a nonzero norm. LocalRecall normalizes vectors for storage.
+Each upsert replaces the entire record. Omitted text clears text; omitted metadata becomes an empty object.
+A batch contains at most 100 unique IDs. The whole batch commits atomically. Request bodies cannot exceed 8 MiB.
+
+## Retrieve records
+
+```http
+POST /api/raw/v1/collections/COLLECTION_ID/search
+Content-Type: application/json
+
+{"mode":"vector","accuracy":"exact","embedding_space_id":"example-v1","vector":[1,0,0],"top_k":10,"filter":{"metadata_eq":{"category":"science"}}}
+```
+
+Choose `vector`, `lexical`, or `hybrid` explicitly. Lexical queries supply `text` instead of a vector. Hybrid queries supply both.
+Vector-bearing searches default to `approximate`. Use `exact` when you require exhaustive filtered ranking.
+Approximate search can return fewer results than requested, especially with selective filters. It does not guarantee recall.
+
+Hybrid search combines independent ranks with equal-weight reciprocal rank fusion. Each contribution is `0.5 / (60 + rank)`.
+Each retrieval branch considers `min(1000, max(100, 10 * top_k))` candidates. Responses expose component ranks and fusion parameters.
+Raw lexical retrieval never substitutes native full-text search for BM25.
+
+Additional routes:
+
+| Method | Collection-relative path | Purpose |
+| --- | --- | --- |
+| POST | `/records/get` | Fetch 1–100 IDs; report missing IDs |
+| POST | `/records/list` | Traverse records with a keyset cursor |
+| POST | `/records/delete` | Delete IDs or a bounded metadata selection |
+| POST | `/reset` | Delete all records transactionally |
+| DELETE | collection URL | Delete the descriptor and table atomically |
+
+List pages default to 50 records and allow at most 100. Cursors bind to collection, filters, and projection.
+Cursors do not provide snapshots across concurrent writes. Filter deletion rejects more than 1000 matches without partial deletion.
+
+## Retries and migrations
+
+Writes use last-committed-write-wins semantics. Retrying an older batch can overwrite a newer record.
+After an uncertain commit, read the affected IDs before deciding whether to retry. The API has no revision locks or idempotency receipts.
+
+Reset is a transactional deletion, not a barrier against concurrent writers. Stop your writers when you require an empty collection after reset.
+
+For an incompatible embedding space, create a new collection. Populate and validate it before switching your application's active collection reference.
+Your application owns that switch. Keep the old collection until in-flight readers finish, then delete it explicitly.
+LocalRecall does not activate generations or automatically remove stale collections.
+
+## Verification and evidence limits
+
+Run `sh test/raw/verify.sh all` from the worktree root after provisioning the [genuine consumer dependencies](../test/raw/LOCALAI-SMOKE.md).
+Missing dependencies, failed suites, and skipped required tests fail this command. Logs and private artifacts stay under `.sdd-work`.
+The runner uses disposable databases and owned loopback processes, not existing host services.
+
+The synthetic retrieval fixture contains 600 three-dimensional vectors with a selective 1-in-17 filter. Logs record EXPLAIN, recall@10, and underfill.
+These measurements test query behavior; they do not predict production latency or embedding quality.
+The legacy planning fixture uses 2000 eight-dimensional synthetic vectors. Its plan checks vector index ordering.
+
+The real consumer fixture uses pinned LocalAI source and genuine 384-dimensional MiniLM text embeddings. Provider identity differs from consumer identity.
+Its forwarding counter measures actual consumer embedding requests and checks zero additional requests during raw operations.
+This evidence does not establish multimodal quality, large-corpus throughput, or production resource limits.
+
+### Recorded acceptance sample
+
+The owned attempt5 full gate passed, including ten race-enabled repetitions of
+outage recovery and active-query shutdown. An actual `pg_ctl stop/start` retained
+the vectors; unavailable operations returned the raw unavailable error (HTTP 503),
+and readiness became false. The supervisor keeps private tmpfs alive, and restart
+preserves the required BM25 preload. Shutdown canceled a database-lock waiter and
+released all sessions and pool connections.
+
+The real consumer overlap sample held one embedding HTTP request open for
+28.13 ms while three raw collections completed writes and exact searches.
+The counter observed exactly one legacy request and zero additional raw requests.
+Two legacy collections and one shared raw pool used three sampled database
+sessions, excluding the observer connection; the configured pool ceiling was 12.
+This is a sampled count, not a continuous peak measurement. Public catalog objects
+numbered 8 before raw bootstrap, 30 with the overlap collections, and 18 after
+removing them: 12 objects removed. The remainder includes shared raw schema and
+the second legacy collection, not leaked per-collection indexes.
+The raw lifecycle and overlap portion took 71.04 ms. These are local synthetic
+measurements, not throughput or production sizing guarantees.
+
+The filtered 600-vector fixture returned 10 of 10 results from 36 eligible rows:
+recall@10 was 1.00 and underfill was zero. Planner choice can affect this result;
+it does not establish universal ANN recall. Full EXPLAIN JSON and timings are in
+`.sdd-work/logs/full-attempt5.log`; repeated suites are in `race10.log`, legacy
+coverage in `legacy.log`, and downstream evidence in `consumer-all.log`.
diff --git a/main.go b/main.go
index 1bc4b91..4b446f2 100644
--- a/main.go
+++ b/main.go
@@ -1,9 +1,18 @@
package main
import (
+ "context"
+ "errors"
+ "github.com/mudler/localrecall/rag/raw"
+ rawpostgres "github.com/mudler/localrecall/rag/raw/postgres"
+ "net"
+ "net/http"
"os"
+ "os/signal"
"strconv"
"strings"
+ "syscall"
+ "time"
"github.com/labstack/echo/v4"
"github.com/labstack/echo/v4/middleware"
@@ -52,7 +61,7 @@ func init() {
sourceManager.Start()
}
-func startAPI(listenAddress string) {
+func startAPI(listenAddress string) error {
e := echo.New()
e.Use(middleware.Logger())
e.Use(middleware.Recover())
@@ -89,9 +98,55 @@ func startAPI(listenAddress string) {
registerAPIRoutes(e, openAIClient, chunkingSize, overlap, keys)
- e.Logger.Fatal(e.Start(listenAddress))
+ // The raw pool belongs to this server, independently of legacy collections.
+ var store raw.Store
+ if vectorEngine == "postgres" {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ service, err := rawpostgres.New(ctx, rawpostgres.Config{DatabaseURL: os.Getenv("DATABASE_URL")})
+ cancel()
+ if err != nil {
+ e.Logger.Error("Raw PostgreSQL unavailable")
+ } else {
+ store = service
+ }
+ }
+ registerRawAPIRoutes(e, store)
+ stop, cancelSignal := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+ defer cancelSignal()
+ return serveAPI(e, listenAddress, store, stop)
+}
+
+// serveAPI returns startup failures only after releasing server-owned resources.
+func serveAPI(e *echo.Echo, listenAddress string, store raw.Store, stop context.Context) error {
+ requests, cancelRequests := context.WithCancel(context.Background())
+ e.Server.BaseContext = func(_ net.Listener) context.Context { return requests }
+ var serveErr error
+ exited := make(chan error, 1)
+ go func() { exited <- e.Start(listenAddress) }()
+ select {
+ case err := <-exited:
+ if err != nil && !errors.Is(err, http.ErrServerClosed) {
+ e.Logger.Error("HTTP server stopped")
+ serveErr = err
+ }
+ case <-stop.Done():
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ if err := e.Shutdown(ctx); err != nil {
+ cancelRequests()
+ _ = e.Close()
+ }
+ cancel()
+ <-exited
+ }
+ cancelRequests()
+ if store != nil {
+ store.Close()
+ }
+ return serveErr
}
func main() {
- startAPI(listeningAddress)
+ if err := startAPI(listeningAddress); err != nil {
+ os.Exit(1)
+ }
}
diff --git a/pkg/client/raw.go b/pkg/client/raw.go
new file mode 100644
index 0000000..107a494
--- /dev/null
+++ b/pkg/client/raw.go
@@ -0,0 +1,229 @@
+// Package client provides additive context-aware access to raw vectors.
+package client
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "github.com/google/uuid"
+ "github.com/mudler/localrecall/rag/raw"
+ "io"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+type RawClient struct {
+ baseURL, apiKey string
+ httpClient *http.Client
+}
+
+func NewRawClient(baseURL, apiKey string, httpClient *http.Client) *RawClient {
+ if httpClient == nil {
+ httpClient = http.DefaultClient
+ }
+ return &RawClient{baseURL, apiKey, httpClient}
+}
+func (c *RawClient) do(ctx context.Context, method, path string, q url.Values, body, out any) (int, error) {
+ u, err := url.Parse(c.baseURL)
+ if err != nil {
+ return 0, err
+ }
+ if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" {
+ return 0, fmt.Errorf("invalid raw API base URL")
+ }
+ escapedPath := strings.TrimRight(u.EscapedPath(), "/") + "/api/raw/v1" + path
+ u.Path, err = url.PathUnescape(escapedPath)
+ if err != nil {
+ return 0, err
+ }
+ u.RawPath = escapedPath
+ u.RawQuery = q.Encode()
+ var reader io.Reader
+ if body != nil {
+ b, err := json.Marshal(body)
+ if err != nil {
+ return 0, err
+ }
+ reader = bytes.NewReader(b)
+ }
+ req, err := http.NewRequestWithContext(ctx, method, u.String(), reader)
+ if err != nil {
+ return 0, err
+ }
+ if body != nil {
+ req.Header.Set("Content-Type", "application/json")
+ }
+ if c.apiKey != "" {
+ req.Header.Set("Authorization", "Bearer "+c.apiKey)
+ }
+ resp, err := c.httpClient.Do(req)
+ if err != nil {
+ return 0, err
+ }
+ defer resp.Body.Close()
+ var envelope struct {
+ Success bool `json:"success"`
+ Data json.RawMessage `json:"data"`
+ Error *raw.Error `json:"error"`
+ }
+ // Decode from a bounded stream rather than allocating the theoretical cap.
+ limit := rawResponseLimit(method, path)
+ readerLimit := &io.LimitedReader{R: resp.Body, N: limit + 1}
+ decoder := json.NewDecoder(readerLimit)
+ decodeErr := decoder.Decode(&envelope)
+ // Drain the remaining bounded stream, including decoder read-ahead. This
+ // detects overflow even when a complete JSON value precedes excess bytes.
+ _, drainErr := io.Copy(rawJSONWhitespace{}, io.MultiReader(decoder.Buffered(), readerLimit))
+ if readerLimit.N == 0 {
+ return resp.StatusCode, fmt.Errorf("raw API response exceeds %d bytes", limit)
+ }
+ if drainErr != nil {
+ return resp.StatusCode, drainErr
+ }
+ if decodeErr != nil {
+ return resp.StatusCode, fmt.Errorf("invalid raw API response (HTTP %d)", resp.StatusCode)
+ }
+
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 || !envelope.Success {
+ if envelope.Error != nil {
+ e := raw.NewError(envelope.Error.Code)
+ e.Status = resp.StatusCode
+ return resp.StatusCode, e
+ }
+ return resp.StatusCode, fmt.Errorf("raw API request failed (HTTP %d)", resp.StatusCode)
+ }
+ if out != nil {
+ err = json.Unmarshal(envelope.Data, out)
+ }
+ return resp.StatusCode, err
+}
+
+// Bounds cover compact server JSON. Strings can expand to six bytes per input
+// byte; metadata already has a serialized-byte limit. 32 bytes per vector
+// component covers every finite float32 spelling plus its comma. Per-item slack
+// covers keys, punctuation, scores/ranks, timestamps and other scalar fields.
+const (
+ rawItemOverhead int64 = 2048
+ rawEnvelopeOverhead int64 = 4096 + 6*raw.MaxCursorBytes
+ rawRecordBytes int64 = 6*(raw.MaxTextBytes+raw.MaxIDBytes) + raw.MaxMetadataBytes + 32*raw.MaxDimensions + rawItemOverhead
+ // Disabled lexical indexing retains text_config without catalog resolution;
+ // its existing validation limit is MaxBodyBytes, not PostgreSQL name length.
+ rawCollectionBytes int64 = 6*(raw.MaxBodyBytes+raw.MaxNameBytes+raw.MaxSpaceIDBytes) + rawItemOverhead
+)
+
+func rawResponseLimit(method, path string) int64 {
+ switch {
+ case path == "/collections" && method == http.MethodGet:
+ return rawEnvelopeOverhead + raw.MaxListLimit*rawCollectionBytes
+ case path == "/collections" || (strings.HasPrefix(path, "/collections/") && strings.Count(path, "/") == 2):
+ return rawEnvelopeOverhead + rawCollectionBytes
+ default:
+ return rawEnvelopeOverhead + int64(max(raw.MaxBatch, raw.MaxListLimit, raw.MaxTopK))*rawRecordBytes
+ }
+}
+
+// Write checks trailing JSON whitespace with constant additional memory.
+type rawJSONWhitespace struct{}
+
+func (rawJSONWhitespace) Write(p []byte) (int, error) {
+ for _, c := range p {
+ if c != ' ' && c != '\n' && c != '\r' && c != '\t' {
+ return 0, fmt.Errorf("invalid trailing raw API response data")
+ }
+ }
+ return len(p), nil
+}
+
+func rawCollectionPath(cid string) (string, error) {
+ id, err := uuid.Parse(cid)
+ if err != nil || id.String() != cid {
+ return "", raw.NewError(raw.CodeValidation)
+ }
+ return "/collections/" + cid, nil
+}
+func (c *RawClient) Capabilities(ctx context.Context) (out raw.Capabilities, err error) {
+ _, err = c.do(ctx, "GET", "/capabilities", nil, nil, &out)
+ return
+}
+func (c *RawClient) CreateCollection(ctx context.Context, req raw.CreateCollectionRequest) (out raw.Collection, created bool, err error) {
+ status, err := c.do(ctx, "POST", "/collections", nil, req, &out)
+ return out, status == 201, err
+}
+func (c *RawClient) ListCollections(ctx context.Context, req raw.ListCollectionsRequest) (out raw.CollectionPage, err error) {
+ q := url.Values{}
+ if req.Limit != nil {
+ q.Set("limit", strconv.Itoa(*req.Limit))
+ }
+ if req.Cursor != nil {
+ q.Set("cursor", *req.Cursor)
+ }
+ _, err = c.do(ctx, "GET", "/collections", q, nil, &out)
+ return
+}
+func (c *RawClient) GetCollection(ctx context.Context, cid string) (out raw.Collection, err error) {
+ path, err := rawCollectionPath(cid)
+ if err != nil {
+ return
+ }
+ _, err = c.do(ctx, "GET", path, nil, nil, &out)
+ return
+}
+func (c *RawClient) Upsert(ctx context.Context, cid string, req raw.UpsertRequest) (out raw.UpsertResponse, err error) {
+ path, err := rawCollectionPath(cid)
+ if err != nil {
+ return
+ }
+ _, err = c.do(ctx, "POST", path+"/records/upsert", nil, req, &out)
+ return
+}
+func (c *RawClient) GetRecords(ctx context.Context, cid string, req raw.GetRecordsRequest) (out raw.GetRecordsResponse, err error) {
+ path, err := rawCollectionPath(cid)
+ if err != nil {
+ return
+ }
+ _, err = c.do(ctx, "POST", path+"/records/get", nil, req, &out)
+ return
+}
+func (c *RawClient) ListRecords(ctx context.Context, cid string, req raw.ListRecordsRequest) (out raw.RecordPage, err error) {
+ path, err := rawCollectionPath(cid)
+ if err != nil {
+ return
+ }
+ _, err = c.do(ctx, "POST", path+"/records/list", nil, req, &out)
+ return
+}
+func (c *RawClient) DeleteRecords(ctx context.Context, cid string, req raw.DeleteRecordsRequest) (out raw.DeleteRecordsResponse, err error) {
+ path, err := rawCollectionPath(cid)
+ if err != nil {
+ return
+ }
+ _, err = c.do(ctx, "POST", path+"/records/delete", nil, req, &out)
+ return
+}
+func (c *RawClient) Search(ctx context.Context, cid string, req raw.SearchRequest) (out raw.SearchResponse, err error) {
+ path, err := rawCollectionPath(cid)
+ if err != nil {
+ return
+ }
+ _, err = c.do(ctx, "POST", path+"/search", nil, req, &out)
+ return
+}
+func (c *RawClient) Reset(ctx context.Context, cid string) error {
+ path, err := rawCollectionPath(cid)
+ if err != nil {
+ return err
+ }
+ _, err = c.do(ctx, "POST", path+"/reset", nil, raw.ResetRequest{}, nil)
+ return err
+}
+func (c *RawClient) DeleteCollection(ctx context.Context, cid string) error {
+ path, err := rawCollectionPath(cid)
+ if err != nil {
+ return err
+ }
+ _, err = c.do(ctx, "DELETE", path, nil, nil, nil)
+ return err
+}
diff --git a/pkg/client/raw_test.go b/pkg/client/raw_test.go
new file mode 100644
index 0000000..695d16a
--- /dev/null
+++ b/pkg/client/raw_test.go
@@ -0,0 +1,254 @@
+package client
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "github.com/mudler/localrecall/rag/raw"
+ "io"
+ "math"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func TestRawClient(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") != "Bearer key" {
+ t.Error("auth")
+ }
+ if r.URL.Path != "/api/raw/v1/collections" {
+ t.Error(r.URL)
+ }
+ w.WriteHeader(201)
+ w.Write([]byte(`{"success":true,"data":{"collection_id":"id"}}`))
+ }))
+ defer srv.Close()
+ c := NewRawClient(srv.URL, "key", srv.Client())
+ d, created, err := c.CreateCollection(context.Background(), raw.CreateCollectionRequest{})
+ if err != nil || !created || d.CollectionID != "id" {
+ t.Fatal(d, created, err)
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ _, _, err = c.CreateCollection(ctx, raw.CreateCollectionRequest{})
+ if !errors.Is(err, context.Canceled) {
+ t.Fatal(err)
+ }
+}
+
+func TestRawClientSafety(t *testing.T) {
+ calls := 0
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ calls++
+ w.WriteHeader(503)
+ w.Write([]byte(`{"success":false,"error":{"code":"UNAVAILABLE","message":"secret SQL"}}`))
+ }))
+ defer srv.Close()
+ c := NewRawClient(srv.URL, "", srv.Client())
+ for _, cid := range []string{"../capabilities", "foo/bar", "?x=1", "%2f", "."} {
+ if _, err := c.GetCollection(context.Background(), cid); err == nil {
+ t.Fatal(cid)
+ }
+ }
+ if calls != 0 {
+ t.Fatal(calls)
+ }
+ _, err := c.Capabilities(context.Background())
+ var re *raw.Error
+ if !errors.As(err, &re) || re.Status != 503 || !re.Retryable || strings.Contains(err.Error(), "secret") {
+ t.Fatal(err)
+ }
+ for _, base := range []string{srv.URL + "?bad=1", srv.URL + "#fragment", "file:///tmp/foo", "http://user:password@localhost"} {
+ if _, err := NewRawClient(base, "key", srv.Client()).Capabilities(context.Background()); err == nil {
+ t.Fatal(base)
+ }
+ }
+ if calls != 1 {
+ t.Fatal(calls)
+ }
+}
+
+// spaceReader supplies large padding without allocating the response cap.
+type spaceReader struct{}
+
+func (spaceReader) Read(p []byte) (int, error) {
+ for i := range p {
+ p[i] = ' '
+ }
+ return len(p), nil
+}
+
+func TestRawClientResponseOverflow(t *testing.T) {
+ limit := rawResponseLimit(http.MethodGet, "/capabilities")
+ prefix := `{"success":true,"data":{}}`
+ for _, extra := range []int64{0, 1} {
+ body := io.MultiReader(strings.NewReader(prefix), io.LimitReader(spaceReader{}, limit-int64(len(prefix))+extra))
+ c := NewRawClient("http://fixture.invalid", "", &http.Client{Transport: rawTransport(func(*http.Request) (*http.Response, error) {
+ return &http.Response{StatusCode: 200, Body: io.NopCloser(body), Header: http.Header{}}, nil
+ })})
+ _, err := c.Capabilities(context.Background())
+ if extra == 0 && err != nil {
+ t.Fatal("exact bound rejected", err)
+ }
+ if extra == 1 && (err == nil || !strings.Contains(err.Error(), "exceeds")) {
+ t.Fatal("overflow not detected", err)
+ }
+ }
+ for _, suffix := range []string{"x", ` {"success":true}`, " \n\t"} {
+ c := NewRawClient("http://fixture.invalid", "", &http.Client{Transport: rawTransport(func(*http.Request) (*http.Response, error) {
+ return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(prefix + suffix)), Header: http.Header{}}, nil
+ })})
+ _, err := c.Capabilities(context.Background())
+ if (err == nil) != (suffix == " \n\t") {
+ t.Fatalf("trailing data %q: %v", suffix, err)
+ }
+ }
+}
+
+func TestRawClientResponseBoundAccounting(t *testing.T) {
+ l := raw.ContractLimits()
+ text := strings.Repeat("\x01", l.TextBytes)
+ metadata := map[string]string{}
+ // Maximal serialized metadata is already covered by its own contract cap.
+ for i := 0; i < 15; i++ {
+ metadata[fmt.Sprintf("%02d", i)] = strings.Repeat("v", l.MetadataValueBytes)
+ }
+ metadata["last"] = ""
+ b, err := json.Marshal(metadata)
+ if err != nil {
+ t.Fatal(err)
+ }
+ metadata["last"] = strings.Repeat("v", l.MetadataBytes-len(b))
+ vector := make([]float32, l.Dimensions)
+ for i := range vector {
+ vector[i] = -1.2345678e-30
+ }
+ record := raw.CanonicalRecord{ID: strings.Repeat("<", l.IDBytes), Text: &text, Vector: vector, Metadata: metadata}
+ b, err = json.Marshal(record)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if int64(len(b)) > rawRecordBytes {
+ t.Fatal("maximal record exceeds budget", len(b), rawRecordBytes)
+ }
+ score := math.MaxFloat64
+ rank := int(^uint(0) >> 1)
+ result := raw.SearchResult{ID: record.ID, Text: &text, Metadata: metadata, Score: score, VectorSimilarity: &score, VectorRank: &rank, LexicalRank: &rank}
+ b, err = json.Marshal(result)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if int64(len(b)) > rawRecordBytes {
+ t.Fatal("scalar/search overhead exceeds budget")
+ }
+ // Collection lists have a larger cumulative cap without allocating that cap.
+ if rawResponseLimit("GET", "/collections") != rawEnvelopeOverhead+int64(l.ListMax)*rawCollectionBytes {
+ t.Fatal("collection list bound")
+ }
+ if rawCollectionBytes < 6*int64(raw.MaxBodyBytes+l.NameBytes+l.SpaceIDBytes)+rawItemOverhead {
+ t.Fatal("descriptor bound")
+ }
+}
+func TestRawClientEncodedPrefix(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.EscapedPath() != "/tenant%2Fprivate/api/raw/v1/capabilities" {
+ t.Errorf("prefix semantics changed: %s", r.URL.EscapedPath())
+ }
+ w.Write([]byte(`{"success":true,"data":{}}`))
+ }))
+ defer srv.Close()
+ if _, err := NewRawClient(srv.URL+"/tenant%2Fprivate", "", srv.Client()).Capabilities(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// A transport fixture keeps these contract tests independent of a database.
+type rawTransport func(*http.Request) (*http.Response, error)
+
+func (f rawTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+func TestRawClientSelectorPresence(t *testing.T) {
+ for _, ids := range [][]string{nil, {}, {"x"}} {
+ for _, vector := range [][]float64{nil, {}, {1}} {
+ c := NewRawClient("http://fixture.invalid", "", &http.Client{Transport: rawTransport(func(r *http.Request) (*http.Response, error) {
+ b, err := io.ReadAll(r.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var fields map[string]json.RawMessage
+ if err := json.Unmarshal(b, &fields); err != nil {
+ t.Fatal(err)
+ }
+ if strings.HasSuffix(r.URL.Path, "/delete") {
+ if _, present := fields["ids"]; present != (ids != nil) {
+ t.Errorf("IDs presence changed: %s", b)
+ }
+ var req raw.DeleteRecordsRequest
+ if err := raw.DecodeRequest(b, &req); err != nil {
+ t.Fatal(err)
+ }
+ if (raw.ValidateDelete(req) == nil) != (ids == nil) {
+ t.Errorf("delete validity changed: %s", b)
+ }
+ } else {
+ if _, present := fields["vector"]; present != (vector != nil) {
+ t.Errorf("vector presence changed: %s", b)
+ }
+ var req raw.SearchRequest
+ if err := raw.DecodeRequest(b, &req); err != nil {
+ t.Fatal(err)
+ }
+ _, err := raw.ValidateSearch(raw.Collection{Lexical: raw.LexicalConfig{Enabled: true}}, req)
+ if (err == nil) != (vector == nil) {
+ t.Errorf("search validity changed: %s", b)
+ }
+ }
+ return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"success":true,"data":{}}`)), Header: http.Header{}}, nil
+ })})
+ cid := "12345678-1234-1234-1234-123456789abc"
+ if _, err := c.DeleteRecords(context.Background(), cid, raw.DeleteRecordsRequest{IDs: ids, Filter: &raw.Filter{MetadataEq: map[string]string{"source": "asset"}}}); err != nil {
+ t.Fatal(err)
+ }
+ text := "query"
+ if _, err := c.Search(context.Background(), cid, raw.SearchRequest{Mode: "lexical", Vector: vector, Text: &text}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+}
+
+func TestRawClientMaxEscapedRecords(t *testing.T) {
+ text := strings.Repeat("\x01", raw.MaxTextBytes)
+ records := make([]raw.CanonicalRecord, raw.MaxBatch)
+ ids := make([]string, raw.MaxBatch)
+ for i := range records {
+ ids[i] = fmt.Sprint(i)
+ canonical, err := raw.ValidateUpsert(raw.Collection{Dimensions: 1, EmbeddingSpaceID: "s"}, raw.UpsertRequest{EmbeddingSpaceID: "s", Records: []raw.Record{{ID: ids[i], Vector: []float64{1}, Text: &text}}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ records[i] = canonical[0]
+ }
+ b, err := json.Marshal(struct {
+ Success bool `json:"success"`
+ Data raw.GetRecordsResponse `json:"data"`
+ }{true, raw.GetRecordsResponse{Records: records, MissingIDs: []string{}}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("legal response bytes: %d", len(b))
+ c := NewRawClient("http://fixture.invalid", "", &http.Client{Transport: rawTransport(func(*http.Request) (*http.Response, error) {
+ return &http.Response{StatusCode: 200, Body: io.NopCloser(bytes.NewReader(b)), Header: http.Header{}}, nil
+ })})
+ got, err := c.GetRecords(context.Background(), "12345678-1234-1234-1234-123456789abc", raw.GetRecordsRequest{IDs: ids, IncludeVectors: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(got.Records) != len(records) || *got.Records[0].Text != text {
+ t.Fatal("response changed")
+ }
+}
diff --git a/rag/engine/postgres.go b/rag/engine/postgres.go
index eb97a5e..1e80d32 100644
--- a/rag/engine/postgres.go
+++ b/rag/engine/postgres.go
@@ -11,8 +11,8 @@ import (
"sync"
"github.com/jackc/pgx/v5"
- "github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
+ "github.com/mudler/localrecall/rag/internal/pgutil"
"github.com/mudler/localrecall/rag/types"
"github.com/mudler/xlog"
"github.com/sashabaranov/go-openai"
@@ -187,31 +187,13 @@ const defaultPoolMaxConns = int32(4)
// connection string has priority, then POSTGRES_POOL_MAX_CONNS, then
// defaultPoolMaxConns. Invalid environment values are ignored.
func applyPoolLimits(config *pgxpool.Config, getenv func(string) string) {
- if hasPoolMaxConns(config.ConnString()) {
- return
- }
- config.MaxConns = defaultPoolMaxConns
- if v := getenv("POSTGRES_POOL_MAX_CONNS"); v != "" {
- n, err := strconv.ParseInt(v, 10, 32)
- if err != nil || n < 1 {
- xlog.Warn("Ignoring invalid POSTGRES_POOL_MAX_CONNS", "value", v)
- return
- }
- config.MaxConns = int32(n)
- }
+ pgutil.ApplyPoolLimits(config, getenv)
}
// hasPoolMaxConns reports whether the connection string sets pool_max_conns.
// pgxpool.ParseConfig removes the setting after it reads it, so parse the
// string again at the connection level, where it is kept as a runtime param.
-func hasPoolMaxConns(connString string) bool {
- cfg, err := pgconn.ParseConfig(connString)
- if err != nil {
- return false
- }
- _, ok := cfg.RuntimeParams["pool_max_conns"]
- return ok
-}
+func hasPoolMaxConns(connString string) bool { return pgutil.HasPoolMaxConns(connString) }
// applyConnTimeouts sets per-connection safety timeouts on the pool config so
// that a single wedged or corrupt index can never hang a backend forever and
@@ -237,22 +219,7 @@ func hasPoolMaxConns(connString string) bool {
//
// Any value of "0" or "off" is treated as an explicit opt-out.
func applyConnTimeouts(config *pgxpool.Config, getenv func(string) string) {
- if config.ConnConfig.RuntimeParams == nil {
- config.ConnConfig.RuntimeParams = map[string]string{}
- }
- set := func(param, env, def string) {
- v := def
- if e := getenv(env); e != "" {
- v = e
- }
- if v == "" || v == "0" || strings.EqualFold(v, "off") {
- return
- }
- config.ConnConfig.RuntimeParams[param] = v
- }
- set("lock_timeout", "POSTGRES_LOCK_TIMEOUT", "30s")
- set("idle_in_transaction_session_timeout", "POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT", "300s")
- set("statement_timeout", "POSTGRES_STATEMENT_TIMEOUT", "")
+ pgutil.ApplyConnTimeouts(config, getenv)
}
// execNoStatementTimeout runs a DDL statement (typically a CREATE INDEX) with
@@ -362,10 +329,7 @@ func (p *PostgresDB) setupDatabase() error {
if err := p.ensureBM25IndexConfig(ctx, indexName); err != nil {
return fmt.Errorf("failed to ensure BM25 index text_config: %w", err)
}
- err = p.execNoStatementTimeout(ctx, fmt.Sprintf(`
- CREATE INDEX IF NOT EXISTS %s ON %s
- USING bm25(full_text) WITH (text_config='%s')
- `, indexName, p.tableName, p.bm25TextConfig))
+ err = p.execNoStatementTimeout(ctx, pgutil.BM25IndexSQL(p.tableName, indexName, "full_text", p.bm25TextConfig))
if err != nil {
return fmt.Errorf("failed to create BM25 index (required for hybrid search): %w", err)
}
@@ -453,10 +417,7 @@ func (p *PostgresDB) createVectorIndex(ctx context.Context, vectorscaleInstalled
indexName := fmt.Sprintf("idx_%s_embedding", p.tableName)
if vectorscaleInstalled {
- err := p.execNoStatementTimeout(ctx, fmt.Sprintf(`
- CREATE INDEX IF NOT EXISTS %s ON %s
- USING diskann(embedding)
- `, indexName, p.tableName))
+ err := p.execNoStatementTimeout(ctx, pgutil.VectorIndexSQL(p.tableName, indexName, "embedding", true))
if err == nil {
xlog.Info("Created DiskANN index for vector search")
return nil
@@ -464,10 +425,7 @@ func (p *PostgresDB) createVectorIndex(ctx context.Context, vectorscaleInstalled
xlog.Warn("Failed to create DiskANN index, trying HNSW", "error", err)
}
- err := p.execNoStatementTimeout(ctx, fmt.Sprintf(`
- CREATE INDEX IF NOT EXISTS %s ON %s
- USING hnsw(embedding vector_cosine_ops)
- `, indexName, p.tableName))
+ err := p.execNoStatementTimeout(ctx, pgutil.VectorIndexSQL(p.tableName, indexName, "embedding", false))
if err != nil {
xlog.Warn("Failed to create HNSW index", "error", err)
return nil
@@ -653,13 +611,7 @@ func (p *PostgresDB) migrateEmbeddingDimensions(ctx context.Context) error {
return nil
}
-func formatVector(vec []float32) string {
- parts := make([]string, len(vec))
- for i, v := range vec {
- parts[i] = fmt.Sprintf("%.6f", v)
- }
- return "[" + strings.Join(parts, ",") + "]"
-}
+func formatVector(vec []float32) string { return pgutil.FormatVector(vec) }
func (p *PostgresDB) Count() int {
ctx := context.Background()
diff --git a/rag/engine/postgres_planning_test.go b/rag/engine/postgres_planning_test.go
index a01c2cf..2652cc1 100644
--- a/rag/engine/postgres_planning_test.go
+++ b/rag/engine/postgres_planning_test.go
@@ -51,6 +51,7 @@ var _ = Describe("hybrid search query planning (LocalAI issue #10186)", func() {
collectionName: collectionName,
tableName: sanitizeTableName(collectionName),
embeddingDims: dims,
+ bm25TextConfig: "english",
bm25Weight: 0.5,
vectorWeight: 0.5,
}
diff --git a/rag/engine/postgres_pool_test.go b/rag/engine/postgres_pool_test.go
index 1c4514a..482ac67 100644
--- a/rag/engine/postgres_pool_test.go
+++ b/rag/engine/postgres_pool_test.go
@@ -178,8 +178,8 @@ var _ = Describe("NewPostgresDBCollection pool lifecycle", func() {
}, 5*time.Second, 100*time.Millisecond).Should(Equal(0))
})
- // Without the BM25 extension the schema setup fails after the pool
- // is open and pinged. That pool must be closed too.
+ // An invalid lexical configuration fails schema setup after the pool
+ // is open and pinged, even when BM25 is installed.
It("keeps no server connection open after a failed schema setup", func() {
client, _, stop := newEmbedder(false)
defer stop()
@@ -187,12 +187,23 @@ var _ = Describe("NewPostgresDBCollection pool lifecycle", func() {
appName := fmt.Sprintf("localrecall_leak_test_%d", time.Now().UnixNano())
url := withApplicationName(databaseURL, appName)
- db, err := NewPostgresDBCollection(fmt.Sprintf("leak_test_%d", time.Now().UnixNano()), url, client, "fake")
- if err == nil {
- _, _ = db.pool.Exec(context.Background(), "DROP TABLE IF EXISTS "+db.tableName+" CASCADE")
- db.Close()
- Skip("schema setup succeeds on this database; the failure path cannot be exercised")
- }
+ oldConfig, hadConfig := os.LookupEnv("BM25_TEXT_CONFIG")
+ Expect(os.Setenv("BM25_TEXT_CONFIG", "lr_nonexistent_text_config")).To(Succeed())
+ defer func() {
+ if hadConfig {
+ _ = os.Setenv("BM25_TEXT_CONFIG", oldConfig)
+ } else {
+ _ = os.Unsetenv("BM25_TEXT_CONFIG")
+ }
+ }()
+ name := fmt.Sprintf("leak_test_%d", time.Now().UnixNano())
+ adminCleanup, err := pgxpool.New(context.Background(), databaseURL)
+ Expect(err).NotTo(HaveOccurred())
+ defer adminCleanup.Close()
+ defer func() {
+ _, _ = adminCleanup.Exec(context.Background(), "DROP TABLE IF EXISTS "+sanitizeTableName(name)+" CASCADE")
+ }()
+ _, err = NewPostgresDBCollection(name, url, client, "fake")
Expect(err).To(MatchError(ContainSubstring("failed to setup database")))
pools := rec.all()
Expect(pools).To(HaveLen(1))
diff --git a/rag/engine/postgres_test.go b/rag/engine/postgres_test.go
index 82546ee..f8f8f0f 100644
--- a/rag/engine/postgres_test.go
+++ b/rag/engine/postgres_test.go
@@ -59,7 +59,10 @@ var _ = Describe("PostgresDB", func() {
// Use PostgreSQL from docker-compose
// Connection string for docker-compose postgres service
- databaseURL = "postgresql://localrecall:localrecall@localhost:5432/localrecall?sslmode=disable"
+ databaseURL = os.Getenv("LOCALRECALL_TEST_DATABASE_URL")
+ if databaseURL == "" {
+ databaseURL = "postgresql://localrecall:localrecall@localhost:5432/localrecall?sslmode=disable"
+ }
// Verify PostgreSQL is accessible
ctx := context.Background()
diff --git a/rag/internal/pgutil/config.go b/rag/internal/pgutil/config.go
new file mode 100644
index 0000000..a065bac
--- /dev/null
+++ b/rag/internal/pgutil/config.go
@@ -0,0 +1,60 @@
+package pgutil
+
+import (
+ "fmt"
+ "github.com/jackc/pgx/v5/pgconn"
+ "github.com/jackc/pgx/v5/pgxpool"
+ "github.com/mudler/xlog"
+ "strconv"
+ "strings"
+)
+
+const defaultPoolMaxConns = int32(4)
+
+func ApplyPoolLimits(config *pgxpool.Config, getenv func(string) string) {
+ if HasPoolMaxConns(config.ConnString()) {
+ return
+ }
+ config.MaxConns = defaultPoolMaxConns
+ if v := getenv("POSTGRES_POOL_MAX_CONNS"); v != "" {
+ n, err := strconv.ParseInt(v, 10, 32)
+ if err != nil || n < 1 {
+ xlog.Warn("Ignoring invalid POSTGRES_POOL_MAX_CONNS", "value", v)
+ return
+ }
+ config.MaxConns = int32(n)
+ }
+}
+func HasPoolMaxConns(connString string) bool {
+ cfg, err := pgconn.ParseConfig(connString)
+ if err != nil {
+ return false
+ }
+ _, ok := cfg.RuntimeParams["pool_max_conns"]
+ return ok
+}
+func ApplyConnTimeouts(config *pgxpool.Config, getenv func(string) string) {
+ if config.ConnConfig.RuntimeParams == nil {
+ config.ConnConfig.RuntimeParams = map[string]string{}
+ }
+ set := func(param, env, def string) {
+ v := def
+ if e := getenv(env); e != "" {
+ v = e
+ }
+ if v == "" || v == "0" || strings.EqualFold(v, "off") {
+ return
+ }
+ config.ConnConfig.RuntimeParams[param] = v
+ }
+ set("lock_timeout", "POSTGRES_LOCK_TIMEOUT", "30s")
+ set("idle_in_transaction_session_timeout", "POSTGRES_IDLE_IN_TRANSACTION_TIMEOUT", "300s")
+ set("statement_timeout", "POSTGRES_STATEMENT_TIMEOUT", "")
+}
+func FormatVector(vec []float32) string {
+ parts := make([]string, len(vec))
+ for i, v := range vec {
+ parts[i] = fmt.Sprintf("%.6f", v)
+ }
+ return "[" + strings.Join(parts, ",") + "]"
+}
diff --git a/rag/internal/pgutil/config_test.go b/rag/internal/pgutil/config_test.go
new file mode 100644
index 0000000..74e97d5
--- /dev/null
+++ b/rag/internal/pgutil/config_test.go
@@ -0,0 +1,53 @@
+package pgutil
+
+import (
+ "github.com/jackc/pgx/v5/pgxpool"
+ "testing"
+)
+
+func TestPoolCompatibility(t *testing.T) {
+ for _, tc := range []struct {
+ dsn, env string
+ want int32
+ }{
+ {"postgres://fixture/db", "", 4},
+ {"postgres://fixture/db", "7", 7},
+ {"postgres://fixture/db", "invalid", 4},
+ {"postgres://fixture/db?pool_max_conns=9", "7", 9},
+ } {
+ c, e := pgxpool.ParseConfig(tc.dsn)
+ if e != nil {
+ t.Fatal(e)
+ }
+ ApplyPoolLimits(c, func(string) string { return tc.env })
+ if c.MaxConns != tc.want {
+ t.Fatalf("%+v: %d", tc, c.MaxConns)
+ }
+ }
+ c, e := pgxpool.ParseConfig("postgres://fixture/db")
+ if e != nil {
+ t.Fatal(e)
+ }
+ ApplyConnTimeouts(c, func(string) string { return "" })
+ if c.ConnConfig.RuntimeParams["lock_timeout"] != "30s" || c.ConnConfig.RuntimeParams["idle_in_transaction_session_timeout"] != "300s" || c.ConnConfig.RuntimeParams["statement_timeout"] != "" {
+ t.Fatal(c.ConnConfig.RuntimeParams)
+ }
+}
+
+func TestIndexSQLCompatibility(t *testing.T) {
+ if got := VectorIndexSQL("documents", "idx_embedding", "embedding", false); got != "CREATE INDEX IF NOT EXISTS idx_embedding ON documents USING hnsw (embedding vector_cosine_ops)" {
+ t.Fatal(got)
+ }
+ if got := VectorIndexSQL("documents", "idx_embedding", "embedding", true); got != "CREATE INDEX IF NOT EXISTS idx_embedding ON documents USING diskann (embedding)" {
+ t.Fatal(got)
+ }
+ if got := BM25IndexSQL("documents", "idx_bm25", "full_text", "english"); got != "CREATE INDEX IF NOT EXISTS idx_bm25 ON documents USING bm25 (full_text) WITH (text_config='english')" {
+ t.Fatal(got)
+ }
+ if got := Vector([]float32{0.12345678}); got != "[0.12345678]" {
+ t.Fatal(got)
+ }
+ if got := FormatVector([]float32{0.12345678}); got != "[0.123457]" {
+ t.Fatal(got)
+ }
+}
diff --git a/rag/internal/pgutil/index.go b/rag/internal/pgutil/index.go
new file mode 100644
index 0000000..16aa028
--- /dev/null
+++ b/rag/internal/pgutil/index.go
@@ -0,0 +1,80 @@
+// Package pgutil contains PostgreSQL primitives with no embedding dependencies.
+package pgutil
+
+import (
+ "context"
+ "fmt"
+ "github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgconn"
+ "strconv"
+ "strings"
+)
+
+type DB interface {
+ Exec(context.Context, string, ...any) (pgconn.CommandTag, error)
+ QueryRow(context.Context, string, ...any) pgx.Row
+}
+
+// Vector preserves float32 round-trip precision; FormatVector preserves legacy six decimals.
+func Vector(v []float32) string {
+ p := make([]string, len(v))
+ for i, x := range v {
+ p[i] = strconv.FormatFloat(float64(x), 'g', -1, 32)
+ }
+ return "[" + strings.Join(p, ",") + "]"
+}
+func ResolveTextConfig(ctx context.Context, db DB, name string) (string, error) {
+ var v string
+ err := db.QueryRow(ctx, `SELECT format('%I.%I', n.nspname, c.cfgname) FROM pg_ts_config c JOIN pg_namespace n ON n.oid=c.cfgnamespace WHERE c.oid=$1::regconfig`, name).Scan(&v)
+ return v, err
+}
+func HasAccessMethod(ctx context.Context, db DB, name string) bool {
+ var yes bool
+ err := db.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_am WHERE amname=$1)`, name).Scan(&yes)
+ return err == nil && yes
+}
+
+// VectorIndexSQL requires trusted SQL identifier fragments for table, index and
+// column (for dynamic identifiers, use pgx.Identifier.Sanitize).
+func VectorIndexSQL(table, index, column string, diskANN bool) string {
+ method := "hnsw (" + column + " vector_cosine_ops)"
+ if diskANN {
+ method = "diskann (" + column + ")"
+ }
+ return fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s USING %s", index, table, method)
+}
+
+// BM25IndexSQL requires trusted SQL identifier fragments for table, index and
+// column. config is a value and is escaped as a SQL string literal.
+func BM25IndexSQL(table, index, column, config string) string {
+ return fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s USING bm25 (%s) WITH (text_config='%s')", index, table, column, strings.ReplaceAll(config, "'", "''"))
+}
+func CreateBM25(ctx context.Context, db DB, table, index, column, config string) error {
+ _, err := db.Exec(ctx, BM25IndexSQL(table, index, column, config))
+ return err
+}
+
+// CreateVectorIndex uses a savepoint so a failed DiskANN build can safely try HNSW.
+func CreateVectorIndex(ctx context.Context, tx pgx.Tx, table, index, column string, dimensions int) error {
+ if dimensions < 1 || dimensions > 2000 {
+ return fmt.Errorf("unsupported dimensions")
+ }
+ if HasAccessMethod(ctx, tx, "diskann") {
+ if _, err := tx.Exec(ctx, "SAVEPOINT raw_ann"); err != nil {
+ return err
+ }
+ _, err := tx.Exec(ctx, VectorIndexSQL(table, index, column, true))
+ if err == nil {
+ _, err = tx.Exec(ctx, "RELEASE SAVEPOINT raw_ann")
+ return err
+ }
+ if _, err = tx.Exec(ctx, "ROLLBACK TO SAVEPOINT raw_ann"); err != nil {
+ return err
+ }
+ }
+ if !HasAccessMethod(ctx, tx, "hnsw") {
+ return nil
+ } // exact-only readiness, never an implicit search fallback
+ _, err := tx.Exec(ctx, VectorIndexSQL(table, index, column, false))
+ return err
+}
diff --git a/rag/internal/pgutil/search.go b/rag/internal/pgutil/search.go
new file mode 100644
index 0000000..9a90196
--- /dev/null
+++ b/rag/internal/pgutil/search.go
@@ -0,0 +1,55 @@
+package pgutil
+
+import (
+ "context"
+ "strings"
+)
+
+// BM25Distance binds a query to its index's declared text configuration. SQL
+// fragments are trusted identifiers/parameters, never caller text. Parentheses
+// matter: PostgreSQL otherwise binds unary minus before the <@> operator.
+func BM25Distance(column, query, index string) string {
+ return column + " <@> to_bm25query(" + query + ", '" + strings.ReplaceAll(index, "'", "''") + "')"
+}
+
+// IndexStrategy reports a configured valid, ready, non-partial index, NOT the
+// plan executed by PostgreSQL. Callers hold a relation lock while using it.
+// ANN indexes must support the query's pgvector cosine ordering, not merely
+// have an ANN access method. Extension membership binds type/operator identity;
+// visibility ensures the unqualified query resolves to those same objects.
+// BM25 retains its existing method-only check.
+func IndexStrategy(ctx context.Context, db DB, table, index, column string) (string, error) {
+ var strategy string
+ err := db.QueryRow(ctx, `
+SELECT am.amname
+FROM pg_catalog.pg_index i
+JOIN pg_catalog.pg_class c ON c.oid=i.indexrelid
+JOIN pg_catalog.pg_am am ON am.oid=c.relam
+JOIN pg_catalog.pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=i.indkey[0]
+WHERE i.indrelid=$1::regclass AND c.relname=$2 AND a.attname=$3
+ AND i.indisvalid AND i.indisready AND i.indpred IS NULL AND i.indexprs IS NULL
+ AND (am.amname NOT IN ('hnsw', 'diskann') OR EXISTS (
+ SELECT 1
+ FROM pg_catalog.pg_opclass oc
+ JOIN pg_catalog.pg_amop ao ON ao.amopfamily=oc.opcfamily AND ao.amopmethod=am.oid
+ JOIN pg_catalog.pg_operator op ON op.oid=ao.amopopr
+ JOIN pg_catalog.pg_type vt ON vt.oid=a.atttypid
+ JOIN pg_catalog.pg_extension ext ON ext.extname='vector'
+ JOIN pg_catalog.pg_depend td ON td.classid='pg_catalog.pg_type'::regclass
+ AND td.objid=vt.oid AND td.refclassid='pg_catalog.pg_extension'::regclass
+ AND td.refobjid=ext.oid AND td.deptype='e'
+ JOIN pg_catalog.pg_depend od ON od.classid='pg_catalog.pg_operator'::regclass
+ AND od.objid=op.oid AND od.refclassid='pg_catalog.pg_extension'::regclass
+ AND od.refobjid=ext.oid AND od.deptype='e'
+ JOIN pg_catalog.pg_opfamily sf ON sf.oid=ao.amopsortfamily
+ JOIN pg_catalog.pg_am sam ON sam.oid=sf.opfmethod
+ WHERE oc.oid=i.indclass[0] AND oc.opcmethod=am.oid AND oc.opcintype=vt.oid
+ AND vt.typname='vector' AND pg_catalog.pg_type_is_visible(vt.oid)
+ AND op.oprname='<=>' AND op.oprleft=vt.oid AND op.oprright=vt.oid
+ AND pg_catalog.pg_operator_is_visible(op.oid)
+ AND ao.amoplefttype=vt.oid AND ao.amoprighttype=vt.oid AND ao.amoppurpose='o'
+ AND sf.opfnamespace='pg_catalog'::regnamespace AND sf.opfname='float_ops'
+ AND sam.amname='btree'
+ ))`, table, index, column).Scan(&strategy)
+ return strategy, err
+}
diff --git a/rag/raw/cursor.go b/rag/raw/cursor.go
new file mode 100644
index 0000000..4a3ecba
--- /dev/null
+++ b/rag/raw/cursor.go
@@ -0,0 +1,117 @@
+package raw
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/hex"
+ "encoding/json"
+ "strings"
+)
+
+type cursor struct {
+ Version int `json:"version"`
+ Kind string `json:"kind"`
+ CollectionID string `json:"collection_id"`
+ LastID string `json:"last_id"`
+ Binding string `json:"binding"`
+}
+
+func ValidateCollectionID(id string) error {
+ if len(id) != 36 || id == "00000000-0000-0000-0000-000000000000" {
+ return invalid()
+ }
+ for i, c := range id {
+ if i == 8 || i == 13 || i == 18 || i == 23 {
+ if c != '-' {
+ return invalid()
+ }
+ } else if !strings.ContainsRune("0123456789abcdef", c) {
+ return invalid()
+ }
+ }
+ return nil
+}
+func binding(f *Filter, v bool) (string, error) {
+ if e := ValidateFilter(f); e != nil {
+ return "", e
+ }
+ m := map[string]string{}
+ if f != nil {
+ for k, v := range f.MetadataEq {
+ m[k] = v
+ }
+ }
+ b, _ := json.Marshal(struct {
+ Metadata map[string]string
+ Vectors bool
+ }{m, v})
+ h := sha256.Sum256(b)
+ return hex.EncodeToString(h[:]), nil
+}
+func encode(c cursor) (string, error) {
+ var b bytes.Buffer
+ enc := json.NewEncoder(&b)
+ // Cursors are base64, not HTML. Without HTML escaping, valid IDs expand
+ // at most 2x (quotes, backslashes, or U+2028/U+2029), fitting the cap.
+ enc.SetEscapeHTML(false)
+ if e := enc.Encode(c); e != nil {
+ return "", NewError(CodeCursor)
+ }
+ s := base64.RawURLEncoding.EncodeToString(bytes.TrimSuffix(b.Bytes(), []byte{'\n'}))
+ if len(s) > MaxCursorBytes {
+ return "", NewError(CodeCursor)
+ }
+ return s, nil
+}
+func decode(s string) (cursor, error) {
+ var c cursor
+ if s == "" || len(s) > MaxCursorBytes {
+ return c, NewError(CodeCursor)
+ }
+ b, e := base64.RawURLEncoding.Strict().DecodeString(s)
+ if e != nil || DecodeRequest(b, &c) != nil || c.Version != 1 {
+ return c, NewError(CodeCursor)
+ }
+ return c, nil
+}
+func EncodeRecordCursor(cid, lastID string, f *Filter, includeVectors bool) (string, error) {
+ if ValidateCollectionID(cid) != nil || ValidateID(lastID) != nil {
+ return "", NewError(CodeCursor)
+ }
+ h, e := binding(f, includeVectors)
+ if e != nil {
+ return "", e
+ }
+ return encode(cursor{1, "records", cid, lastID, h})
+}
+func DecodeRecordCursor(s, cid string, f *Filter, includeVectors bool) (string, error) {
+ c, e := decode(s)
+ if e != nil {
+ return "", e
+ }
+ h, e := binding(f, includeVectors)
+ if e != nil {
+ return "", e
+ }
+ if ValidateCollectionID(cid) != nil || c.Kind != "records" || c.CollectionID != cid || ValidateID(c.LastID) != nil || c.Binding != h {
+ return "", NewError(CodeCursor)
+ }
+ return c.LastID, nil
+}
+func EncodeCollectionCursor(lastID string) (string, error) {
+ if ValidateCollectionID(lastID) != nil {
+ return "", NewError(CodeCursor)
+ }
+ return encode(cursor{Version: 1, Kind: "collections", LastID: lastID})
+}
+func DecodeCollectionCursor(s string) (string, error) {
+ c, e := decode(s)
+ if e != nil {
+ return "", e
+ }
+ if c.Kind != "collections" || c.CollectionID != "" || c.Binding != "" || ValidateCollectionID(c.LastID) != nil {
+ return "", NewError(CodeCursor)
+ }
+ return c.LastID, nil
+}
diff --git a/rag/raw/cursor_test.go b/rag/raw/cursor_test.go
new file mode 100644
index 0000000..1281aba
--- /dev/null
+++ b/rag/raw/cursor_test.go
@@ -0,0 +1,84 @@
+package raw
+
+import (
+ "encoding/base64"
+ "encoding/json"
+ "strings"
+ "testing"
+)
+
+func TestRawCursorMaxLengthIDRoundTrip(t *testing.T) {
+ cid := descriptor().CollectionID
+ for _, tc := range []struct {
+ name string
+ unit string
+ }{
+ {"less_than", "<"},
+ {"greater_than", ">"},
+ {"ampersand", "&"},
+ {"quote", "\""},
+ {"backslash", "\\"},
+ {"mixed_escapes", "<>&\"\\"},
+ {"two_byte_unicode", "é"},
+ {"three_byte_unicode", "界"},
+ {"four_byte_unicode", "😀"},
+ {"line_separator", "\u2028"},
+ {"paragraph_separator", "\u2029"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ id := strings.Repeat(tc.unit, 256/len(tc.unit))
+ id += strings.Repeat("x", 256-len(id))
+ if len(id) != 256 || ValidateID(id) != nil {
+ t.Fatal("fixture must be a valid maximum-length ID")
+ }
+ encoded, err := EncodeRecordCursor(cid, id, nil, false)
+ if err != nil {
+ t.Fatal("encode maximum-length ID:", err)
+ }
+ if len(encoded) > MaxCursorBytes {
+ t.Fatalf("cursor length %d exceeds %d", len(encoded), MaxCursorBytes)
+ }
+ decoded, err := DecodeRecordCursor(encoded, cid, nil, false)
+ if err != nil || decoded != id {
+ t.Fatalf("round trip: got %q, error %v", decoded, err)
+ }
+ })
+ }
+}
+
+func TestRawCursorUntrusted(t *testing.T) {
+ cid := descriptor().CollectionID
+ c, _ := EncodeRecordCursor(cid, "last", nil, false)
+ b, _ := base64.RawURLEncoding.DecodeString(c)
+ var fields map[string]any
+ _ = json.Unmarshal(b, &fields)
+ for _, tc := range []struct {
+ key string
+ value any
+ }{{"version", 2}, {"version", nil}, {"kind", "collections"}, {"last_id", "\x00"}, {"last_id", ""}, {"binding", strings.Repeat("a", 64)}, {"collection_id", "not uuid"}, {"unexpected", true}} {
+ t.Run(tc.key, func(t *testing.T) {
+ m := map[string]any{}
+ for k, v := range fields {
+ m[k] = v
+ }
+ m[tc.key] = tc.value
+ b, _ := json.Marshal(m)
+ if _, e := DecodeRecordCursor(base64.RawURLEncoding.EncodeToString(b), cid, nil, false); e == nil {
+ t.Fatal(m)
+ }
+ })
+ }
+ if _, e := DecodeCollectionCursor(c); e == nil {
+ t.Fatal("record cursor reused for collections")
+ }
+ a, _ := EncodeRecordCursor(cid, "last", &Filter{MetadataEq: map[string]string{"a": "b", "c": "d"}}, true)
+ if _, e := DecodeRecordCursor(a, cid, &Filter{MetadataEq: map[string]string{"c": "d", "a": "b"}}, true); e != nil {
+ t.Fatal("map ordering", e)
+ }
+ if _, e := EncodeRecordCursor(cid, "\n", nil, false); e == nil {
+ t.Fatal("invalid last ID")
+ }
+ if _, e := EncodeCollectionCursor("bad"); e == nil {
+ t.Fatal("invalid UUID")
+ }
+}
diff --git a/rag/raw/postgres/acceptance_test.go b/rag/raw/postgres/acceptance_test.go
new file mode 100644
index 0000000..4522322
--- /dev/null
+++ b/rag/raw/postgres/acceptance_test.go
@@ -0,0 +1,211 @@
+package postgres
+
+import (
+ "context"
+ "os"
+ "strings"
+ "testing"
+
+ "github.com/mudler/localrecall/rag/internal/pgutil"
+ "github.com/mudler/localrecall/rag/raw"
+)
+
+func TestRawLifecycleSharedCursor(t *testing.T) {
+ s, ctx := fixture(t)
+ for i := 0; i < 2; i++ {
+ c, _, e := s.CreateCollection(ctx, request())
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer s.DeleteCollection(ctx, c.CollectionID)
+ }
+ n := 1
+ p, e := s.ListCollections(ctx, raw.ListCollectionsRequest{Limit: &n})
+ if e != nil || p.NextCursor == nil {
+ t.Fatalf("%+v %v", p, e)
+ }
+ last, e := raw.DecodeCollectionCursor(*p.NextCursor)
+ if e != nil || last != p.Collections[0].CollectionID {
+ t.Fatalf("shared cursor: %s %v", last, e)
+ }
+ p, e = s.ListCollections(ctx, raw.ListCollectionsRequest{Limit: &n, Cursor: p.NextCursor})
+ if e != nil || len(p.Collections) != 1 || p.Collections[0].CollectionID <= last {
+ t.Fatalf("next: %+v %v", p, e)
+ }
+}
+
+func TestRawCapabilitiesSchemaDegradation(t *testing.T) {
+ s, ctx := fixture(t)
+ exec(t, s, "UPDATE lr_raw_schema SET version=2")
+ defer exec(t, s, "UPDATE lr_raw_schema SET version=1")
+ c, e := s.Capabilities(ctx)
+ if e != nil || c.Ready || c.ModeReadiness["vector"].Ready {
+ t.Fatalf("corrupt schema ready: %+v %v", c, e)
+ }
+}
+
+func TestRawLifecycleWithoutBM25(t *testing.T) {
+ s, ctx := fixture(t)
+ exec(t, s, "DROP EXTENSION pg_textsearch")
+ defer exec(t, s, "CREATE EXTENSION pg_textsearch VERSION '1.3.1'")
+ c, e := s.Capabilities(ctx)
+ if e != nil || !c.Ready || c.ModeReadiness["lexical"].Ready {
+ t.Fatalf("degraded: %+v %v", c, e)
+ }
+ var before, after int
+ count := `SELECT count(*) FROM pg_class WHERE relname LIKE 'lr_raw_%'`
+ if e = s.pool.QueryRow(ctx, count).Scan(&before); e != nil {
+ t.Fatal(e)
+ }
+ _, _, e = s.CreateCollection(ctx, request())
+ checkCode(t, e, raw.CodeUnavailable)
+ if e = s.pool.QueryRow(ctx, count).Scan(&after); e != nil || before != after {
+ t.Fatalf("DDL leaked %d %d %v", before, after, e)
+ }
+ r := request()
+ no := false
+ r.Lexical = &raw.LexicalRequest{Enabled: &no}
+ got, _, e := s.CreateCollection(ctx, r)
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer s.DeleteCollection(ctx, got.CollectionID)
+}
+
+func TestRawLifecycleSnapshotAndRestart(t *testing.T) {
+ s, ctx := fixture(t)
+ t.Setenv("BM25_TEXT_CONFIG", "simple")
+ c, _, e := s.CreateCollection(ctx, request())
+ if e != nil {
+ t.Fatal(e)
+ }
+ t.Setenv("BM25_TEXT_CONFIG", "english")
+ s.Close()
+ reopened, e := New(ctx, Config{os.Getenv("LOCALRECALL_TEST_DATABASE_URL")})
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer reopened.Close()
+ defer reopened.DeleteCollection(ctx, c.CollectionID)
+ got, e := reopened.GetCollection(ctx, c.CollectionID)
+ if e != nil || got.Lexical.TextConfig != "pg_catalog.simple" {
+ t.Fatalf("snapshot %+v %v", got, e)
+ }
+ var legacy bool
+ if e = reopened.pool.QueryRow(ctx, `SELECT to_regclass('collection_config') IS NOT NULL`).Scan(&legacy); e != nil || legacy {
+ t.Fatalf("legacy registry touched: %v %v", legacy, e)
+ }
+}
+
+func TestRawPoolCorruptTableCleanup(t *testing.T) {
+ s, ctx := fixture(t)
+ c, _, e := s.CreateCollection(ctx, request())
+ if e != nil {
+ t.Fatal(e)
+ }
+ table, _ := tableName(c.CollectionID)
+ exec(t, s, "ALTER TABLE "+table+" ALTER COLUMN metadata DROP NOT NULL")
+ defer exec(t, s, "DELETE FROM lr_raw_collections WHERE collection_id=$1", c.CollectionID)
+ defer exec(t, s, "DROP TABLE "+table)
+ url := os.Getenv("LOCALRECALL_TEST_DATABASE_URL") + "&application_name=raw_failed_constructor"
+ for i := 0; i < 3; i++ {
+ broken, e := New(ctx, Config{url})
+ if broken != nil {
+ broken.Close()
+ t.Fatal("corruption admitted")
+ }
+ checkCode(t, e, raw.CodeUnavailable)
+ }
+ var n int
+ if e = s.pool.QueryRow(ctx, `SELECT count(*) FROM pg_stat_activity WHERE application_name='raw_failed_constructor'`).Scan(&n); e != nil || n != 0 {
+ t.Fatalf("constructor connections leaked: %d %v", n, e)
+ }
+ _, e = s.GetCollection(ctx, c.CollectionID)
+ checkCode(t, e, raw.CodeUnavailable)
+}
+
+func TestRawLegacyHelpers(t *testing.T) {
+ s, ctx := fixture(t)
+ tx, e := s.pool.Begin(ctx)
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer tx.Rollback(context.Background())
+ _, e = tx.Exec(ctx, `CREATE TABLE legacy_helper_fixture (embedding vector(3),full_text text)`)
+ if e != nil {
+ t.Fatal(e)
+ }
+ for _, q := range []string{pgutil.BM25IndexSQL("legacy_helper_fixture", "legacy_bm25", "full_text", "english"), pgutil.VectorIndexSQL("legacy_helper_fixture", "legacy_ann", "embedding", false)} {
+ if _, e = tx.Exec(ctx, q); e != nil {
+ t.Fatal(e)
+ }
+ }
+ _, e = tx.Exec(ctx, `INSERT INTO legacy_helper_fixture VALUES ('[1,0,0]','legacy regression')`)
+ if e != nil {
+ t.Fatal(e)
+ }
+ var distance float64
+ if e = tx.QueryRow(ctx, `SELECT embedding <=> '[1,0,0]'::vector FROM legacy_helper_fixture`).Scan(&distance); e != nil || distance != 0 {
+ t.Fatal(distance, e)
+ }
+ if e = tx.QueryRow(ctx, `SELECT full_text <@> 'regression' FROM legacy_helper_fixture ORDER BY full_text <@> 'regression' LIMIT 1`).Scan(&distance); e != nil {
+ t.Fatal(e)
+ }
+ if pgutil.FormatVector([]float32{0.12345678}) != "[0.123457]" || !strings.Contains(pgutil.VectorIndexSQL("t", "i", "embedding", true), "diskann (embedding)") {
+ t.Fatal("legacy formatting changed")
+ }
+}
+
+func TestRawCapabilitiesMissingCreatePrivilege(t *testing.T) {
+ s, ctx := fixture(t)
+ role := "raw_probe_denied"
+ exec(t, s, "CREATE ROLE "+role)
+ defer exec(t, s, "DROP ROLE "+role)
+ exec(t, s, "GRANT SELECT ON lr_raw_schema,lr_raw_collections TO "+role)
+ defer exec(t, s, "REVOKE SELECT ON lr_raw_schema,lr_raw_collections FROM "+role)
+ // Bootstrap reads an empty registry successfully, but CREATE capability must
+ // be false even though pgvector and BM25 remain installed and selectable.
+ denied, e := New(ctx, Config{os.Getenv("LOCALRECALL_TEST_DATABASE_URL") + "&options=-c%20role%3D" + role})
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer denied.Close()
+ c, e := denied.Capabilities(ctx)
+ if e != nil || c.Ready {
+ t.Fatalf("unprivileged ready: %+v %v", c, e)
+ }
+ if len(c.Indexes) != 3 || !c.Indexes[1].Installed || c.Indexes[1].Ready {
+ t.Fatalf("installed vs usable: %+v", c.Indexes)
+ }
+ _, _, e = denied.CreateCollection(ctx, request())
+ checkCode(t, e, raw.CodeUnavailable)
+}
+
+func TestRawLifecycleResetStatementSnapshot(t *testing.T) {
+ s, ctx := fixture(t)
+ c, _, e := s.CreateCollection(ctx, request())
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer s.DeleteCollection(ctx, c.CollectionID)
+ table, _ := tableName(c.CollectionID)
+ exec(t, s, "INSERT INTO "+table+" (id,vector) VALUES ('before','[1,0,0]')")
+ writer, e := s.pool.Begin(ctx)
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer writer.Rollback(ctx)
+ if _, e = writer.Exec(ctx, "INSERT INTO "+table+" (id,vector) VALUES ('concurrent','[0,1,0]')"); e != nil {
+ t.Fatal(e)
+ }
+ if e = s.Reset(ctx, c.CollectionID); e != nil {
+ t.Fatal(e)
+ }
+ if e = writer.Commit(ctx); e != nil {
+ t.Fatal(e)
+ }
+ var id string
+ if e = s.pool.QueryRow(ctx, "SELECT id FROM "+table).Scan(&id); e != nil || id != "concurrent" {
+ t.Fatalf("reset became a writer barrier: %s %v", id, e)
+ }
+}
diff --git a/rag/raw/postgres/capabilities.go b/rag/raw/postgres/capabilities.go
new file mode 100644
index 0000000..e6affdd
--- /dev/null
+++ b/rag/raw/postgres/capabilities.go
@@ -0,0 +1,97 @@
+package postgres
+
+import (
+ "context"
+ "os"
+
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
+ "github.com/mudler/localrecall/rag/internal/pgutil"
+ "github.com/mudler/localrecall/rag/raw"
+)
+
+func readiness(ok bool) raw.Readiness {
+ if ok {
+ return raw.Readiness{Ready: true}
+ }
+ return raw.Readiness{Reason: "Required dependency unavailable"}
+}
+
+// Capabilities probes actual DDL and operators under the service role. All probe
+// objects are rolled back; installed access methods alone do not imply usability.
+func (s *Service) Capabilities(ctx context.Context) (raw.Capabilities, error) {
+ c := raw.Capabilities{Backend: "postgres", RawCollections: true, Modes: []string{"vector", "lexical", "hybrid"}, AccuracyOptions: []string{"approximate", "exact"}, DefaultAccuracy: "approximate", Metrics: []string{"cosine"}, Limits: raw.ContractLimits(), AtomicBatch: true, Persistent: true, ModeReadiness: map[string]raw.Readiness{}}
+ for _, mode := range c.Modes {
+ c.ModeReadiness[mode] = readiness(false)
+ }
+ c.Readiness = readiness(false)
+ ctx, done, e := s.operation(ctx)
+ if e != nil {
+ return c, nil
+ }
+ defer done()
+ tx, e := s.pool.Begin(ctx)
+ if e != nil {
+ return c, nil
+ }
+ defer tx.Rollback(context.Background())
+ installed := map[string]bool{}
+ for _, name := range []string{"diskann", "hnsw", "bm25"} {
+ installed[name] = pgutil.HasAccessMethod(ctx, tx, name)
+ c.Indexes = append(c.Indexes, raw.IndexCapability{Strategy: name, Installed: installed[name], Readiness: readiness(false)})
+ }
+ if schemaVersion(ctx, tx) != nil {
+ return c, nil
+ }
+ if _, e = tx.Exec(ctx, "SELECT "+columns+" FROM lr_raw_collections LIMIT 0"); e != nil {
+ return c, nil
+ }
+ table := pgx.Identifier{"lr_probe_" + uuid.NewString()}.Sanitize()
+ if _, e = tx.Exec(ctx, "CREATE TABLE "+table+" (id text PRIMARY KEY, vector vector(3), text text)"); e != nil {
+ return c, nil
+ }
+ if _, e = tx.Exec(ctx, "INSERT INTO "+table+" VALUES ('probe','[1,0,0]','readiness probe')"); e != nil {
+ return c, nil
+ }
+ var score float64
+ if e = tx.QueryRow(ctx, "SELECT vector <=> '[1,0,0]'::vector FROM "+table).Scan(&score); e != nil {
+ return c, nil
+ }
+ c.Readiness = readiness(true)
+ c.ModeReadiness["vector"] = readiness(true)
+ for i, name := range []string{"diskann", "hnsw", "bm25"} {
+ if !installed[name] {
+ continue
+ }
+ if _, e = tx.Exec(ctx, "SAVEPOINT capability"); e != nil {
+ return c, nil
+ }
+ index := pgx.Identifier{"lr_probe_index_" + uuid.NewString()}.Sanitize()
+ if name == "bm25" {
+ var config string
+ config, e = pgutil.ResolveTextConfig(ctx, tx, raw.ResolveLexical(nil, os.Getenv("BM25_TEXT_CONFIG")).TextConfig)
+ if e == nil {
+ e = pgutil.CreateBM25(ctx, tx, table, index, "text", config)
+ }
+ if e == nil {
+ e = tx.QueryRow(ctx, "SELECT text <@> 'probe' FROM "+table+" ORDER BY text <@> 'probe' LIMIT 1").Scan(&score)
+ }
+ } else {
+ _, e = tx.Exec(ctx, pgutil.VectorIndexSQL(table, index, "vector", name == "diskann"))
+ }
+ ok := e == nil
+ // Isolate each failed dependency and remove each successful probe index too.
+ if _, e = tx.Exec(ctx, "ROLLBACK TO SAVEPOINT capability"); e != nil {
+ return c, nil
+ }
+ if _, e = tx.Exec(ctx, "RELEASE SAVEPOINT capability"); e != nil {
+ return c, nil
+ }
+ c.Indexes[i].Readiness = readiness(ok)
+ if name == "bm25" {
+ c.ModeReadiness["lexical"] = readiness(ok)
+ c.ModeReadiness["hybrid"] = readiness(ok)
+ }
+ }
+ return c, nil
+}
diff --git a/rag/raw/postgres/cursors.go b/rag/raw/postgres/cursors.go
new file mode 100644
index 0000000..e2f31f9
--- /dev/null
+++ b/rag/raw/postgres/cursors.go
@@ -0,0 +1,21 @@
+package postgres
+
+import "github.com/mudler/localrecall/rag/raw"
+
+// Record cursors share the contract's untrusted-input validation and binding.
+// They do not encode a revision: mutations never globally invalidate paging.
+func recordPageInput(id string, r raw.ListRecordsRequest) (limit int, after string, err error) {
+ if err = raw.ValidateCollectionID(id); err != nil {
+ return
+ }
+ if err = raw.ValidateFilter(r.Filter); err != nil {
+ return
+ }
+ if limit, err = raw.ValidateLimit(r.Limit); err != nil {
+ return
+ }
+ if r.Cursor != nil {
+ after, err = raw.DecodeRecordCursor(*r.Cursor, id, r.Filter, r.IncludeVectors)
+ }
+ return
+}
diff --git a/rag/raw/postgres/explain_test.go b/rag/raw/postgres/explain_test.go
new file mode 100644
index 0000000..a0858d9
--- /dev/null
+++ b/rag/raw/postgres/explain_test.go
@@ -0,0 +1,128 @@
+package postgres
+
+import (
+ "encoding/json"
+ "fmt"
+ "math"
+ "sort"
+ "strings"
+ "testing"
+
+ "github.com/mudler/localrecall/rag/internal/pgutil"
+ "github.com/mudler/localrecall/rag/raw"
+)
+
+// Deterministic synthetic corpus, including many nearer excluded records.
+func rankedCorpus(t *testing.T, s *Service, id string) []raw.Record {
+ t.Helper()
+ rs := make([]raw.Record, 600)
+ for i := range rs {
+ group := "other"
+ if i%17 == 0 {
+ group = "keep"
+ }
+ text := "bicycle"
+ rs[i] = raw.Record{ID: fmt.Sprintf("%04d", i), Vector: []float64{math.Cos(float64(i) * 0.017), math.Sin(float64(i) * 0.017), 0.1}, Text: &text, Metadata: map[string]string{"group": group}}
+ }
+ for i := 0; i < len(rs); i += 100 {
+ put(t, s, t.Context(), id, rs[i:i+100]...)
+ }
+ return rs
+}
+func TestRawExplain(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ rankedCorpus(t, s, id)
+ table, _ := tableName(id)
+ exec(t, s, "ANALYZE "+table)
+ tx, e := s.pool.Begin(ctx)
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer tx.Rollback(ctx)
+ strategy, e := pgutil.IndexStrategy(ctx, tx, table, "lr_ann_"+id, "vector")
+ if e != nil || (strategy != "hnsw" && strategy != "diskann") {
+ t.Fatalf("ANN unavailable: %q %v", strategy, e)
+ }
+ // Bias toward indexes to make the exact fence assertion meaningful.
+ if _, e = tx.Exec(ctx, "SET LOCAL enable_seqscan=off"); e != nil {
+ t.Fatal(e)
+ }
+ for _, exact := range []bool{true, false} {
+ args := []any{"[1,0,0]", 10}
+ predicate := metadataPredicate(&raw.Filter{MetadataEq: map[string]string{"group": "keep"}}, &args)
+ var plan []byte
+ if e = tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) "+vectorQuery(table, predicate, exact), args...).Scan(&plan); e != nil {
+ t.Fatal(e)
+ }
+ if !json.Valid(plan) {
+ t.Fatal(string(plan))
+ }
+ t.Logf("exact=%v dimensions=3 corpus=600 selective=1/17 plan=%s", exact, plan)
+ if exact && (strings.Contains(string(plan), "lr_ann_") || !strings.Contains(string(plan), "CTE Scan")) {
+ t.Fatal("exact ranking must be fenced from ANN", string(plan))
+ }
+ if !exact && !strings.Contains(string(plan), "lr_ann_"+id) {
+ t.Fatal("fixture must demonstrate generated approximate query has an ANN access path", string(plan))
+ }
+ }
+}
+func TestRawRecallBruteForce(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ rs := rankedCorpus(t, s, id)
+ type hit struct {
+ id string
+ score float64
+ }
+ want := []hit{}
+ for _, r := range rs {
+ if r.Metadata["group"] != "keep" {
+ continue
+ }
+ v, e := raw.NormalizeVector(r.Vector, 3)
+ if e != nil {
+ t.Fatal(e)
+ }
+ norm := 0.0
+ for _, x := range v {
+ norm += float64(x) * float64(x)
+ }
+ want = append(want, hit{r.ID, float64(v[0]) / math.Sqrt(norm)})
+ }
+ sort.Slice(want, func(i, j int) bool {
+ if want[i].score == want[j].score {
+ return want[i].id < want[j].id
+ }
+ return want[i].score > want[j].score
+ })
+ r := searchRequest("vector")
+ r.Filter = &raw.Filter{MetadataEq: map[string]string{"group": "keep"}}
+ exact, e := s.Search(ctx, id, r)
+ if e != nil || exact.Count != 10 {
+ t.Fatal(exact, e)
+ }
+ for i, h := range exact.Results {
+ if h.ID != want[i].id || math.Abs(h.Score-want[i].score) > 1e-6 || h.VectorSimilarity == nil || *h.VectorSimilarity != h.Score {
+ t.Fatalf("rank %d got %+v want %+v", i, h, want[i])
+ }
+ }
+ r.Accuracy = nil
+ approx, e := s.Search(ctx, id, r)
+ if e != nil {
+ t.Fatal(e)
+ }
+ hits := 0
+ for _, h := range approx.Results {
+ if h.Metadata["group"] != "keep" {
+ t.Fatal("filter escaped", h)
+ }
+ for _, w := range want[:10] {
+ if w.id == h.ID {
+ hits++
+ }
+ }
+ }
+ if approx.Count > 10 || approx.Accuracy != "approximate" {
+ t.Fatal(approx)
+ }
+ t.Logf("synthetic dimensions=3 corpus=600 eligible=%d k=10 returned=%d recall@10=%.2f underfill=%d (planner-dependent, not an ANN quality guarantee)", len(want), approx.Count, float64(hits)/10, 10-approx.Count)
+}
diff --git a/rag/raw/postgres/filters.go b/rag/raw/postgres/filters.go
new file mode 100644
index 0000000..e91850c
--- /dev/null
+++ b/rag/raw/postgres/filters.go
@@ -0,0 +1,28 @@
+package postgres
+
+import (
+ "fmt"
+ "github.com/mudler/localrecall/rag/raw"
+ "sort"
+ "strings"
+)
+
+// metadataPredicate only accepts validated equality filters. Both keys and
+// values are parameters, including keys that resemble SQL or JSON paths.
+func metadataPredicate(f *raw.Filter, args *[]any) string {
+ if f == nil || len(f.MetadataEq) == 0 {
+ return "TRUE"
+ }
+ keys := make([]string, 0, len(f.MetadataEq))
+ for k := range f.MetadataEq {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ parts := make([]string, 0, len(keys))
+ for _, k := range keys {
+ n := len(*args) + 1
+ *args = append(*args, k, f.MetadataEq[k])
+ parts = append(parts, fmt.Sprintf("metadata ->> $%d::text = $%d::text", n, n+1))
+ }
+ return strings.Join(parts, " AND ")
+}
diff --git a/rag/raw/postgres/lifecycle.go b/rag/raw/postgres/lifecycle.go
new file mode 100644
index 0000000..1c210d8
--- /dev/null
+++ b/rag/raw/postgres/lifecycle.go
@@ -0,0 +1,174 @@
+package postgres
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgconn"
+ "github.com/mudler/localrecall/rag/internal/pgutil"
+ "github.com/mudler/localrecall/rag/raw"
+ "os"
+)
+
+func (s *Service) CreateCollection(ctx context.Context, r raw.CreateCollectionRequest) (c raw.Collection, created bool, err error) {
+ if err = raw.ValidateCreate(r); err != nil {
+ return
+ }
+ lex := raw.ResolveLexical(r.Lexical, os.Getenv("BM25_TEXT_CONFIG"))
+ ctx, done, e := s.operation(ctx)
+ if e != nil {
+ return c, false, e
+ }
+ defer done()
+ // READ COMMITTED gives the post-advisory-lock lookup a fresh snapshot.
+ tx, e := s.pool.Begin(ctx)
+ if e != nil {
+ return c, false, dbError(e)
+ }
+ defer tx.Rollback(context.Background())
+ _, e = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,763214992))`, r.Name)
+ if e != nil {
+ return c, false, dbError(e)
+ }
+ if e = schemaVersion(ctx, tx); e != nil {
+ return c, false, dbError(e)
+ }
+ if lex.Enabled {
+ lex.TextConfig, e = pgutil.ResolveTextConfig(ctx, tx, lex.TextConfig)
+ if e != nil {
+ // Only malformed or unknown catalog names are caller errors.
+ // Cancellation, privileges and dependency failures retain dbError mapping.
+ var pe *pgconn.PgError
+ if errors.As(e, &pe) && (pe.Code == "42704" || pe.Code == "42602" || pe.Code == "42601" || pe.Code == "3F000") {
+ return c, false, raw.NewError(raw.CodeValidation)
+ }
+ return c, false, dbError(e)
+ }
+ }
+ c, e = scan(tx.QueryRow(ctx, "SELECT "+columns+" FROM lr_raw_collections WHERE name=$1", r.Name))
+ if e == nil {
+ if c.Dimensions != r.Dimensions || c.Metric != r.Metric || c.EmbeddingSpaceID != r.EmbeddingSpaceID || c.Lexical != lex {
+ return c, false, raw.NewError(raw.CodeConflict)
+ }
+ if e = validateTable(ctx, tx, c); e != nil {
+ return raw.Collection{}, false, dbError(e)
+ }
+ return c, false, dbError(tx.Commit(ctx))
+ }
+ if e != pgx.ErrNoRows {
+ return c, false, dbError(e)
+ }
+ id := uuid.NewString()
+ table, _ := tableName(id)
+ _, e = tx.Exec(ctx, fmt.Sprintf(`CREATE TABLE %s (id text COLLATE "C" PRIMARY KEY,vector vector(%d) NOT NULL,text text,metadata jsonb NOT NULL DEFAULT '{}')`, table, r.Dimensions))
+ if e != nil {
+ return c, false, dbError(e)
+ }
+ if lex.Enabled {
+ if e = pgutil.CreateBM25(ctx, tx, table, pgx.Identifier{"lr_bm25_" + id}.Sanitize(), "text", lex.TextConfig); e != nil {
+ return c, false, dbError(e)
+ }
+ }
+ if e = pgutil.CreateVectorIndex(ctx, tx, table, pgx.Identifier{"lr_ann_" + id}.Sanitize(), "vector", r.Dimensions); e != nil {
+ return c, false, dbError(e)
+ }
+ c, e = scan(tx.QueryRow(ctx, `INSERT INTO lr_raw_collections(collection_id,name,dimensions,metric,embedding_space_id,lexical_enabled,text_config,schema_version) VALUES($1,$2,$3,$4,$5,$6,$7,1) RETURNING `+columns, id, r.Name, r.Dimensions, r.Metric, r.EmbeddingSpaceID, lex.Enabled, lex.TextConfig))
+ if e != nil {
+ return raw.Collection{}, false, dbError(e)
+ }
+ if e = tx.Commit(ctx); e != nil {
+ return raw.Collection{}, false, dbError(e)
+ }
+ return c, true, nil
+}
+func (s *Service) GetCollection(ctx context.Context, id string) (c raw.Collection, err error) {
+ err = s.transaction(ctx, pgx.RepeatableRead, func(ctx context.Context, tx pgx.Tx) error { var e error; c, e = lookup(ctx, tx, id); return e })
+ if err != nil {
+ c = raw.Collection{}
+ }
+ return
+}
+func (s *Service) Reset(ctx context.Context, id string) error {
+ // Each statement needs a fresh snapshot: inserts committed after lookup
+ // but before DELETE must be cleared. lookup holds the relation lock until
+ // commit, protecting table lifetime without blocking independent writers.
+ return s.transaction(ctx, pgx.ReadCommitted, func(ctx context.Context, tx pgx.Tx) error {
+ c, e := lookup(ctx, tx, id)
+ if e != nil {
+ return e
+ }
+ table, _ := tableName(c.CollectionID)
+ _, e = tx.Exec(ctx, "DELETE FROM "+table)
+ return e
+ })
+}
+func (s *Service) DeleteCollection(ctx context.Context, id string) error {
+ return s.transaction(ctx, pgx.ReadCommitted, func(ctx context.Context, tx pgx.Tx) error {
+ c, e := lookup(ctx, tx, id)
+ if e != nil {
+ return e
+ }
+ table, _ := tableName(c.CollectionID)
+ if _, e = tx.Exec(ctx, "DROP TABLE "+table); e != nil {
+ return e
+ }
+ _, e = tx.Exec(ctx, "DELETE FROM lr_raw_collections WHERE collection_id=$1", id)
+ return e
+ })
+}
+func (s *Service) ListCollections(ctx context.Context, r raw.ListCollectionsRequest) (p raw.CollectionPage, err error) {
+ limit, e := raw.ValidateLimit(r.Limit)
+ if e != nil {
+ return p, e
+ }
+ after := ""
+ if r.Cursor != nil {
+ after, e = raw.DecodeCollectionCursor(*r.Cursor)
+ if e != nil {
+ return p, e
+ }
+ }
+ err = s.transaction(ctx, pgx.RepeatableRead, func(ctx context.Context, tx pgx.Tx) error {
+ if e := schemaVersion(ctx, tx); e != nil {
+ return e
+ }
+ rows, e := tx.Query(ctx, "SELECT "+columns+" FROM lr_raw_collections WHERE collection_id::text > $1 COLLATE \"C\" ORDER BY collection_id::text COLLATE \"C\" LIMIT $2", after, limit+1)
+ if e != nil {
+ return e
+ }
+ p.Collections = []raw.Collection{}
+ for rows.Next() {
+ c, e := scan(rows)
+ if e != nil {
+ rows.Close()
+ return e
+ }
+ p.Collections = append(p.Collections, c)
+ }
+ e = rows.Err()
+ rows.Close()
+ if e != nil {
+ return e
+ }
+ for _, c := range p.Collections {
+ if e = validateTable(ctx, tx, c); e != nil {
+ return e
+ }
+ }
+ if len(p.Collections) > limit {
+ p.Collections = p.Collections[:limit]
+ v, e := raw.EncodeCollectionCursor(p.Collections[limit-1].CollectionID)
+ if e != nil {
+ return e
+ }
+ p.NextCursor = &v
+ }
+ return nil
+ })
+ if err != nil {
+ p = raw.CollectionPage{}
+ }
+ return
+}
diff --git a/rag/raw/postgres/lifecycle_test.go b/rag/raw/postgres/lifecycle_test.go
new file mode 100644
index 0000000..474877e
--- /dev/null
+++ b/rag/raw/postgres/lifecycle_test.go
@@ -0,0 +1,253 @@
+package postgres
+
+import (
+ "context"
+ "errors"
+ "os"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
+ "github.com/mudler/localrecall/rag/raw"
+)
+
+func fixture(t *testing.T) (*Service, context.Context) {
+ t.Helper()
+ if os.Getenv("LOCALRECALL_TEST_DATABASE_URL") == "" || os.Getenv("LOCALRECALL_TEST_DATABASE_OWNER") == "" {
+ t.Fatal("owned database fixture required; run sh test/raw/verify.sh lifecycle")
+ }
+ ctx := context.Background()
+ s, e := New(ctx, Config{os.Getenv("LOCALRECALL_TEST_DATABASE_URL")})
+ if e != nil {
+ t.Fatal(e)
+ }
+ t.Cleanup(s.Close)
+ return s, ctx
+}
+func request() raw.CreateCollectionRequest {
+ return raw.CreateCollectionRequest{Name: uuid.NewString(), Dimensions: 3, Metric: "cosine", EmbeddingSpaceID: "space"}
+}
+func checkCode(t *testing.T, e error, code string) {
+ t.Helper()
+ var r *raw.Error
+ if !errors.As(e, &r) || r.Code != code {
+ t.Fatalf("error %v; want %s", e, code)
+ }
+}
+func exec(t *testing.T, s *Service, q string, args ...any) {
+ t.Helper()
+ if _, e := s.pool.Exec(context.Background(), q, args...); e != nil {
+ t.Fatal(e)
+ }
+}
+
+func TestRawLifecycle(t *testing.T) {
+ s, ctx := fixture(t)
+ r := request()
+ r.Name = "quoted ' ; " + r.Name
+ c, created, e := s.CreateCollection(ctx, r)
+ if e != nil || !created {
+ t.Fatalf("create %v %v", created, e)
+ }
+ defer s.DeleteCollection(ctx, c.CollectionID)
+ if c.Kind != "raw" || c.Backend != "postgres" || c.SchemaVersion != 1 || raw.ValidateCollectionID(c.CollectionID) != nil {
+ t.Fatal(c)
+ }
+ again, created, e := s.CreateCollection(ctx, r)
+ if e != nil || created || again.CollectionID != c.CollectionID {
+ t.Fatalf("idempotency %v %v", created, e)
+ }
+ r.Dimensions++
+ _, _, e = s.CreateCollection(ctx, r)
+ checkCode(t, e, raw.CodeConflict)
+ other, e := New(ctx, Config{os.Getenv("LOCALRECALL_TEST_DATABASE_URL")})
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer other.Close()
+ got, e := other.GetCollection(ctx, c.CollectionID)
+ if e != nil || got.CollectionID != c.CollectionID {
+ t.Fatalf("reopen %v", e)
+ }
+ page, e := other.ListCollections(ctx, raw.ListCollectionsRequest{})
+ if e != nil || len(page.Collections) == 0 {
+ t.Fatalf("list %v", e)
+ }
+ table, _ := tableName(c.CollectionID)
+ exec(t, s, "INSERT INTO "+table+" (id,vector,text,metadata) VALUES ('a','[1,0,0]','hello','{}')")
+ if e = s.Reset(ctx, c.CollectionID); e != nil {
+ t.Fatal(e)
+ }
+ var n int
+ if e = s.pool.QueryRow(ctx, "SELECT count(*) FROM "+table).Scan(&n); e != nil || n != 0 {
+ t.Fatalf("reset %d %v", n, e)
+ }
+ if e = s.DeleteCollection(ctx, c.CollectionID); e != nil {
+ t.Fatal(e)
+ }
+ _, e = s.GetCollection(ctx, c.CollectionID)
+ checkCode(t, e, raw.CodeNotFound)
+ _, e = s.GetCollection(ctx, "'; drop table lr_raw_collections;")
+ checkCode(t, e, raw.CodeValidation)
+}
+func TestRawLifecycleConcurrentCreate(t *testing.T) {
+ s, ctx := fixture(t)
+ r := request()
+ var wg sync.WaitGroup
+ ids := make(chan string, 8)
+ for i := 0; i < 8; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ c, _, e := s.CreateCollection(ctx, r)
+ if e != nil {
+ t.Error(e)
+ return
+ }
+ ids <- c.CollectionID
+ }()
+ }
+ wg.Wait()
+ close(ids)
+ first := ""
+ for id := range ids {
+ if first == "" {
+ first = id
+ }
+ if first != id {
+ t.Fatal("different IDs")
+ }
+ }
+ if first != "" {
+ defer s.DeleteCollection(ctx, first)
+ }
+}
+func TestRawLifecycleLexicalRollback(t *testing.T) {
+ s, ctx := fixture(t)
+ r := request()
+ bad := "not_a_config'; --"
+ r.Lexical = &raw.LexicalRequest{TextConfig: &bad}
+ _, _, e := s.CreateCollection(ctx, r)
+ if e == nil {
+ t.Fatal("invalid catalog config accepted")
+ }
+ var n int
+ s.pool.QueryRow(ctx, "SELECT count(*) FROM lr_raw_collections WHERE name=$1", r.Name).Scan(&n)
+ if n != 0 {
+ t.Fatal("partial descriptor")
+ }
+ no := false
+ r.Lexical.Enabled = &no
+ c, _, e := s.CreateCollection(ctx, r)
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer s.DeleteCollection(ctx, c.CollectionID)
+ if c.Lexical.Enabled {
+ t.Fatal("lexical enabled")
+ }
+}
+func TestRawBootstrapRejectCorruption(t *testing.T) {
+ s, ctx := fixture(t)
+ r := request()
+ c, _, e := s.CreateCollection(ctx, r)
+ if e != nil {
+ t.Fatal(e)
+ }
+ table, _ := tableName(c.CollectionID)
+ exec(t, s, "DROP TABLE "+table)
+ _, e = s.GetCollection(ctx, c.CollectionID)
+ checkCode(t, e, raw.CodeUnavailable)
+ broken, e := New(ctx, Config{os.Getenv("LOCALRECALL_TEST_DATABASE_URL")})
+ if broken != nil {
+ broken.Close()
+ }
+ if e == nil {
+ t.Fatal("reconstructed missing table")
+ }
+ exec(t, s, "DELETE FROM lr_raw_collections WHERE collection_id=$1", c.CollectionID)
+ exec(t, s, "UPDATE lr_raw_schema SET version=2")
+ broken, e = New(ctx, Config{os.Getenv("LOCALRECALL_TEST_DATABASE_URL")})
+ if broken != nil {
+ broken.Close()
+ }
+ exec(t, s, "UPDATE lr_raw_schema SET version=1")
+ if e == nil {
+ t.Fatal("accepted future schema")
+ }
+}
+func TestRawLifecycleBoundedLocks(t *testing.T) {
+ s, ctx := fixture(t)
+ c, _, e := s.CreateCollection(ctx, request())
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer s.DeleteCollection(ctx, c.CollectionID)
+ tx, e := s.pool.Begin(ctx)
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer tx.Rollback(ctx)
+ table, _ := tableName(c.CollectionID)
+ if _, e = tx.Exec(ctx, "LOCK TABLE "+table+" IN ACCESS EXCLUSIVE MODE"); e != nil {
+ t.Fatal(e)
+ }
+ start := time.Now()
+ e = s.Reset(ctx, c.CollectionID)
+ if e == nil || time.Since(start) > 7*time.Second {
+ t.Fatalf("unbounded wait %s %v", time.Since(start), e)
+ }
+}
+func TestRawCapabilities(t *testing.T) {
+ s, ctx := fixture(t)
+ c, e := s.Capabilities(ctx)
+ if e != nil || !c.Ready || !c.ModeReadiness["lexical"].Ready {
+ t.Fatalf("capabilities %+v %v", c, e)
+ }
+}
+func TestRawPool(t *testing.T) {
+ s, ctx := fixture(t)
+ p := s.pool
+ for i := 0; i < 3; i++ {
+ c, _, e := s.CreateCollection(ctx, request())
+ if e != nil {
+ t.Fatal(e)
+ }
+ if s.pool != p {
+ t.Fatal("pool changed")
+ }
+ if e = s.DeleteCollection(ctx, c.CollectionID); e != nil {
+ t.Fatal(e)
+ }
+ }
+ s.Close()
+ s.Close()
+ _, e := s.GetCollection(ctx, uuid.NewString())
+ if e == nil {
+ t.Fatal("closed service accepted operation")
+ }
+}
+func TestRawBootstrapPrivileges(t *testing.T) {
+ s, ctx := fixture(t)
+ role := "raw_denied_" + uuid.New().String()[:8]
+ ident := pgx.Identifier{role}.Sanitize()
+ exec(t, s, "CREATE ROLE "+ident+" LOGIN PASSWORD 'fixture-only'")
+ defer exec(t, s, "DROP ROLE "+ident)
+ cfg, e := pgx.ParseConfig(os.Getenv("LOCALRECALL_TEST_DATABASE_URL"))
+ if e != nil {
+ t.Fatal(e)
+ }
+ cfg.User = role
+ cfg.Password = "fixture-only"
+ // SET ROLE on the connection string preserves the fixture's endpoint.
+ url := os.Getenv("LOCALRECALL_TEST_DATABASE_URL") + "&options=-c%20role%3D" + role
+ denied, e := New(ctx, Config{url})
+ if denied != nil {
+ denied.Close()
+ }
+ if e == nil {
+ t.Fatal("restricted role admitted")
+ }
+}
diff --git a/rag/raw/postgres/operations_test.go b/rag/raw/postgres/operations_test.go
new file mode 100644
index 0000000..1b1c317
--- /dev/null
+++ b/rag/raw/postgres/operations_test.go
@@ -0,0 +1,166 @@
+package postgres
+
+import (
+ "context"
+ "os"
+ osexec "os/exec"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/mudler/localrecall/rag/raw"
+)
+
+// Never control a server selected only by its address. Match both fixture name
+// and the random owner label before every administrative command.
+func ownedPG(t *testing.T, args ...string) string {
+ t.Helper()
+ owner, name := os.Getenv("LOCALRECALL_TEST_DATABASE_OWNER"), os.Getenv("LOCALRECALL_TEST_DATABASE_CONTAINER")
+ if owner == "" || name != "lr-raw-"+owner {
+ t.Fatal("owned container identity required")
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+ b, e := osexec.CommandContext(ctx, "docker", "inspect", "-f", `{{index .Config.Labels "localrecall.raw.owner"}}`, name).CombinedOutput()
+ if e != nil || strings.TrimSpace(string(b)) != owner {
+ t.Fatalf("owner check: %s %v", b, e)
+ }
+ b, e = osexec.CommandContext(ctx, "docker", append([]string{"exec", "-u", "postgres", name}, args...)...).CombinedOutput()
+ if e != nil {
+ t.Fatalf("owned postgres %v: %s %v", args, b, e)
+ }
+ return strings.TrimSpace(string(b))
+}
+
+func TestRawOwnedOutage(t *testing.T) {
+ s, ctx := fixture(t)
+ c, _, e := s.CreateCollection(ctx, request())
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer s.DeleteCollection(ctx, c.CollectionID)
+ put := raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{{ID: "retained", Vector: []float64{1, 0, 0}}}}
+ if _, e = s.Upsert(ctx, c.CollectionID, put); e != nil {
+ t.Fatal(e)
+ }
+ data := ownedPG(t, "printenv", "PGDATA")
+ ownedPG(t, "pg_ctl", "-D", data, "-w", "-t", "15", "stop", "-m", "fast")
+ restarted := false
+ defer func() {
+ if !restarted {
+ ownedPG(t, "pg_ctl", "-D", data, "-l", "/tmp/raw-restart.log", "-w", "-t", "15", "-o", "-c shared_preload_libraries=pg_textsearch", "start")
+ }
+ }()
+ start := time.Now()
+ limited, cancel := context.WithTimeout(ctx, 2*time.Second)
+ _, e = s.GetCollection(limited, c.CollectionID)
+ cancel()
+ checkCode(t, e, raw.CodeUnavailable)
+ limited, cancel = context.WithTimeout(ctx, 2*time.Second)
+ caps, e := s.Capabilities(limited)
+ cancel()
+ if e != nil || caps.Ready {
+ t.Fatalf("outage readiness %+v %v", caps, e)
+ }
+ if time.Since(start) > 5*time.Second {
+ t.Fatal("outage was not bounded")
+ }
+ ownedPG(t, "pg_ctl", "-D", data, "-l", "/tmp/raw-restart.log", "-w", "-t", "15", "-o", "-c shared_preload_libraries=pg_textsearch", "start")
+ restarted = true
+ caps, e = s.Capabilities(ctx)
+ if e != nil || !caps.Ready {
+ t.Fatalf("recovery %+v %v", caps, e)
+ }
+ got, e := s.GetRecords(ctx, c.CollectionID, raw.GetRecordsRequest{IDs: []string{"retained"}, IncludeVectors: true})
+ if e != nil || len(got.Records) != 1 || len(got.Records[0].Vector) != 3 || got.Records[0].Vector[0] != 1 || got.Records[0].Vector[1] != 0 || got.Records[0].Vector[2] != 0 {
+ t.Fatalf("retention %+v %v", got, e)
+ }
+ put.Records[0].ID = "after"
+ if _, e = s.Upsert(ctx, c.CollectionID, put); e != nil {
+ t.Fatal(e)
+ }
+ space, accuracy := "space", "exact"
+ if _, e = s.Search(ctx, c.CollectionID, raw.SearchRequest{Mode: "vector", Accuracy: &accuracy, EmbeddingSpaceID: &space, Vector: []float64{1, 0, 0}}); e != nil {
+ t.Fatal(e)
+ }
+ if e = s.Reset(ctx, c.CollectionID); e != nil {
+ t.Fatal(e)
+ }
+ t.Logf("owned pg_ctl outage/recovery/retained record/CRUD/search elapsed=%s", time.Since(start))
+}
+
+func TestRawActiveShutdown(t *testing.T) {
+ admin, ctx := fixture(t)
+ url := os.Getenv("LOCALRECALL_TEST_DATABASE_URL") + "&application_name=raw_active_shutdown"
+ s, e := New(ctx, Config{url})
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer s.Close()
+ caps, e := s.Capabilities(ctx)
+ if e != nil || !caps.Ready {
+ t.Fatalf("shutdown fixture not ready: %+v %v", caps, e)
+ }
+ c, _, e := s.CreateCollection(ctx, request())
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer admin.DeleteCollection(ctx, c.CollectionID)
+ table, _ := tableName(c.CollectionID)
+ lock, e := admin.pool.Begin(ctx)
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer lock.Rollback(ctx)
+ if _, e = lock.Exec(ctx, "LOCK TABLE "+table+" IN ACCESS EXCLUSIVE MODE"); e != nil {
+ t.Fatal(e)
+ }
+ result := make(chan error, 1)
+ go func() {
+ _, err := s.GetRecords(ctx, c.CollectionID, raw.GetRecordsRequest{IDs: []string{"held"}})
+ result <- err
+ }()
+ deadline := time.Now().Add(3 * time.Second)
+ for {
+ var n int
+ e = admin.pool.QueryRow(ctx, `SELECT count(*) FROM pg_stat_activity WHERE application_name='raw_active_shutdown' AND wait_event_type='Lock'`).Scan(&n)
+ if e != nil {
+ t.Fatal(e)
+ }
+ if n > 0 {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("query never blocked")
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ start := time.Now()
+ done := make(chan struct{})
+ go func() { s.Close(); close(done) }()
+ select {
+ case <-done:
+ case <-time.After(3 * time.Second):
+ t.Fatal("close did not cancel active DB work")
+ }
+ checkCode(t, <-result, raw.CodeTimeout)
+ deadline = time.Now().Add(3 * time.Second)
+ for {
+ var n int
+ e = admin.pool.QueryRow(ctx, `SELECT count(*) FROM pg_stat_activity WHERE application_name='raw_active_shutdown'`).Scan(&n)
+ if e != nil {
+ t.Fatal(e)
+ }
+ if n == 0 {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("leaked sessions=%d", n)
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ if s.pool.Stat().TotalConns() != 0 {
+ t.Fatal("pool not released")
+ }
+ t.Logf("active lock cancellation/drain=%s; sessions baseline=0 final=0 pool=0", time.Since(start))
+}
diff --git a/rag/raw/postgres/process_test.go b/rag/raw/postgres/process_test.go
new file mode 100644
index 0000000..20c6b01
--- /dev/null
+++ b/rag/raw/postgres/process_test.go
@@ -0,0 +1,48 @@
+package postgres
+
+import (
+ "context"
+ "os"
+ osexec "os/exec"
+ "testing"
+ "time"
+
+ "github.com/mudler/localrecall/rag/raw"
+)
+
+// Re-exec the test binary, not another Service in the creator process.
+func TestRawProcessDiscovery(t *testing.T) {
+ if id := os.Getenv("LOCALRECALL_RAW_CHILD_ID"); id != "" {
+ s, ctx := fixture(t)
+ c, err := s.GetCollection(ctx, id)
+ if err != nil || c.CollectionID != id {
+ t.Fatalf("discover: %+v %v", c, err)
+ }
+ page, err := s.GetRecords(ctx, id, raw.GetRecordsRequest{IDs: []string{"persisted"}})
+ if err != nil || len(page.Records) != 1 {
+ t.Fatalf("persisted records: %+v %v", page, err)
+ }
+ return
+ }
+ s, ctx := fixture(t)
+ c, _, err := s.CreateCollection(ctx, request())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer s.DeleteCollection(ctx, c.CollectionID)
+ _, err = s.Upsert(ctx, c.CollectionID, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{{ID: "persisted", Vector: []float64{1, 0, 0}}}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ for i := 0; i < 2; i++ {
+ childCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ cmd := osexec.CommandContext(childCtx, os.Args[0], "-test.run=^TestRawProcessDiscovery$", "-test.v")
+ cmd.Env = append(os.Environ(), "LOCALRECALL_RAW_CHILD_ID="+c.CollectionID)
+ output, err := cmd.CombinedOutput()
+ cancel()
+ if err != nil {
+ t.Fatalf("child %d: %v\n%s", i, err, output)
+ }
+ t.Logf("fresh process %d: %s", i, output)
+ }
+}
diff --git a/rag/raw/postgres/records.go b/rag/raw/postgres/records.go
new file mode 100644
index 0000000..42d2902
--- /dev/null
+++ b/rag/raw/postgres/records.go
@@ -0,0 +1,206 @@
+package postgres
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "github.com/jackc/pgx/v5"
+ "github.com/mudler/localrecall/rag/internal/pgutil"
+ "github.com/mudler/localrecall/rag/raw"
+ "sort"
+)
+
+// Upsert fully replaces records. Stable IDs prevent duplicates, not stale
+// retries: a delayed retry can overwrite a newer committed value.
+func (s *Service) Upsert(ctx context.Context, id string, r raw.UpsertRequest) (out raw.UpsertResponse, err error) {
+ err = s.transaction(ctx, pgx.ReadCommitted, func(ctx context.Context, tx pgx.Tx) error {
+ c, e := lookup(ctx, tx, id)
+ if e != nil {
+ return e
+ }
+ records, e := raw.ValidateUpsert(c, r)
+ if e != nil {
+ return e
+ }
+ out.IDs = make([]string, len(records))
+ for i, v := range records {
+ out.IDs[i] = v.ID
+ }
+ // Every writer takes row locks in C order, independent of request order.
+ sort.Slice(records, func(i, j int) bool { return records[i].ID < records[j].ID })
+ table, _ := tableName(id)
+ for _, v := range records {
+ metadata, e := json.Marshal(v.Metadata)
+ if e != nil {
+ return e
+ }
+ _, e = tx.Exec(ctx, "INSERT INTO "+table+" (id,vector,text,metadata) VALUES ($1,$2::vector,$3,$4::jsonb) ON CONFLICT (id) DO UPDATE SET vector=EXCLUDED.vector,text=EXCLUDED.text,metadata=EXCLUDED.metadata", v.ID, pgutil.Vector(v.Vector), v.Text, string(metadata))
+ if e != nil {
+ return e
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ out = raw.UpsertResponse{}
+ }
+ return
+}
+
+func recordColumns(vectors bool) string {
+ if vectors {
+ return "id,text,metadata,vector::text"
+ }
+ return "id,text,metadata"
+}
+func readRecords(rows pgx.Rows, vectors bool) ([]raw.CanonicalRecord, error) {
+ defer rows.Close()
+ out := []raw.CanonicalRecord{}
+ for rows.Next() {
+ var r raw.CanonicalRecord
+ var vector string
+ dst := []any{&r.ID, &r.Text, &r.Metadata}
+ if vectors {
+ dst = append(dst, &vector)
+ }
+ if e := rows.Scan(dst...); e != nil {
+ return nil, e
+ }
+ if vectors {
+ if e := json.Unmarshal([]byte(vector), &r.Vector); e != nil {
+ return nil, e
+ }
+ }
+ out = append(out, r)
+ }
+ return out, rows.Err()
+}
+func (s *Service) GetRecords(ctx context.Context, id string, r raw.GetRecordsRequest) (out raw.GetRecordsResponse, err error) {
+ if err = raw.ValidateIDs(r.IDs); err != nil {
+ return
+ }
+ err = s.transaction(ctx, pgx.RepeatableRead, func(ctx context.Context, tx pgx.Tx) error {
+ if _, e := lookup(ctx, tx, id); e != nil {
+ return e
+ }
+ table, _ := tableName(id)
+ rows, e := tx.Query(ctx, "SELECT "+recordColumns(r.IncludeVectors)+" FROM "+table+" WHERE id=ANY($1::text[])", r.IDs)
+ if e != nil {
+ return e
+ }
+ records, e := readRecords(rows, r.IncludeVectors)
+ if e != nil {
+ return e
+ }
+ byID := make(map[string]raw.CanonicalRecord, len(records))
+ for _, v := range records {
+ byID[v.ID] = v
+ }
+ out.Records = []raw.CanonicalRecord{}
+ out.MissingIDs = []string{}
+ for _, id := range r.IDs {
+ if v, ok := byID[id]; ok {
+ out.Records = append(out.Records, v)
+ } else {
+ out.MissingIDs = append(out.MissingIDs, id)
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ out = raw.GetRecordsResponse{}
+ }
+ return
+}
+func (s *Service) ListRecords(ctx context.Context, id string, r raw.ListRecordsRequest) (out raw.RecordPage, err error) {
+ limit, after, e := recordPageInput(id, r)
+ if e != nil {
+ return out, e
+ }
+ err = s.transaction(ctx, pgx.RepeatableRead, func(ctx context.Context, tx pgx.Tx) error {
+ if _, e := lookup(ctx, tx, id); e != nil {
+ return e
+ }
+ table, _ := tableName(id)
+ args := []any{after, limit + 1}
+ predicate := metadataPredicate(r.Filter, &args)
+ rows, e := tx.Query(ctx, "SELECT "+recordColumns(r.IncludeVectors)+" FROM "+table+` WHERE id > $1 COLLATE "C" AND `+predicate+` ORDER BY id COLLATE "C" LIMIT $2`, args...)
+ if e != nil {
+ return e
+ }
+ out.Records, e = readRecords(rows, r.IncludeVectors)
+ if e != nil {
+ return e
+ }
+ if len(out.Records) > limit {
+ out.Records = out.Records[:limit]
+ cursor, e := raw.EncodeRecordCursor(id, out.Records[limit-1].ID, r.Filter, r.IncludeVectors)
+ if e != nil {
+ return e
+ }
+ out.NextCursor = &cursor
+ }
+ return nil
+ })
+ if err != nil {
+ out = raw.RecordPage{}
+ }
+ return
+}
+func (s *Service) DeleteRecords(ctx context.Context, id string, r raw.DeleteRecordsRequest) (out raw.DeleteRecordsResponse, err error) {
+ if err = raw.ValidateDelete(r); err != nil {
+ return
+ }
+ err = s.transaction(ctx, pgx.ReadCommitted, func(ctx context.Context, tx pgx.Tx) error {
+ if _, e := lookup(ctx, tx, id); e != nil {
+ return e
+ }
+ table, _ := tableName(id)
+ args := []any{}
+ predicate := ""
+ limit := raw.MaxDeleteMatches + 1
+ if r.IDs != nil {
+ args = append(args, r.IDs)
+ predicate = "id=ANY($1::text[])"
+ limit = raw.MaxBatch
+ } else {
+ predicate = metadataPredicate(r.Filter, &args)
+ }
+ // Lock a bounded, deterministically ordered selection. Delete only these
+ // IDs: later matching inserts are allowed to survive this request.
+ rows, e := tx.Query(ctx, "SELECT id FROM "+table+" WHERE "+predicate+fmt.Sprintf(` ORDER BY id COLLATE "C" LIMIT %d FOR UPDATE`, limit), args...)
+ if e != nil {
+ return e
+ }
+ ids := []string{}
+ for rows.Next() {
+ var id string
+ if e = rows.Scan(&id); e != nil {
+ rows.Close()
+ return e
+ }
+ ids = append(ids, id)
+ }
+ e = rows.Err()
+ rows.Close()
+ if e != nil {
+ return e
+ }
+ if len(ids) > raw.MaxDeleteMatches {
+ return raw.NewError(raw.CodeDeleteBound)
+ }
+ if len(ids) == 0 {
+ return nil
+ }
+ tag, e := tx.Exec(ctx, "DELETE FROM "+table+" WHERE id=ANY($1::text[])", ids)
+ if e != nil {
+ return e
+ }
+ out.DeletedCount = int(tag.RowsAffected())
+ return nil
+ })
+ if err != nil {
+ out = raw.DeleteRecordsResponse{}
+ }
+ return
+}
diff --git a/rag/raw/postgres/records_concurrency_test.go b/rag/raw/postgres/records_concurrency_test.go
new file mode 100644
index 0000000..473f252
--- /dev/null
+++ b/rag/raw/postgres/records_concurrency_test.go
@@ -0,0 +1,160 @@
+package postgres
+
+import (
+ "context"
+ "github.com/jackc/pgx/v5"
+ "github.com/mudler/localrecall/rag/raw"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+)
+
+func TestRawConcurrencyOverlapping(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ var wg sync.WaitGroup
+ errs := make(chan error, 12)
+ for i := 0; i < 12; i++ {
+ wg.Add(1)
+ go func(i int) {
+ defer wg.Done()
+ a, b := rec("a"), rec("b")
+ text := string(rune('A' + i))
+ a.Text = &text
+ b.Text = &text
+ rs := []raw.Record{a, b}
+ if i%2 == 0 {
+ rs = []raw.Record{b, a}
+ }
+ _, e := s.Upsert(ctx, id, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: rs})
+ errs <- e
+ }(i)
+ }
+ wg.Wait()
+ close(errs)
+ for e := range errs {
+ if e != nil {
+ t.Fatal(e)
+ }
+ }
+ got, e := s.GetRecords(ctx, id, raw.GetRecordsRequest{IDs: []string{"a", "b"}})
+ if e != nil || *got.Records[0].Text != *got.Records[1].Text {
+ t.Fatal(got, e)
+ }
+}
+func TestRawConcurrencyDisjointAndBounds(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ put(t, s, ctx, id, rec("a"))
+ table, _ := tableName(id)
+ tx, e := s.pool.Begin(ctx)
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer tx.Rollback(ctx)
+ if _, e = tx.Exec(ctx, "SELECT id FROM "+table+" WHERE id='a' FOR UPDATE"); e != nil {
+ t.Fatal(e)
+ }
+ // A locked record must not impose a registry-wide writer lock or block reads.
+ fast, cancel := context.WithTimeout(ctx, 2*time.Second)
+ defer cancel()
+ put(t, s, fast, id, rec("disjoint"))
+ if _, e = s.ListRecords(fast, id, raw.ListRecordsRequest{}); e != nil {
+ t.Fatal(e)
+ }
+ short, stop := context.WithTimeout(ctx, 100*time.Millisecond)
+ defer stop()
+ _, e = s.Upsert(short, id, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{rec("a")}})
+ checkCode(t, e, raw.CodeTimeout)
+ start := time.Now()
+ _, e = s.Upsert(ctx, id, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{rec("a")}})
+ checkCode(t, e, raw.CodeTimeout)
+ if elapsed := time.Since(start); elapsed < 4*time.Second || elapsed > 8*time.Second {
+ t.Fatal(elapsed)
+ }
+ op, done, e := s.operation(ctx)
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer done()
+ deadline, ok := op.Deadline()
+ if !ok || time.Until(deadline) > 30*time.Second {
+ t.Fatal("request deadline absent")
+ }
+ canceled, c := context.WithCancel(ctx)
+ c()
+ _, e = s.GetRecords(canceled, id, raw.GetRecordsRequest{IDs: []string{"a"}})
+ checkCode(t, e, raw.CodeTimeout)
+}
+func TestRawConcurrencyLockedDeleteLaterInsert(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ writer, _ := fixture(t)
+ table, _ := tableName(id)
+ f := &raw.Filter{MetadataEq: map[string]string{"source": "x"}}
+ r := rec("a")
+ r.Metadata = f.MetadataEq
+ put(t, s, ctx, id, r)
+ reached := false
+ tracedPool(t, s, func(ctx context.Context, c *pgx.Conn, q string) context.Context {
+ if strings.HasPrefix(q, "DELETE FROM "+table+" WHERE") {
+ reached = true
+ r.ID = "later"
+ put(t, writer, ctx, id, r)
+ }
+ return ctx
+ })
+ got, e := s.DeleteRecords(ctx, id, raw.DeleteRecordsRequest{Filter: f})
+ if e != nil || got.DeletedCount != 1 || !reached {
+ t.Fatal(got, e, reached)
+ }
+ p, e := writer.ListRecords(ctx, id, raw.ListRecordsRequest{})
+ if e != nil || len(p.Records) != 1 || p.Records[0].ID != "later" {
+ t.Fatal(p, e)
+ }
+}
+func TestRawConcurrencyResetWriter(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ writer, _ := fixture(t)
+ put(t, s, ctx, id, rec("a"))
+ // Pause reset after its DELETE statement (at commit). A later insert can
+ // commit while reset still holds its row locks and is allowed to survive.
+ reached := false
+ tracedPool(t, s, func(ctx context.Context, c *pgx.Conn, q string) context.Context {
+ if q == "commit" && !reached {
+ reached = true
+ put(t, writer, ctx, id, rec("later"))
+ }
+ return ctx
+ })
+ if e := s.Reset(ctx, id); e != nil {
+ t.Fatal(e)
+ }
+ p, e := writer.ListRecords(ctx, id, raw.ListRecordsRequest{})
+ if e != nil || !reached || len(p.Records) != 1 || p.Records[0].ID != "later" {
+ t.Fatal(p, e, reached)
+ }
+}
+func TestRawConcurrencyReadSnapshotAndDDL(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ writer, _ := fixture(t)
+ put(t, s, ctx, id, rec("a"))
+ table, _ := tableName(id)
+ reached := false
+ tracedPool(t, s, func(ctx context.Context, c *pgx.Conn, q string) context.Context {
+ if strings.HasPrefix(q, "SELECT id,text,metadata FROM "+table) {
+ reached = true
+ put(t, writer, ctx, id, rec("b"))
+ short, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
+ defer cancel()
+ checkCode(t, writer.DeleteCollection(short, id), raw.CodeTimeout)
+ }
+ return ctx
+ })
+ p, e := s.ListRecords(ctx, id, raw.ListRecordsRequest{})
+ if e != nil || !reached || len(p.Records) != 1 {
+ t.Fatal(p, e, reached)
+ }
+ p, e = writer.ListRecords(ctx, id, raw.ListRecordsRequest{})
+ if e != nil || len(p.Records) != 2 {
+ t.Fatal(p, e)
+ }
+}
diff --git a/rag/raw/postgres/records_test.go b/rag/raw/postgres/records_test.go
new file mode 100644
index 0000000..0f09f19
--- /dev/null
+++ b/rag/raw/postgres/records_test.go
@@ -0,0 +1,210 @@
+package postgres
+
+import (
+ "context"
+ "fmt"
+ "github.com/mudler/localrecall/rag/raw"
+ "math"
+ "reflect"
+ "sort"
+ "strings"
+ "testing"
+)
+
+func recordFixture(t *testing.T) (*Service, context.Context, string) {
+ t.Helper()
+ s, ctx := fixture(t)
+ c, _, e := s.CreateCollection(ctx, request())
+ if e != nil {
+ t.Fatal(e)
+ }
+ t.Cleanup(func() { s.DeleteCollection(ctx, c.CollectionID) })
+ return s, ctx, c.CollectionID
+}
+func rec(id string) raw.Record { return raw.Record{ID: id, Vector: []float64{1, 2, 3}} }
+func put(t *testing.T, s *Service, ctx context.Context, id string, rs ...raw.Record) {
+ t.Helper()
+ got, e := s.Upsert(ctx, id, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: rs})
+ if e != nil {
+ t.Fatal(e)
+ }
+ for i, r := range rs {
+ if got.IDs[i] != r.ID {
+ t.Fatal(got)
+ }
+ }
+}
+func TestRawRecordsReplacement(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ text := "bicycle"
+ r := rec("b")
+ r.Text = &text
+ r.Metadata = map[string]string{"source": "file:///not/resolved"}
+ put(t, s, ctx, id, r, rec("a"))
+ got, e := s.GetRecords(ctx, id, raw.GetRecordsRequest{IDs: []string{"b", "missing", "a"}, IncludeVectors: true})
+ if e != nil || len(got.Records) != 2 || !reflect.DeepEqual(got.MissingIDs, []string{"missing"}) {
+ t.Fatalf("%+v %v", got, e)
+ }
+ v, _ := raw.NormalizeVector(r.Vector, 3)
+ if !reflect.DeepEqual(got.Records[0].Vector, v) {
+ t.Fatal(got)
+ }
+ put(t, s, ctx, id, rec("b"))
+ got, e = s.GetRecords(ctx, id, raw.GetRecordsRequest{IDs: []string{"b"}})
+ if e != nil || got.Records[0].Text != nil || len(got.Records[0].Metadata) != 0 || got.Records[0].Vector != nil {
+ t.Fatalf("%+v %v", got, e)
+ }
+ // Stable IDs avoid duplicates, but a delayed retry intentionally overwrites newer data.
+ put(t, s, ctx, id, r)
+ got, e = s.GetRecords(ctx, id, raw.GetRecordsRequest{IDs: []string{"b"}})
+ if e != nil || *got.Records[0].Text != text {
+ t.Fatal(got, e)
+ }
+ d, e := s.DeleteRecords(ctx, id, raw.DeleteRecordsRequest{IDs: []string{"missing", "a"}})
+ if e != nil || d.DeletedCount != 1 {
+ t.Fatal(d, e)
+ }
+}
+func TestRawRecordsValidationRollback(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ for _, v := range [][]float64{{0, 0, 0}, {1, 2}, {math.NaN(), 0, 1}, {math.Inf(1), 0, 1}, {math.MaxFloat64, 0, 1}, {1e-100, 0, 0}} {
+ bad := rec("z")
+ bad.Vector = v
+ _, e := s.Upsert(ctx, id, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{rec("a"), bad}})
+ checkCode(t, e, raw.CodeValidation)
+ }
+ _, e := s.Upsert(ctx, id, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{rec("a"), rec("a")}})
+ checkCode(t, e, raw.CodeValidation)
+ _, e = s.Upsert(ctx, id, raw.UpsertRequest{EmbeddingSpaceID: "wrong", Records: []raw.Record{rec("a")}})
+ checkCode(t, e, raw.CodeSpaceMismatch)
+ table, _ := tableName(id)
+ exec(t, s, "ALTER TABLE "+table+" ADD CHECK (id <> 'z')")
+ text := "rollbackuniqueterm"
+ r := rec("a")
+ r.Text = &text
+ _, e = s.Upsert(ctx, id, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{r, rec("z")}})
+ if e == nil {
+ t.Fatal("injected failure accepted")
+ }
+ got, e := s.GetRecords(ctx, id, raw.GetRecordsRequest{IDs: []string{"a", "z"}})
+ if e != nil || len(got.Records) != 0 {
+ t.Fatal(got, e)
+ }
+ // Force the BM25 access path after rollback; no aborted tuple may escape.
+ tx, e := s.pool.Begin(ctx)
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer tx.Rollback(ctx)
+ if _, e = tx.Exec(ctx, "SET LOCAL enable_seqscan=off"); e != nil {
+ t.Fatal(e)
+ }
+ rows, e := tx.Query(ctx, "SELECT id FROM "+table+" ORDER BY text <@> 'rollbackuniqueterm' LIMIT 10")
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer rows.Close()
+ if rows.Next() {
+ t.Fatal("aborted BM25 record visible")
+ }
+ if rows.Err() != nil {
+ t.Fatal(rows.Err())
+ }
+}
+func TestRawCursorPaging(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ ids := []string{"é", "ä¸", "😀", "z", "A", "\\\"", "\u2028"}
+ for _, v := range ids {
+ put(t, s, ctx, id, rec(v))
+ }
+ sort.Strings(ids)
+ n := 2
+ req := raw.ListRecordsRequest{Limit: &n}
+ var seen []string
+ for {
+ p, e := s.ListRecords(ctx, id, req)
+ if e != nil {
+ t.Fatal(e)
+ }
+ for _, r := range p.Records {
+ seen = append(seen, r.ID)
+ if r.Vector != nil {
+ t.Fatal(r)
+ }
+ }
+ if p.NextCursor == nil {
+ break
+ }
+ req.Cursor = p.NextCursor
+ }
+ if !reflect.DeepEqual(seen, ids) {
+ t.Fatal(seen, ids)
+ }
+ p, e := s.ListRecords(ctx, id, raw.ListRecordsRequest{Limit: &n})
+ if e != nil {
+ t.Fatal(e)
+ }
+ for _, r := range []raw.ListRecordsRequest{{Cursor: p.NextCursor, IncludeVectors: true}, {Cursor: p.NextCursor, Filter: &raw.Filter{MetadataEq: map[string]string{"x": "y"}}}, {Cursor: ptr("bad")}, {Cursor: ptr(strings.Repeat("x", 2049))}} {
+ _, e = s.ListRecords(ctx, id, r)
+ checkCode(t, e, raw.CodeCursor)
+ }
+ other, _, cid := recordFixture(t)
+ _, e = other.ListRecords(ctx, cid, raw.ListRecordsRequest{Cursor: p.NextCursor})
+ checkCode(t, e, raw.CodeCursor)
+ put(t, s, ctx, id, rec("0"), rec("zz"))
+ _, e = s.DeleteRecords(ctx, id, raw.DeleteRecordsRequest{IDs: []string{ids[0]}})
+ if e != nil {
+ t.Fatal(e)
+ }
+ next, e := s.ListRecords(ctx, id, raw.ListRecordsRequest{Cursor: p.NextCursor})
+ if e != nil || len(next.Records) == 0 {
+ t.Fatal(next, e)
+ }
+ for _, r := range next.Records {
+ if r.ID <= ids[1] {
+ t.Fatal("cursor invalidated", r)
+ }
+ }
+}
+func ptr(s string) *string { return &s }
+func TestRawDeleteBound(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ f := &raw.Filter{MetadataEq: map[string]string{"x' OR true --": "v'"}}
+ for i := 0; i < 1001; i += 100 {
+ rs := []raw.Record{}
+ for j := i; j < i+100 && j < 1001; j++ {
+ r := rec(fmt.Sprintf("%04d", j))
+ r.Metadata = f.MetadataEq
+ rs = append(rs, r)
+ }
+ put(t, s, ctx, id, rs...)
+ }
+ _, e := s.DeleteRecords(ctx, id, raw.DeleteRecordsRequest{Filter: f})
+ checkCode(t, e, raw.CodeDeleteBound)
+ table, _ := tableName(id)
+ var count int
+ if e = s.pool.QueryRow(ctx, "SELECT count(*) FROM "+table).Scan(&count); e != nil || count != 1001 {
+ t.Fatal(count, e)
+ }
+ _, e = s.DeleteRecords(ctx, id, raw.DeleteRecordsRequest{IDs: []string{"1000"}})
+ if e != nil {
+ t.Fatal(e)
+ }
+ d, e := s.DeleteRecords(ctx, id, raw.DeleteRecordsRequest{Filter: f})
+ if e != nil || d.DeletedCount != 1000 {
+ t.Fatal(d, e)
+ }
+ for _, r := range []raw.DeleteRecordsRequest{{}, {IDs: []string{}}, {Filter: &raw.Filter{}}, {IDs: []string{"x"}, Filter: f}} {
+ _, e = s.DeleteRecords(ctx, id, r)
+ checkCode(t, e, raw.CodeValidation)
+ }
+ f.MetadataEq = map[string]string{}
+ for i := 0; i < 9; i++ {
+ f.MetadataEq[fmt.Sprint(i)] = "x"
+ }
+ _, e = s.ListRecords(ctx, id, raw.ListRecordsRequest{Filter: f})
+ checkCode(t, e, raw.CodeValidation)
+ zero := 0
+ _, e = s.ListRecords(ctx, id, raw.ListRecordsRequest{Limit: &zero})
+ checkCode(t, e, raw.CodeValidation)
+}
diff --git a/rag/raw/postgres/registry.go b/rag/raw/postgres/registry.go
new file mode 100644
index 0000000..a3aee38
--- /dev/null
+++ b/rag/raw/postgres/registry.go
@@ -0,0 +1,132 @@
+package postgres
+
+import (
+ "context"
+ "fmt"
+ "github.com/jackc/pgx/v5"
+ "github.com/mudler/localrecall/rag/raw"
+ "strings"
+)
+
+const columns = `collection_id::text,name,dimensions,metric,embedding_space_id,lexical_enabled,text_config,schema_version,created_at`
+
+func tableName(id string) (string, error) {
+ if e := raw.ValidateCollectionID(id); e != nil {
+ return "", e
+ }
+ return pgx.Identifier{"lr_raw_" + strings.ReplaceAll(id, "-", "")}.Sanitize(), nil
+}
+func scan(row pgx.Row) (c raw.Collection, e error) {
+ c.Kind = "raw"
+ c.Backend = "postgres"
+ e = row.Scan(&c.CollectionID, &c.Name, &c.Dimensions, &c.Metric, &c.EmbeddingSpaceID, &c.Lexical.Enabled, &c.Lexical.TextConfig, &c.SchemaVersion, &c.CreatedAt)
+ return
+}
+func schemaVersion(ctx context.Context, tx pgx.Tx) error {
+ var n, v int
+ if e := tx.QueryRow(ctx, "SELECT count(*),coalesce(min(version),0) FROM lr_raw_schema").Scan(&n, &v); e != nil {
+ return e
+ }
+ if n != 1 || v != 1 {
+ return raw.NewError(raw.CodeUnavailable)
+ }
+ return nil
+}
+func (s *Service) bootstrap(ctx context.Context) error {
+ tx, e := s.pool.Begin(ctx)
+ if e != nil {
+ return e
+ }
+ defer tx.Rollback(ctx)
+ if _, e = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(763214991)`); e != nil {
+ return e
+ }
+ var vector bool
+ if e = tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname='vector')`).Scan(&vector); e != nil || !vector {
+ return raw.NewError(raw.CodeUnavailable)
+ }
+ var marker, registry bool
+ if e = tx.QueryRow(ctx, `SELECT to_regclass('lr_raw_schema') IS NOT NULL,to_regclass('lr_raw_collections') IS NOT NULL`).Scan(&marker, ®istry); e != nil {
+ return e
+ }
+ if marker != registry {
+ return raw.NewError(raw.CodeUnavailable)
+ }
+ if !marker {
+ _, e = tx.Exec(ctx, `CREATE TABLE lr_raw_schema (version integer NOT NULL); INSERT INTO lr_raw_schema VALUES (1);
+ CREATE TABLE lr_raw_collections (collection_id uuid PRIMARY KEY,name text COLLATE "C" NOT NULL UNIQUE,dimensions integer NOT NULL CHECK(dimensions BETWEEN 1 AND 2000),metric text NOT NULL CHECK(metric='cosine'),embedding_space_id text NOT NULL,lexical_enabled boolean NOT NULL,text_config text NOT NULL,schema_version integer NOT NULL CHECK(schema_version=1),created_at timestamptz NOT NULL DEFAULT now())`)
+ if e != nil {
+ return e
+ }
+ }
+ if e = schemaVersion(ctx, tx); e != nil {
+ return e
+ }
+ rows, e := tx.Query(ctx, "SELECT "+columns+" FROM lr_raw_collections")
+ if e != nil {
+ return e
+ }
+ var cs []raw.Collection
+ for rows.Next() {
+ c, e := scan(rows)
+ if e != nil {
+ rows.Close()
+ return e
+ }
+ cs = append(cs, c)
+ }
+ e = rows.Err()
+ rows.Close()
+ if e != nil {
+ return e
+ }
+ for _, c := range cs {
+ if e = validateTable(ctx, tx, c); e != nil {
+ return e
+ }
+ }
+ return tx.Commit(ctx)
+}
+func validateTable(ctx context.Context, tx pgx.Tx, c raw.Collection) error {
+ if c.SchemaVersion != 1 || raw.ValidateCreate(raw.CreateCollectionRequest{Name: c.Name, Dimensions: c.Dimensions, Metric: c.Metric, EmbeddingSpaceID: c.EmbeddingSpaceID}) != nil {
+ return raw.NewError(raw.CodeUnavailable)
+ }
+ table, e := tableName(c.CollectionID)
+ if e != nil {
+ return e
+ }
+ // Hold a relation lock until transaction end, preventing concurrent DROP/ALTER.
+ if _, e = tx.Exec(ctx, "SELECT id,vector,text,metadata FROM "+table+" LIMIT 0"); e != nil {
+ return e
+ }
+ var valid bool
+ e = tx.QueryRow(ctx, `SELECT count(*)=4 AND bool_and(CASE attname
+ WHEN 'id' THEN atttypid='text'::regtype AND attnotnull AND attcollation='"C"'::regcollation AND EXISTS(SELECT 1 FROM pg_index WHERE indrelid=attrelid AND indisprimary AND indkey::text=attnum::text)
+ WHEN 'vector' THEN atttypid='vector'::regtype AND atttypmod=$2 AND attnotnull
+ WHEN 'text' THEN atttypid='text'::regtype AND NOT attnotnull
+ WHEN 'metadata' THEN atttypid='jsonb'::regtype AND attnotnull ELSE false END)
+ FROM pg_attribute WHERE attrelid=$1::regclass AND attnum>0 AND NOT attisdropped`, table, c.Dimensions).Scan(&valid)
+ if e != nil {
+ return e
+ }
+ if !valid {
+ return fmt.Errorf("invalid raw table structure")
+ }
+ return nil
+}
+func lookup(ctx context.Context, tx pgx.Tx, id string) (raw.Collection, error) {
+ if e := raw.ValidateCollectionID(id); e != nil {
+ return raw.Collection{}, e
+ }
+ if e := schemaVersion(ctx, tx); e != nil {
+ return raw.Collection{}, e
+ }
+ c, e := scan(tx.QueryRow(ctx, "SELECT "+columns+" FROM lr_raw_collections WHERE collection_id=$1", id))
+ if e == pgx.ErrNoRows {
+ return c, raw.NewError(raw.CodeNotFound)
+ }
+ if e != nil {
+ return c, e
+ }
+ return c, validateTable(ctx, tx, c)
+}
diff --git a/rag/raw/postgres/review_test.go b/rag/raw/postgres/review_test.go
new file mode 100644
index 0000000..5092c67
--- /dev/null
+++ b/rag/raw/postgres/review_test.go
@@ -0,0 +1,137 @@
+package postgres
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgxpool"
+ "github.com/mudler/localrecall/rag/raw"
+)
+
+// TraceQueryStart runs before PostgreSQL receives the statement, giving these
+// tests a deterministic boundary without sleeps or production test hooks.
+type queryBoundary func(context.Context, *pgx.Conn, string) context.Context
+
+func (f queryBoundary) TraceQueryStart(ctx context.Context, c *pgx.Conn, d pgx.TraceQueryStartData) context.Context {
+ return f(ctx, c, d.SQL)
+}
+func (f queryBoundary) TraceQueryEnd(context.Context, *pgx.Conn, pgx.TraceQueryEndData) {}
+func tracedPool(t *testing.T, s *Service, f queryBoundary) {
+ t.Helper()
+ cfg := s.pool.Config()
+ cfg.ConnConfig.Tracer = f
+ p, err := pgxpool.NewWithConfig(context.Background(), cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ s.pool.Close()
+ s.pool = p
+}
+
+func TestRawLifecycleResetFreshDeleteSnapshot(t *testing.T) {
+ s, ctx := fixture(t)
+ writer, _ := fixture(t)
+ c, _, err := s.CreateCollection(ctx, request())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer writer.DeleteCollection(ctx, c.CollectionID)
+ table, _ := tableName(c.CollectionID)
+ reached := false
+ tracedPool(t, s, func(ctx context.Context, conn *pgx.Conn, q string) context.Context {
+ if q == "DELETE FROM "+table {
+ reached = true
+ // Lookup and its table validation have finished; DELETE has not started.
+ exec(t, writer, "INSERT INTO "+table+" (id,vector) VALUES ('between','[1,0,0]')")
+ }
+ return ctx
+ })
+ if err := s.Reset(ctx, c.CollectionID); err != nil {
+ t.Fatal(err)
+ }
+ var n int
+ if err := writer.pool.QueryRow(ctx, "SELECT count(*) FROM "+table).Scan(&n); err != nil {
+ t.Fatal(err)
+ }
+ if !reached || n != 0 {
+ t.Fatalf("DELETE boundary reached=%v; surviving committed records=%d", reached, n)
+ }
+}
+
+func TestRawLifecycleQuotedTextConfig(t *testing.T) {
+ s, ctx := fixture(t)
+ exec(t, s, `CREATE SCHEMA "Mixed.Schema space"`)
+ defer exec(t, s, `DROP SCHEMA "Mixed.Schema space" CASCADE`)
+ exec(t, s, `CREATE TEXT SEARCH CONFIGURATION "Mixed.Schema space"."Config.Name space" (COPY=pg_catalog.english)`)
+ for _, name := range []string{`"Mixed.Schema space"."Config.Name space"`, "english"} {
+ t.Run(name, func(t *testing.T) {
+ r := request()
+ r.Lexical = &raw.LexicalRequest{TextConfig: &name}
+ c, created, err := s.CreateCollection(ctx, r)
+ if err != nil || !created {
+ t.Fatalf("create: %v %v", created, err)
+ }
+ defer s.DeleteCollection(ctx, c.CollectionID)
+ want := name
+ if name == "english" {
+ want = "pg_catalog.english"
+ }
+ if c.Lexical.TextConfig != want {
+ t.Fatalf("config=%q want=%q", c.Lexical.TextConfig, want)
+ }
+ reopened, _ := fixture(t)
+ got, err := reopened.GetCollection(ctx, c.CollectionID)
+ if err != nil || got.Lexical != c.Lexical {
+ t.Fatalf("reopen: %+v %v", got, err)
+ }
+ again, created, err := reopened.CreateCollection(ctx, r)
+ if err != nil || created || again.CollectionID != c.CollectionID {
+ t.Fatalf("idempotent reopen: %+v %v %v", again, created, err)
+ }
+ })
+ }
+}
+
+func TestRawLifecycleTextConfigErrors(t *testing.T) {
+ for _, tc := range []struct{ name, code string }{
+ {"unknown", raw.CodeValidation}, {"malformed", raw.CodeValidation},
+ {"cancel", raw.CodeTimeout}, {"dependency", raw.CodeUnavailable},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ s, ctx := fixture(t)
+ reached := false
+ tracedPool(t, s, func(ctx context.Context, c *pgx.Conn, q string) context.Context {
+ if strings.Contains(q, "FROM pg_ts_config") {
+ reached = true
+ if tc.name == "cancel" {
+ canceled, cancel := context.WithCancel(ctx)
+ cancel()
+ return canceled
+ }
+ if tc.name == "dependency" {
+ if err := c.Close(ctx); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ return ctx
+ })
+ name := "english"
+ if tc.name == "unknown" {
+ name = "no_such_raw_config"
+ }
+ if tc.name == "malformed" {
+ name = `"unterminated`
+ }
+ r := request()
+ r.Lexical = &raw.LexicalRequest{TextConfig: &name}
+ _, _, err := s.CreateCollection(ctx, r)
+ if !reached {
+ t.Fatal("configuration lookup not reached")
+ }
+ checkCode(t, err, tc.code)
+ })
+ }
+}
diff --git a/rag/raw/postgres/search.go b/rag/raw/postgres/search.go
new file mode 100644
index 0000000..2707f7e
--- /dev/null
+++ b/rag/raw/postgres/search.go
@@ -0,0 +1,160 @@
+package postgres
+
+import (
+ "context"
+ "sort"
+
+ "github.com/jackc/pgx/v5"
+ "github.com/mudler/localrecall/rag/internal/pgutil"
+ "github.com/mudler/localrecall/rag/raw"
+)
+
+var _ raw.Store = (*Service)(nil)
+
+func candidateN(k int) int { return min(1000, max(100, 10*k)) }
+
+// vectorQuery fences exact ranking behind materialization: no ANN access path
+// can implement an ORDER BY on this CTE. Approximate keeps the bare operator.
+// SQL predicates precede LIMIT, but ANN traversal can still underfill or lose
+// filtered recall. Never expand candidates without bounds or retry as exact.
+func vectorQuery(table, predicate string, exact bool) string {
+ if exact {
+ return `WITH eligible AS MATERIALIZED (SELECT id,text,metadata,vector FROM ` + table + ` WHERE ` + predicate + `) SELECT id,text,metadata,1-(vector <=> $1::vector) FROM eligible ORDER BY vector <=> $1::vector,id COLLATE "C" LIMIT $2`
+ }
+ return `SELECT id,text,metadata,1-(vector <=> $1::vector) FROM ` + table + ` WHERE ` + predicate + ` ORDER BY vector <=> $1::vector,id COLLATE "C" LIMIT $2`
+}
+func lexicalQuery(table, index, predicate string) string {
+ distance := pgutil.BM25Distance("text", "$1", index)
+ return `SELECT id,text,metadata,-(` + distance + `) FROM ` + table + ` WHERE ` + predicate + ` AND (` + distance + `) < 0 ORDER BY ` + distance + `,id COLLATE "C" LIMIT $2`
+}
+func searchRows(ctx context.Context, tx pgx.Tx, q string, args []any) ([]raw.SearchResult, error) {
+ rows, e := tx.Query(ctx, q, args...)
+ if e != nil {
+ return nil, e
+ }
+ defer rows.Close()
+ out := []raw.SearchResult{}
+ for rows.Next() {
+ var r raw.SearchResult
+ if e = rows.Scan(&r.ID, &r.Text, &r.Metadata, &r.Score); e != nil {
+ return nil, e
+ }
+ out = append(out, r)
+ }
+ return out, rows.Err()
+}
+
+// Search never accepts an embedder: opaque caller vectors identify their space.
+// Both arms and their payloads use one repeatable-read snapshot. Errors, including
+// cancellation/commit failures, discard all results; BM25 never falls back.
+func (s *Service) Search(ctx context.Context, id string, r raw.SearchRequest) (out raw.SearchResponse, err error) {
+ err = s.transaction(ctx, pgx.RepeatableRead, func(ctx context.Context, tx pgx.Tx) error {
+ c, e := lookup(ctx, tx, id)
+ if e != nil {
+ return e
+ }
+ r, e = raw.ValidateSearch(c, r)
+ if e != nil {
+ return e
+ }
+ table, _ := tableName(id)
+ k := *r.TopK
+ n := k
+ if r.Mode == "hybrid" {
+ n = candidateN(k)
+ }
+ out = raw.SearchResponse{Mode: r.Mode, Accuracy: "not_applicable", ScoreKind: "bm25", Results: []raw.SearchResult{}}
+ var vs, ls []raw.SearchResult
+ if r.Mode != "lexical" {
+ out.Accuracy = *r.Accuracy
+ out.ScoreKind = "cosine_similarity"
+ if *r.Accuracy == "approximate" {
+ strategy, e := pgutil.IndexStrategy(ctx, tx, table, "lr_ann_"+id, "vector")
+ if e != nil {
+ return e
+ }
+ if strategy != "hnsw" && strategy != "diskann" {
+ return raw.NewError(raw.CodeUnavailable)
+ }
+ }
+ v := make([]float32, len(r.Vector))
+ for i, x := range r.Vector {
+ v[i] = float32(x)
+ }
+ args := []any{pgutil.Vector(v), n}
+ predicate := metadataPredicate(r.Filter, &args)
+ vs, e = searchRows(ctx, tx, vectorQuery(table, predicate, *r.Accuracy == "exact"), args)
+ if e != nil {
+ return e
+ }
+ for i := range vs {
+ rank, sim := i+1, vs[i].Score
+ vs[i].VectorRank = &rank
+ vs[i].VectorSimilarity = &sim
+ }
+ }
+ if r.Mode != "vector" {
+ strategy, e := pgutil.IndexStrategy(ctx, tx, table, "lr_bm25_"+id, "text")
+ if e != nil {
+ return e
+ }
+ if strategy != "bm25" {
+ return raw.NewError(raw.CodeUnavailable)
+ }
+ args := []any{*r.Text, n}
+ predicate := metadataPredicate(r.Filter, &args)
+ // pg_textsearch resolves the literal index name, not a regclass literal.
+ ls, e = searchRows(ctx, tx, lexicalQuery(table, "lr_bm25_"+id, predicate), args)
+ if e != nil {
+ return e
+ }
+ for i := range ls {
+ rank := i + 1
+ ls[i].LexicalRank = &rank
+ }
+ }
+ switch r.Mode {
+ case "vector":
+ out.Results = vs
+ case "lexical":
+ out.Results = ls
+ case "hybrid":
+ out.ScoreKind = "rrf"
+ out.Fusion = &raw.FusionParameters{K: 60, VectorWeight: 0.5, LexicalWeight: 0.5, CandidatesPerArm: n}
+ merged := map[string]raw.SearchResult{}
+ for _, v := range vs {
+ v.Score = 0.5 / float64(60+*v.VectorRank)
+ merged[v.ID] = v
+ }
+ for _, l := range ls {
+ v, ok := merged[l.ID]
+ if !ok {
+ v = l
+ v.Score = 0
+ }
+ v.LexicalRank = l.LexicalRank
+ v.Score += 0.5 / float64(60+*l.LexicalRank)
+ merged[l.ID] = v
+ }
+ for _, v := range merged {
+ out.Results = append(out.Results, v)
+ }
+ sort.Slice(out.Results, func(i, j int) bool {
+ a, b := out.Results[i], out.Results[j]
+ if a.Score == b.Score {
+ return a.ID < b.ID
+ }
+ return a.Score > b.Score
+ })
+ if len(out.Results) > k {
+ out.Results = out.Results[:k]
+ }
+ }
+ out.Count = len(out.Results)
+ return ctx.Err()
+ })
+ if err != nil {
+ out = raw.SearchResponse{}
+ }
+ return
+}
diff --git a/rag/raw/postgres/search_snapshot_test.go b/rag/raw/postgres/search_snapshot_test.go
new file mode 100644
index 0000000..8839467
--- /dev/null
+++ b/rag/raw/postgres/search_snapshot_test.go
@@ -0,0 +1,87 @@
+package postgres
+
+import (
+ "context"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgxpool"
+ "github.com/mudler/localrecall/rag/raw"
+)
+
+// Query tracing supplies a deterministic arm boundary without production hooks.
+type armBarrier struct {
+ entered chan struct{}
+ release chan struct{}
+}
+
+func (b *armBarrier) TraceQueryStart(ctx context.Context, _ *pgx.Conn, d pgx.TraceQueryStartData) context.Context {
+ if strings.HasPrefix(d.SQL, "SELECT id,text,metadata,-(") {
+ close(b.entered)
+ select {
+ case <-b.release:
+ case <-ctx.Done():
+ }
+ }
+ return ctx
+}
+func (*armBarrier) TraceQueryEnd(context.Context, *pgx.Conn, pgx.TraceQueryEndData) {}
+func TestRawSearchSnapshotAndCancellation(t *testing.T) {
+ for _, cancelArm := range []bool{false, true} {
+ t.Run(map[bool]string{false: "snapshot", true: "cancel_after_vector"}[cancelArm], func(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ text := "bicycle"
+ put(t, s, ctx, id, raw.Record{ID: "a", Vector: []float64{1, 0, 0}, Text: &text})
+ writer, e := New(ctx, Config{s.pool.Config().ConnString()})
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer writer.Close()
+ barrier := &armBarrier{make(chan struct{}), make(chan struct{})}
+ cfg := s.pool.Config()
+ cfg.ConnConfig.Tracer = barrier
+ s.pool.Close()
+ s.pool, e = pgxpool.NewWithConfig(ctx, cfg)
+ if e != nil {
+ t.Fatal(e)
+ }
+ requestCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+ defer cancel()
+ type answer struct {
+ out raw.SearchResponse
+ err error
+ }
+ done := make(chan answer, 1)
+ go func() { out, e := s.Search(requestCtx, id, searchRequest("hybrid")); done <- answer{out, e} }()
+ select {
+ case <-barrier.entered:
+ case <-requestCtx.Done():
+ t.Fatal("lexical arm not reached")
+ }
+ if cancelArm {
+ cancel()
+ } else {
+ other := "ocean"
+ put(t, writer, ctx, id, raw.Record{ID: "a", Vector: []float64{0, 1, 0}, Text: &other})
+ }
+ close(barrier.release)
+ got := <-done
+ if cancelArm {
+ checkCode(t, got.err, raw.CodeTimeout)
+ if got.out.Results != nil || got.out.Count != 0 {
+ t.Fatal("partial response", got)
+ }
+ return
+ }
+ if got.err != nil || got.out.Count != 1 {
+ t.Fatal(got)
+ }
+ hit := got.out.Results[0]
+ if hit.Text == nil || *hit.Text != "bicycle" || hit.LexicalRank == nil || hit.VectorRank == nil || hit.Score != 1.0/61 {
+ t.Fatalf("mixed snapshot: %+v", hit)
+ }
+ })
+ }
+}
diff --git a/rag/raw/postgres/search_test.go b/rag/raw/postgres/search_test.go
new file mode 100644
index 0000000..e211fd1
--- /dev/null
+++ b/rag/raw/postgres/search_test.go
@@ -0,0 +1,273 @@
+package postgres
+
+import (
+ "context"
+ "fmt"
+ "github.com/mudler/localrecall/rag/raw"
+ "math"
+ "testing"
+)
+
+func searchRequest(mode string) raw.SearchRequest {
+ space, text, exact := "space", "bicycle", "exact"
+ r := raw.SearchRequest{Mode: mode}
+ if mode != "lexical" {
+ r.Vector = []float64{1, 0, 0}
+ r.EmbeddingSpaceID = &space
+ r.Accuracy = &exact
+ }
+ if mode != "vector" {
+ r.Text = &text
+ }
+ return r
+}
+func TestRawSearchModes(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ text := "bicycle bicycle"
+ other := "ocean"
+ put(t, s, ctx, id, raw.Record{ID: "b", Vector: []float64{1, 0, 0}, Text: &text, Metadata: map[string]string{"keep": "yes"}}, raw.Record{ID: "a", Vector: []float64{1, 0, 0}, Text: &text, Metadata: map[string]string{"keep": "yes"}}, raw.Record{ID: "z", Vector: []float64{0, 1, 0}, Text: &other})
+ for _, mode := range []string{"vector", "lexical", "hybrid"} {
+ r := searchRequest(mode)
+ r.Filter = &raw.Filter{MetadataEq: map[string]string{"keep": "yes"}}
+ got, e := s.Search(ctx, id, r)
+ if e != nil || got.Count != 2 || got.Results[0].ID != "a" {
+ t.Fatalf("%s: %+v %v", mode, got, e)
+ }
+ if got.Mode != mode || got.Count != len(got.Results) {
+ t.Fatal(got)
+ }
+ wantAccuracy, wantKind := "exact", "cosine_similarity"
+ if mode == "lexical" {
+ wantAccuracy, wantKind = "not_applicable", "bm25"
+ }
+ if mode == "hybrid" {
+ wantKind = "rrf"
+ }
+ if got.Accuracy != wantAccuracy || got.ScoreKind != wantKind {
+ t.Fatal(got)
+ }
+ if mode == "hybrid" {
+ if got.Fusion == nil || got.Fusion.K != 60 || got.Fusion.VectorWeight != 0.5 || got.Fusion.LexicalWeight != 0.5 {
+ t.Fatal(got)
+ }
+ if got.Results[0].VectorRank == nil || *got.Results[0].VectorRank != 1 || got.Results[0].LexicalRank == nil || *got.Results[0].LexicalRank != 1 {
+ t.Fatal(got)
+ }
+ if got.Fusion.CandidatesPerArm != 100 || math.Abs(got.Results[0].Score-1.0/61) > 1e-12 {
+ t.Fatal(got)
+ }
+ }
+ if mode == "lexical" && got.Results[0].Score <= 0 {
+ t.Fatal("BM25 must be higher-is-better", got)
+ }
+ }
+ r := searchRequest("lexical")
+ empty := "nomatchingterm"
+ r.Text = &empty
+ got, e := s.Search(ctx, id, r)
+ if e != nil || got.Count != 0 {
+ t.Fatal(got, e)
+ }
+ r = searchRequest("vector")
+ wrong := "wrong"
+ r.EmbeddingSpaceID = &wrong
+ _, e = s.Search(ctx, id, r)
+ checkCode(t, e, raw.CodeSpaceMismatch)
+ r = searchRequest("vector")
+ r.Vector = []float64{1, 0}
+ _, e = s.Search(ctx, id, r)
+ checkCode(t, e, raw.CodeValidation)
+ canceled, cancel := context.WithCancel(ctx)
+ cancel()
+ got, e = s.Search(canceled, id, searchRequest("hybrid"))
+ checkCode(t, e, raw.CodeTimeout)
+ if got.Results != nil {
+ t.Fatal("partial response")
+ }
+}
+func TestRawSearchMissingIndexes(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ put(t, s, ctx, id, rec("a"))
+ exec(t, s, `DROP INDEX "lr_ann_`+id+`"`)
+ r := searchRequest("vector")
+ r.Accuracy = nil
+ _, e := s.Search(ctx, id, r)
+ checkCode(t, e, raw.CodeUnavailable)
+ if _, e = s.Search(ctx, id, searchRequest("vector")); e != nil {
+ t.Fatal(e)
+ }
+ exec(t, s, `DROP INDEX "lr_bm25_`+id+`"`)
+ _, e = s.Search(ctx, id, searchRequest("lexical"))
+ checkCode(t, e, raw.CodeUnavailable)
+}
+
+// An ANN access method alone is insufficient: L2 cannot order a cosine query.
+func TestRawSearchANNOperatorClass(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ text := "bicycle"
+ put(t, s, ctx, id, raw.Record{ID: "a", Vector: []float64{1, 0, 0}, Text: &text})
+ table, _ := tableName(id)
+ for _, opclass := range []string{"vector_l2_ops", "vector_cosine_ops"} {
+ exec(t, s, `DROP INDEX "lr_ann_`+id+`"`)
+ exec(t, s, `CREATE INDEX "lr_ann_`+id+`" ON `+table+` USING hnsw (vector `+opclass+`)`)
+ for _, mode := range []string{"vector", "hybrid"} {
+ for _, accuracy := range []string{"approximate", "exact"} {
+ t.Run(opclass+"/"+mode+"/"+accuracy, func(t *testing.T) {
+ r := searchRequest(mode)
+ r.Accuracy = &accuracy
+ got, e := s.Search(ctx, id, r)
+ if opclass == "vector_l2_ops" && accuracy == "approximate" {
+ checkCode(t, e, raw.CodeUnavailable) // HTTP 503, not silent exact fallback.
+ if got.Results != nil {
+ t.Fatal("partial response", got)
+ }
+ return
+ }
+ if e != nil || got.Count != 1 || got.Results[0].ID != "a" || got.Accuracy != accuracy {
+ t.Fatalf("response=%+v error=%v", got, e)
+ }
+ })
+ }
+ }
+ }
+}
+
+// Regression: pg_textsearch takes an index *name*, not SQL-quoted regclass
+// syntax. UUID hyphens make this distinction observable on every collection.
+func TestRawSearchBM25LiteralIndex(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ text := "bicycle"
+ put(t, s, ctx, id, raw.Record{ID: "a", Vector: []float64{1, 0, 0}, Text: &text})
+ table, _ := tableName(id)
+ var distance float64
+ if e := s.pool.QueryRow(ctx, `SELECT text <@> to_bm25query($1,$2) FROM `+table, text, "lr_bm25_"+id).Scan(&distance); e != nil {
+ t.Fatal(e)
+ }
+ got, e := s.Search(ctx, id, searchRequest("lexical"))
+ if e != nil || got.Count != 1 || got.Results[0].Score != -distance || distance >= 0 {
+ t.Fatalf("literal index distance=%g response=%+v error=%v", distance, got, e)
+ }
+}
+func TestRawSearchCandidateBounds(t *testing.T) {
+ for _, c := range []struct{ k, n int }{{1, 100}, {10, 100}, {11, 110}, {100, 1000}} {
+ if candidateN(c.k) != c.n {
+ t.Fatal(c)
+ }
+ }
+ s, ctx, id := recordFixture(t)
+ // Globally higher-ranked excluded rows exceed either candidate limit.
+ for batch := 0; batch < 3; batch++ {
+ rs := []raw.Record{}
+ for i := 0; i < 100; i++ {
+ text := "bicycle"
+ rs = append(rs, raw.Record{ID: fmt.Sprintf("a%03d", batch*100+i), Vector: []float64{1, 0, 0}, Text: &text})
+ }
+ put(t, s, ctx, id, rs...)
+ }
+ text := "bicycle"
+ put(t, s, ctx, id, raw.Record{ID: "z", Vector: []float64{0, 1, 0}, Text: &text, Metadata: map[string]string{"keep": "yes"}})
+ for _, mode := range []string{"vector", "lexical", "hybrid"} {
+ r := searchRequest(mode)
+ k := 1
+ r.TopK = &k
+ r.Filter = &raw.Filter{MetadataEq: map[string]string{"keep": "yes"}}
+ got, e := s.Search(ctx, id, r)
+ if e != nil || got.Count != 1 || got.Results[0].ID != "z" {
+ t.Fatal(mode, got, e)
+ }
+ if mode == "hybrid" && (got.Results[0].VectorRank == nil || got.Results[0].LexicalRank == nil || got.Results[0].Score != 1.0/61) {
+ t.Fatal(got)
+ }
+ }
+ // Without a filter, z lies outside BOTH bounded arms, even exact hybrid.
+ r := searchRequest("hybrid")
+ k := 10
+ r.TopK = &k
+ got, e := s.Search(ctx, id, r)
+ if e != nil {
+ t.Fatal(e)
+ }
+ for _, h := range got.Results {
+ if h.ID == "z" {
+ t.Fatal("unexpected boundary result")
+ }
+ }
+}
+func TestRawSearchDisabledLexical(t *testing.T) {
+ s, ctx := fixture(t)
+ r := request()
+ disabled := false
+ r.Lexical = &raw.LexicalRequest{Enabled: &disabled}
+ c, _, e := s.CreateCollection(ctx, r)
+ if e != nil {
+ t.Fatal(e)
+ }
+ defer s.DeleteCollection(ctx, c.CollectionID)
+ for _, mode := range []string{"lexical", "hybrid"} {
+ got, e := s.Search(ctx, c.CollectionID, searchRequest(mode))
+ // Immutable configuration does not implement this collection capability (501).
+ // Unlike a configured capability with a missing runtime index (503),
+ // retrying cannot restore it. Preserve the Task1 validator contract.
+ checkCode(t, e, raw.CodeUnsupported)
+ if got.Results != nil {
+ t.Fatal(got)
+ }
+ }
+ if _, e = s.Search(ctx, c.CollectionID, searchRequest("vector")); e != nil {
+ t.Fatal(e)
+ }
+}
+
+// A lexical-only hit beyond the bounded vector arm must not acquire a global
+// vector rank. Approximate candidates may differ, but remain bounded and tied
+// by ID within each returned arm; no exact membership is promised for ANN.
+func TestRawSearchCandidateUnion(t *testing.T) {
+ s, ctx, id := recordFixture(t)
+ for batch := 0; batch < 2; batch++ {
+ records := []raw.Record{}
+ for i := batch * 100; i < min(110, (batch+1)*100); i++ {
+ text := "ocean"
+ if i >= 99 {
+ text = "bicycle"
+ }
+ records = append(records, raw.Record{ID: fmt.Sprintf("a%03d", i), Vector: []float64{1, 0, 0}, Text: &text})
+ }
+ put(t, s, ctx, id, records...)
+ }
+ for _, accuracy := range []string{"exact", "approximate"} {
+ r := searchRequest("hybrid")
+ r.Accuracy = &accuracy
+ got, e := s.Search(ctx, id, r)
+ if e != nil || got.Count != 10 || got.Fusion.CandidatesPerArm != 100 {
+ t.Fatal(got, e)
+ }
+ foundBoundary := false
+ for i, h := range got.Results {
+ score := 0.0
+ if h.VectorRank != nil {
+ if *h.VectorRank < 1 || *h.VectorRank > 100 {
+ t.Fatal(h)
+ }
+ score += 0.5 / float64(60+*h.VectorRank)
+ }
+ if h.LexicalRank != nil {
+ score += 0.5 / float64(60+*h.LexicalRank)
+ }
+ if math.Abs(score-h.Score) > 1e-12 {
+ t.Fatal(h)
+ }
+ if i > 0 && (got.Results[i-1].Score < h.Score || (got.Results[i-1].Score == h.Score && got.Results[i-1].ID > h.ID)) {
+ t.Fatal("unstable fusion order", got)
+ }
+ if accuracy == "exact" && h.ID == "a100" {
+ foundBoundary = true
+ if h.VectorRank != nil || h.LexicalRank == nil || *h.LexicalRank != 2 {
+ t.Fatal("candidate boundary ignored", h)
+ }
+ }
+ }
+ if accuracy == "exact" && (!foundBoundary || got.Results[0].ID != "a099") {
+ t.Fatal("expected bounded union", got)
+ }
+ }
+}
diff --git a/rag/raw/postgres/service.go b/rag/raw/postgres/service.go
new file mode 100644
index 0000000..8710db6
--- /dev/null
+++ b/rag/raw/postgres/service.go
@@ -0,0 +1,127 @@
+// Package postgres implements isolated, embedder-free raw PostgreSQL storage.
+package postgres
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgconn"
+ "github.com/jackc/pgx/v5/pgxpool"
+ "github.com/mudler/localrecall/rag/internal/pgutil"
+ "github.com/mudler/localrecall/rag/raw"
+ "os"
+ "sync"
+ "time"
+)
+
+type Config struct{ DatabaseURL string }
+type Service struct {
+ pool *pgxpool.Pool
+ mu sync.Mutex
+ closed bool
+ wg sync.WaitGroup
+ stop context.Context
+ cancel context.CancelFunc
+ once sync.Once
+}
+
+func New(ctx context.Context, cfg Config) (*Service, error) {
+ ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ defer cancel()
+ pc, e := pgxpool.ParseConfig(cfg.DatabaseURL)
+ if e != nil {
+ return nil, dbError(e)
+ }
+ pgutil.ApplyPoolLimits(pc, os.Getenv)
+ pgutil.ApplyConnTimeouts(pc, os.Getenv)
+ pc.ConnConfig.RuntimeParams["lock_timeout"] = "5s"
+ pc.ConnConfig.RuntimeParams["statement_timeout"] = "30s"
+ p, e := pgxpool.NewWithConfig(ctx, pc)
+ if e != nil {
+ return nil, dbError(e)
+ }
+ stop, c := context.WithCancel(context.Background())
+ s := &Service{pool: p, stop: stop, cancel: c}
+ if e = s.bootstrap(ctx); e != nil {
+ s.Close()
+ return nil, dbError(e)
+ }
+ return s, nil
+}
+func (s *Service) operation(ctx context.Context) (context.Context, func(), error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if s.closed {
+ return nil, nil, raw.NewError(raw.CodeUnavailable)
+ }
+ ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ stop := context.AfterFunc(s.stop, cancel)
+ s.wg.Add(1)
+ return ctx, func() { stop(); cancel(); s.wg.Done() }, nil
+}
+func (s *Service) Close() {
+ s.once.Do(func() {
+ s.mu.Lock()
+ s.closed = true
+ s.cancel()
+ s.mu.Unlock()
+ s.wg.Wait()
+ s.pool.Close()
+ })
+}
+
+// databaseError retains only allowlisted public information, never the cause.
+// In particular, wrapping a raw.Error must not retain an unsafe outer message.
+type databaseError struct {
+ public *raw.Error
+ canceled bool
+ deadline bool
+}
+
+func (e *databaseError) Error() string { return e.public.Error() }
+func (e *databaseError) Unwrap() error { return e.public }
+func (e *databaseError) Is(target error) bool {
+ return (target == context.Canceled && e.canceled) ||
+ (target == context.DeadlineExceeded && e.deadline)
+}
+func (e *databaseError) MarshalJSON() ([]byte, error) { return json.Marshal(e.public) }
+
+func dbError(e error) error {
+ if e == nil {
+ return nil
+ }
+ safe := &databaseError{
+ canceled: errors.Is(e, context.Canceled),
+ deadline: errors.Is(e, context.DeadlineExceeded),
+ }
+ code := raw.CodeUnavailable
+ var r *raw.Error
+ var p *pgconn.PgError
+ switch {
+ case errors.As(e, &r) && r != nil:
+ code = r.Code
+ case safe.canceled || safe.deadline:
+ code = raw.CodeTimeout
+ case errors.As(e, &p) && p != nil && (p.Code == "57014" || p.Code == "55P03"):
+ code = raw.CodeTimeout
+ }
+ safe.public = raw.NewError(code)
+ return safe
+}
+func (s *Service) transaction(ctx context.Context, isolation pgx.TxIsoLevel, fn func(context.Context, pgx.Tx) error) error {
+ ctx, done, e := s.operation(ctx)
+ if e != nil {
+ return e
+ }
+ defer done()
+ tx, e := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: isolation})
+ if e != nil {
+ return dbError(e)
+ }
+ defer tx.Rollback(context.Background())
+ if e = fn(ctx, tx); e != nil {
+ return dbError(e)
+ }
+ return dbError(tx.Commit(ctx))
+}
diff --git a/rag/raw/postgres/service_error_test.go b/rag/raw/postgres/service_error_test.go
new file mode 100644
index 0000000..a6c85c7
--- /dev/null
+++ b/rag/raw/postgres/service_error_test.go
@@ -0,0 +1,68 @@
+package postgres
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log/slog"
+ "strings"
+ "testing"
+
+ "github.com/jackc/pgx/v5/pgconn"
+ "github.com/mudler/localrecall/rag/raw"
+)
+
+func TestDBErrorSafeClassification(t *testing.T) {
+ secret := "SQL password=secret caller-payload"
+ for _, tc := range []struct {
+ name string
+ cause error
+ code string
+ canceled, deadline bool
+ }{
+ {"cancel", fmt.Errorf("%s: %w", secret, context.Canceled), raw.CodeTimeout, true, false},
+ {"deadline", fmt.Errorf("%s: %w", secret, context.DeadlineExceeded), raw.CodeTimeout, false, true},
+ {"both", errors.Join(context.Canceled, context.DeadlineExceeded), raw.CodeTimeout, true, true},
+ {"statement", &pgconn.PgError{Code: "57014", Message: secret}, raw.CodeTimeout, false, false},
+ {"lock", &pgconn.PgError{Code: "55P03", Detail: secret}, raw.CodeTimeout, false, false},
+ {"driver", &pgconn.PgError{Code: "08006", Message: secret}, raw.CodeUnavailable, false, false},
+ {"unknown code", &raw.Error{Code: secret, Message: secret}, raw.CodeInternal, false, false},
+ {"tainted raw", &raw.Error{Code: raw.CodeValidation, Message: secret}, raw.CodeValidation, false, false},
+ {"unknown", errors.New(secret), raw.CodeUnavailable, false, false},
+ {"raw wrapped", fmt.Errorf("%s: %w", secret, raw.NewError(raw.CodeValidation)), raw.CodeValidation, false, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ err := dbError(tc.cause)
+ var re *raw.Error
+ if !errors.As(err, &re) || re.Code != tc.code {
+ t.Fatalf("classification: %v", err)
+ }
+ if !errors.Is(err, raw.NewError(tc.code)) {
+ t.Error("safe classification identity lost")
+ }
+ if errors.Is(err, context.Canceled) != tc.canceled || errors.Is(err, context.DeadlineExceeded) != tc.deadline {
+ t.Error("context identity lost")
+ }
+ var driver *pgconn.PgError
+ if errors.As(err, &driver) {
+ t.Error("driver exposed")
+ }
+ b, e := json.Marshal(err)
+ if e != nil {
+ t.Fatal(e)
+ }
+ var log bytes.Buffer
+ slog.New(slog.NewJSONHandler(&log, nil)).Error("operation", "error", err)
+ for _, output := range []string{err.Error(), fmt.Sprintf("%+v %#v", err, err), string(b), log.String()} {
+ if strings.Contains(output, secret) {
+ t.Errorf("unsafe diagnostic: %s", output)
+ }
+ }
+ })
+ }
+ if dbError(nil) != nil {
+ t.Fatal("nil changed")
+ }
+}
diff --git a/rag/raw/store.go b/rag/raw/store.go
new file mode 100644
index 0000000..4e11333
--- /dev/null
+++ b/rag/raw/store.go
@@ -0,0 +1,98 @@
+package raw
+
+import "context"
+
+// Store owns raw persistence only, never embeddings, assets or sources.
+// Create's bool reports newly created. Close drains/cancels owned resources.
+type Store interface {
+ Capabilities(context.Context) (Capabilities, error)
+ CreateCollection(context.Context, CreateCollectionRequest) (Collection, bool, error)
+ ListCollections(context.Context, ListCollectionsRequest) (CollectionPage, error)
+ GetCollection(context.Context, string) (Collection, error)
+ Upsert(context.Context, string, UpsertRequest) (UpsertResponse, error)
+ GetRecords(context.Context, string, GetRecordsRequest) (GetRecordsResponse, error)
+ ListRecords(context.Context, string, ListRecordsRequest) (RecordPage, error)
+ DeleteRecords(context.Context, string, DeleteRecordsRequest) (DeleteRecordsResponse, error)
+ Reset(context.Context, string) error
+ DeleteCollection(context.Context, string) error
+ Search(context.Context, string, SearchRequest) (SearchResponse, error)
+ Close()
+}
+
+const (
+ CodeValidation = "VALIDATION_ERROR"
+ CodeCursor = "INVALID_CURSOR"
+ CodeUnauthorized = "UNAUTHORIZED"
+ CodeNotFound = "COLLECTION_NOT_FOUND"
+ CodeConflict = "COLLECTION_CONFIG_CONFLICT"
+ CodeSpaceMismatch = "EMBEDDING_SPACE_MISMATCH"
+ CodeBodyLimit = "BODY_TOO_LARGE"
+ CodeDeleteBound = "DELETE_BOUND_EXCEEDED"
+ CodeUnsupported = "NOT_IMPLEMENTED"
+ CodeUnavailable = "UNAVAILABLE"
+ CodeTimeout = "TIMEOUT"
+ CodeInternal = "INTERNAL_ERROR"
+)
+
+// Error contains only safe public information; never place driver errors,
+// SQL, credentials or caller payloads in Message. Retryable is not replay safety.
+type Error struct {
+ Code string `json:"code"`
+ Status int `json:"status"`
+ Retryable bool `json:"retryable"`
+ Message string `json:"message"`
+}
+
+func (e *Error) Error() string { return e.Message }
+
+// Is compares public classifications, not messages or transport status.
+func (e *Error) Is(target error) bool {
+ other, ok := target.(*Error)
+ return ok && e != nil && other != nil && e.Code == other.Code
+}
+
+func NewError(code string) *Error {
+ e := &Error{Code: code}
+ switch code {
+ case CodeValidation:
+ e.Status = 400
+ e.Message = "Invalid request"
+ case CodeCursor:
+ e.Status = 400
+ e.Message = "Invalid or mismatched cursor"
+ case CodeUnauthorized:
+ e.Status = 401
+ e.Message = "Authentication required"
+ case CodeNotFound:
+ e.Status = 404
+ e.Message = "Collection not found"
+ case CodeConflict:
+ e.Status = 409
+ e.Message = "Collection configuration conflicts"
+ case CodeSpaceMismatch:
+ e.Status = 409
+ e.Message = "Embedding space does not match"
+ case CodeBodyLimit:
+ e.Status = 413
+ e.Message = "Request body too large"
+ case CodeDeleteBound:
+ e.Status = 422
+ e.Message = "Delete selection exceeds limit"
+ case CodeUnsupported:
+ e.Status = 501
+ e.Message = "Capability not implemented"
+ case CodeUnavailable:
+ e.Status = 503
+ e.Retryable = true
+ e.Message = "Required dependency unavailable"
+ case CodeTimeout:
+ e.Status = 504
+ e.Retryable = true
+ e.Message = "Operation timed out"
+ default:
+ e.Code = CodeInternal
+ e.Status = 500
+ e.Message = "Internal error"
+ }
+ return e
+}
diff --git a/rag/raw/store_test.go b/rag/raw/store_test.go
new file mode 100644
index 0000000..b750f63
--- /dev/null
+++ b/rag/raw/store_test.go
@@ -0,0 +1,39 @@
+package raw
+
+import (
+ "errors"
+ "testing"
+)
+
+func TestRawErrorStatuses(t *testing.T) {
+ for _, tc := range []struct {
+ code string
+ status int
+ retryable bool
+ }{
+ {CodeValidation, 400, false}, {CodeCursor, 400, false}, {CodeUnauthorized, 401, false},
+ {CodeNotFound, 404, false}, {CodeConflict, 409, false}, {CodeSpaceMismatch, 409, false},
+ {CodeBodyLimit, 413, false}, {CodeDeleteBound, 422, false}, {CodeUnsupported, 501, false},
+ {CodeUnavailable, 503, true}, {CodeTimeout, 504, true}, {CodeInternal, 500, false},
+ } {
+ t.Run(tc.code, func(t *testing.T) {
+ e := NewError(tc.code)
+ if e.Code != tc.code || e.Status != tc.status || e.Retryable != tc.retryable || e.Error() == "" {
+ t.Fatal(e)
+ }
+ })
+ }
+ e := NewError("secret database error")
+ if e.Code != CodeInternal || e.Error() != "Internal error" {
+ t.Fatal(e)
+ }
+}
+
+func TestRawErrorClassification(t *testing.T) {
+ if !errors.Is(NewError(CodeTimeout), NewError(CodeTimeout)) {
+ t.Fatal("classification lost")
+ }
+ if errors.Is(NewError(CodeTimeout), NewError(CodeUnavailable)) {
+ t.Fatal("different codes match")
+ }
+}
diff --git a/rag/raw/types.go b/rag/raw/types.go
new file mode 100644
index 0000000..3c1911e
--- /dev/null
+++ b/rag/raw/types.go
@@ -0,0 +1,244 @@
+// Package raw defines embedder-free caller-supplied vector contracts.
+package raw
+
+import (
+ "encoding/json"
+ "time"
+)
+
+const (
+ MaxNameBytes = 128
+ MaxSpaceIDBytes = 256
+ MaxIDBytes = 256
+ MaxMetadataKeys = 32
+ MaxMetadataKeyBytes = 64
+ MaxMetadataValueBytes = 1024
+ DefaultListLimit = 50
+ MaxListLimit = 100
+ DefaultTopK = 10
+ MaxTopK = 100
+ MaxQueryTextBytes = 4096
+ MaxBodyBytes = 8 << 20
+ MaxDimensions = 2000
+ MaxBatch = 100
+ MaxTextBytes = 64 << 10
+ MaxMetadataBytes = 16 << 10
+ MaxFilterPredicates = 8
+ MaxDeleteMatches = 1000
+ MaxCursorBytes = 2048
+ SchemaVersion = 1
+)
+
+type LexicalConfig struct {
+ Enabled bool `json:"enabled"`
+ TextConfig string `json:"text_config"`
+}
+
+// Nil fields mean omitted, not an explicit false or empty configuration.
+type LexicalRequest struct {
+ Enabled *bool `json:"enabled,omitempty"`
+ TextConfig *string `json:"text_config,omitempty"`
+}
+type CreateCollectionRequest struct {
+ Name string `json:"name"`
+ Dimensions int `json:"dimensions"`
+ Metric string `json:"metric"`
+ EmbeddingSpaceID string `json:"embedding_space_id"`
+ Lexical *LexicalRequest `json:"lexical,omitempty"`
+}
+type Collection struct {
+ CollectionID string `json:"collection_id"`
+ Name string `json:"name"`
+ Kind string `json:"kind"`
+ Backend string `json:"backend"`
+ Dimensions int `json:"dimensions"`
+ Metric string `json:"metric"`
+ EmbeddingSpaceID string `json:"embedding_space_id"`
+ Lexical LexicalConfig `json:"lexical"`
+ SchemaVersion int `json:"schema_version"`
+ CreatedAt time.Time `json:"created_at"`
+}
+
+// Text omission/null both clear text on replacement; metadata omission becomes {}.
+type Record struct {
+ ID string `json:"id"`
+ Vector []float64 `json:"vector"`
+ Text *string `json:"text,omitempty"`
+ Metadata map[string]string `json:"metadata,omitempty"`
+}
+type CanonicalRecord struct {
+ ID string `json:"id"`
+ Vector []float32 `json:"vector,omitempty"`
+ Text *string `json:"text,omitempty"`
+ Metadata map[string]string `json:"metadata"`
+}
+type Filter struct {
+ MetadataEq map[string]string `json:"metadata_eq"`
+}
+type UpsertRequest struct {
+ EmbeddingSpaceID string `json:"embedding_space_id"`
+ Records []Record `json:"records"`
+}
+type UpsertResponse struct {
+ IDs []string `json:"ids"`
+}
+type GetRecordsRequest struct {
+ IDs []string `json:"ids"`
+ IncludeVectors bool `json:"include_vectors"`
+}
+type GetRecordsResponse struct {
+ Records []CanonicalRecord `json:"records"`
+ MissingIDs []string `json:"missing_ids"`
+}
+type ListCollectionsRequest struct {
+ Limit *int `json:"limit,omitempty"`
+ Cursor *string `json:"cursor,omitempty"`
+}
+type CollectionPage struct {
+ Collections []Collection `json:"collections"`
+ NextCursor *string `json:"next_cursor"`
+}
+type ListRecordsRequest struct {
+ Filter *Filter `json:"filter,omitempty"`
+ Limit *int `json:"limit,omitempty"`
+ Cursor *string `json:"cursor,omitempty"`
+ IncludeVectors bool `json:"include_vectors"`
+}
+type RecordPage struct {
+ Records []CanonicalRecord `json:"records"`
+ NextCursor *string `json:"next_cursor"`
+}
+
+// A nil IDs slice means absent. An explicitly empty slice is an invalid selector.
+type DeleteRecordsRequest struct {
+ IDs []string `json:"ids,omitempty"`
+ Filter *Filter `json:"filter,omitempty"`
+}
+type DeleteRecordsResponse struct {
+ DeletedCount int `json:"deleted_count"`
+}
+type ResetRequest struct{}
+type ResetResponse struct {
+ Reset bool `json:"reset"`
+}
+type DeleteCollectionResponse struct {
+ Deleted bool `json:"deleted"`
+}
+
+// Optional scalar pointers distinguish absence from explicit zero/empty values.
+// DecodeRequest rejects null except where the contract permits it (text on
+// records and cursors). Non-nil empty Vector is present for exclusivity checks.
+type SearchRequest struct {
+ Mode string `json:"mode"`
+ Accuracy *string `json:"accuracy,omitempty"`
+ EmbeddingSpaceID *string `json:"embedding_space_id,omitempty"`
+ Vector []float64 `json:"vector,omitempty"`
+ Text *string `json:"text,omitempty"`
+ TopK *int `json:"top_k,omitempty"`
+ Filter *Filter `json:"filter,omitempty"`
+}
+type SearchResult struct {
+ ID string `json:"id"`
+ Text *string `json:"text,omitempty"`
+ Metadata map[string]string `json:"metadata"`
+ Score float64 `json:"score"`
+ VectorSimilarity *float64 `json:"vector_similarity,omitempty"`
+ VectorRank *int `json:"vector_rank,omitempty"`
+ LexicalRank *int `json:"lexical_rank,omitempty"`
+}
+type FusionParameters struct {
+ K int `json:"k"`
+ VectorWeight float64 `json:"vector_weight"`
+ LexicalWeight float64 `json:"lexical_weight"`
+ CandidatesPerArm int `json:"candidates_per_arm"`
+}
+type SearchResponse struct {
+ Mode string `json:"mode"`
+ Accuracy string `json:"accuracy"`
+ ScoreKind string `json:"score_kind"`
+ Results []SearchResult `json:"results"`
+ Count int `json:"count"`
+ Fusion *FusionParameters `json:"fusion,omitempty"`
+}
+type Readiness struct {
+ Ready bool `json:"ready"`
+ Reason string `json:"reason,omitempty"`
+}
+type IndexCapability struct {
+ Strategy string `json:"strategy"`
+ Installed bool `json:"installed"`
+ Readiness
+}
+type Limits struct {
+ BodyBytes int `json:"body_bytes"`
+ Dimensions int `json:"dimensions"`
+ NameBytes int `json:"name_bytes"`
+ SpaceIDBytes int `json:"space_id_bytes"`
+ IDBytes int `json:"id_bytes"`
+ TextBytes int `json:"text_bytes"`
+ MetadataKeys int `json:"metadata_keys"`
+ MetadataKeyBytes int `json:"metadata_key_bytes"`
+ MetadataValueBytes int `json:"metadata_value_bytes"`
+ MetadataBytes int `json:"metadata_bytes"`
+ Batch int `json:"batch"`
+ FilterPredicates int `json:"filter_predicates"`
+ DeleteMatches int `json:"delete_matches"`
+ ListDefault int `json:"list_default"`
+ ListMax int `json:"list_max"`
+ CursorBytes int `json:"cursor_bytes"`
+ TopKDefault int `json:"top_k_default"`
+ TopKMax int `json:"top_k_max"`
+ QueryTextBytes int `json:"query_text_bytes"`
+}
+type Capabilities struct {
+ Backend string `json:"backend"`
+ RawCollections bool `json:"raw_collections"`
+ Readiness
+ Modes []string `json:"modes"`
+ ModeReadiness map[string]Readiness `json:"mode_readiness"`
+ AccuracyOptions []string `json:"accuracy_options"`
+ DefaultAccuracy string `json:"default_accuracy"`
+ Metrics []string `json:"metrics"`
+ Limits Limits `json:"limits"`
+ AtomicBatch bool `json:"atomic_batch"`
+ Persistent bool `json:"persistent"`
+ Indexes []IndexCapability `json:"indexes"`
+}
+
+func ContractLimits() Limits {
+ return Limits{
+ BodyBytes: MaxBodyBytes, Dimensions: MaxDimensions, NameBytes: MaxNameBytes,
+ SpaceIDBytes: MaxSpaceIDBytes, IDBytes: MaxIDBytes, TextBytes: MaxTextBytes,
+ MetadataKeys: MaxMetadataKeys, MetadataKeyBytes: MaxMetadataKeyBytes,
+ MetadataValueBytes: MaxMetadataValueBytes, MetadataBytes: MaxMetadataBytes,
+ Batch: MaxBatch, FilterPredicates: MaxFilterPredicates, DeleteMatches: MaxDeleteMatches,
+ ListDefault: DefaultListLimit, ListMax: MaxListLimit, CursorBytes: MaxCursorBytes,
+ TopKDefault: DefaultTopK, TopKMax: MaxTopK, QueryTextBytes: MaxQueryTextBytes,
+ }
+}
+
+// MarshalJSON preserves the distinction between absent and explicitly empty IDs.
+func (r DeleteRecordsRequest) MarshalJSON() ([]byte, error) {
+ type plain DeleteRecordsRequest
+ var ids *[]string
+ if r.IDs != nil {
+ ids = &r.IDs
+ }
+ return json.Marshal(struct {
+ plain
+ IDs *[]string `json:"ids,omitempty"`
+ }{plain: plain(r), IDs: ids})
+}
+
+// MarshalJSON preserves vector presence for search-mode exclusivity checks.
+func (r SearchRequest) MarshalJSON() ([]byte, error) {
+ type plain SearchRequest
+ var vector *[]float64
+ if r.Vector != nil {
+ vector = &r.Vector
+ }
+ return json.Marshal(struct {
+ plain
+ Vector *[]float64 `json:"vector,omitempty"`
+ }{plain: plain(r), Vector: vector})
+}
diff --git a/rag/raw/types_test.go b/rag/raw/types_test.go
new file mode 100644
index 0000000..9e9fe43
--- /dev/null
+++ b/rag/raw/types_test.go
@@ -0,0 +1,108 @@
+package raw
+
+import (
+ "encoding/json"
+ "strings"
+ "testing"
+)
+
+func TestRawDTOs(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ value any
+ want string
+ }{
+ {"get", GetRecordsResponse{Records: []CanonicalRecord{}, MissingIDs: []string{}}, `{"records":[],"missing_ids":[]}`},
+ {"page", RecordPage{Records: []CanonicalRecord{}}, `{"records":[],"next_cursor":null}`},
+ {"collection page", CollectionPage{Collections: []Collection{}}, `{"collections":[],"next_cursor":null}`},
+ {"upsert", UpsertResponse{IDs: []string{"x"}}, `{"ids":["x"]}`},
+ {"delete", DeleteRecordsResponse{1}, `{"deleted_count":1}`},
+ {"reset", ResetResponse{true}, `{"reset":true}`},
+ {"drop", DeleteCollectionResponse{true}, `{"deleted":true}`},
+ {"record", CanonicalRecord{ID: "x", Metadata: map[string]string{}}, `{"id":"x","metadata":{}}`},
+ {"empty text", CanonicalRecord{ID: "x", Text: ptr(""), Metadata: map[string]string{}}, `{"id":"x","text":"","metadata":{}}`},
+ {"search", SearchResponse{Mode: "lexical", Accuracy: "not_applicable", ScoreKind: "bm25", Results: []SearchResult{}}, `{"mode":"lexical","accuracy":"not_applicable","score_kind":"bm25","results":[],"count":0}`},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ b, e := json.Marshal(tc.value)
+ if e != nil || string(b) != tc.want {
+ t.Fatal(string(b), e)
+ }
+ })
+ }
+ if got := ResolveLexical(nil, ""); !got.Enabled || got.TextConfig != "english" {
+ t.Fatal(got)
+ }
+ if got := ResolveLexical(&LexicalRequest{Enabled: ptr(false)}, "simple"); got.Enabled || got.TextConfig != "simple" {
+ t.Fatal(got)
+ }
+}
+func TestRawStrictJSON(t *testing.T) {
+ for _, b := range []string{`{"ID":"x"}`, `{"id":"a","ID":"b"}`, `{"text":"\ud800"}`, `{"text":"\udc00"}`, `{"metadata":{"k":"a","k":"b"}}`, `{"metadata":{"text":null}}`, `{"vector":[null]}`, `null`, `[]`, `{"text":"` + string([]byte{255}) + `"}`} {
+ var r Record
+ if DecodeRequest([]byte(b), &r) == nil {
+ t.Fatal(b)
+ }
+ }
+ for _, b := range []string{`{"text":"\ud83d\ude00"}`, `{"text":null}`, `{"text":""}`, `{"text":"\\ud800"}`} {
+ var r Record
+ if e := DecodeRequest([]byte(b), &r); e != nil {
+ t.Fatal(b, e)
+ }
+ }
+ var r Record
+ if e := DecodeRequest([]byte(strings.Repeat(" ", MaxBodyBytes+1)), &r); e.(*Error).Status != 413 {
+ t.Fatal(e)
+ }
+}
+func TestRawLimitsAndReplacement(t *testing.T) {
+ d := descriptor()
+ r := UpsertRequest{EmbeddingSpaceID: "s", Records: []Record{{ID: strings.Repeat("é", 128), Vector: []float64{1, 0}, Text: ptr(strings.Repeat("x", 65536)), Metadata: map[string]string{strings.Repeat("k", 64): strings.Repeat("v", 1024)}}}}
+ out, e := ValidateUpsert(d, r)
+ if e != nil {
+ t.Fatal(e)
+ }
+ r.Records[0].Metadata["new"] = "v"
+ *r.Records[0].Text = "changed"
+ if len(out[0].Metadata) != 1 || len(*out[0].Text) != 65536 {
+ t.Fatal("canonical data aliases input")
+ }
+ m := map[string]string{}
+ for i := 0; i < 33; i++ {
+ m[string(rune('A'+i))] = "x"
+ }
+ r.Records[0].Metadata = m
+ if _, e := ValidateUpsert(d, r); e == nil {
+ t.Fatal("33 keys")
+ }
+ f := &Filter{MetadataEq: map[string]string{}}
+ for i := 0; i < 8; i++ {
+ f.MetadataEq[string(rune('a'+i))] = "x"
+ }
+ if ValidateFilter(f) != nil {
+ t.Fatal("eight predicates")
+ }
+ if ValidateDelete(DeleteRecordsRequest{Filter: f}) != nil {
+ t.Fatal("filter delete")
+ }
+ if ValidateDelete(DeleteRecordsRequest{IDs: []string{"x"}}) != nil {
+ t.Fatal("ID delete")
+ }
+ r.Records[0].Metadata = nil
+ r.Records[0].Text = nil
+ out, e = ValidateUpsert(d, r)
+ if e != nil || out[0].Text != nil || out[0].Metadata == nil || len(out[0].Metadata) != 0 {
+ t.Fatal(out, e)
+ }
+ for _, n := range []int{1, 100} {
+ s, e := ValidateSearch(d, SearchRequest{Mode: "lexical", Text: ptr("cat"), TopK: &n})
+ if e != nil || *s.TopK != n {
+ t.Fatal(s, e)
+ }
+ }
+ for _, s := range []string{strings.Repeat("x", 4097), "\x00", "\xff"} {
+ if _, e := ValidateSearch(d, SearchRequest{Mode: "lexical", Text: &s}); e == nil {
+ t.Fatal("query accepted")
+ }
+ }
+}
diff --git a/rag/raw/validate.go b/rag/raw/validate.go
new file mode 100644
index 0000000..fcad162
--- /dev/null
+++ b/rag/raw/validate.go
@@ -0,0 +1,449 @@
+package raw
+
+import (
+ "bytes"
+ "encoding/json"
+ "io"
+ "math"
+ "reflect"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+)
+
+func invalid() error { return NewError(CodeValidation) }
+func validString(s string, max int) bool {
+ return len(s) <= max && utf8.ValidString(s) && !strings.ContainsRune(s, 0)
+}
+func ValidateCreate(r CreateCollectionRequest) error {
+ if !validString(r.Name, MaxNameBytes) || r.Name == "" || r.Dimensions < 1 || r.Dimensions > MaxDimensions || r.Metric != "cosine" || !validString(r.EmbeddingSpaceID, MaxSpaceIDBytes) || r.EmbeddingSpaceID == "" {
+ return invalid()
+ }
+ if r.Lexical != nil && r.Lexical.TextConfig != nil && (!validString(*r.Lexical.TextConfig, MaxBodyBytes) || *r.Lexical.TextConfig == "") {
+ return invalid()
+ }
+ return nil
+}
+
+// ResolveLexical snapshots the caller-supplied creation-time environment value.
+// Database catalog resolution remains the persistence implementation's duty.
+func ResolveLexical(r *LexicalRequest, configured string) LexicalConfig {
+ if configured == "" {
+ configured = "english"
+ }
+ v := LexicalConfig{true, configured}
+ if r != nil {
+ if r.Enabled != nil {
+ v.Enabled = *r.Enabled
+ }
+ if r.TextConfig != nil {
+ v.TextConfig = *r.TextConfig
+ }
+ }
+ return v
+}
+func NormalizeVector(v []float64, dimensions int) ([]float32, error) {
+ if dimensions < 1 || dimensions > MaxDimensions || len(v) != dimensions {
+ return nil, invalid()
+ }
+ var norm float64
+ nonzero := false
+ for _, x := range v {
+ if math.IsNaN(x) || math.IsInf(x, 0) || math.Abs(x) > math.MaxFloat32 {
+ return nil, invalid()
+ }
+ f := float32(x)
+ if math.IsInf(float64(f), 0) {
+ return nil, invalid()
+ }
+ nonzero = nonzero || f != 0
+ norm = math.Hypot(norm, x)
+ }
+ if !nonzero || norm == 0 || math.IsInf(norm, 0) {
+ return nil, invalid()
+ }
+ out := make([]float32, len(v))
+ nonzero = false
+ for i, x := range v {
+ out[i] = float32(x / norm)
+ if math.IsNaN(float64(out[i])) || math.IsInf(float64(out[i]), 0) {
+ return nil, invalid()
+ }
+ nonzero = nonzero || out[i] != 0
+ }
+ if !nonzero {
+ return nil, invalid()
+ }
+ return out, nil
+}
+func ValidateID(id string) error {
+ if id == "" || !validString(id, MaxIDBytes) {
+ return invalid()
+ }
+ for _, r := range id {
+ if unicode.IsControl(r) {
+ return invalid()
+ }
+ }
+ return nil
+}
+func ValidateIDs(ids []string) error {
+ if len(ids) < 1 || len(ids) > MaxBatch {
+ return invalid()
+ }
+ seen := map[string]bool{}
+ for _, id := range ids {
+ if ValidateID(id) != nil || seen[id] {
+ return invalid()
+ }
+ seen[id] = true
+ }
+ return nil
+}
+func validateMetadata(m map[string]string, maxKeys int) error {
+ if len(m) > maxKeys {
+ return invalid()
+ }
+ for k, v := range m {
+ if !validString(k, MaxMetadataKeyBytes) || !validString(v, MaxMetadataValueBytes) {
+ return invalid()
+ }
+ }
+ b, e := json.Marshal(m)
+ if e != nil || len(b) > MaxMetadataBytes {
+ return invalid()
+ }
+ return nil
+}
+func ValidateUpsert(d Collection, r UpsertRequest) ([]CanonicalRecord, error) {
+ if r.EmbeddingSpaceID == "" {
+ return nil, invalid()
+ }
+ if r.EmbeddingSpaceID != d.EmbeddingSpaceID {
+ return nil, NewError(CodeSpaceMismatch)
+ }
+ ids := make([]string, len(r.Records))
+ for i, v := range r.Records {
+ ids[i] = v.ID
+ }
+ if e := ValidateIDs(ids); e != nil {
+ return nil, e
+ }
+ out := make([]CanonicalRecord, len(r.Records))
+ for i, v := range r.Records {
+ if v.Text != nil && !validString(*v.Text, MaxTextBytes) {
+ return nil, invalid()
+ }
+ if e := validateMetadata(v.Metadata, MaxMetadataKeys); e != nil {
+ return nil, e
+ }
+ vec, e := NormalizeVector(v.Vector, d.Dimensions)
+ if e != nil {
+ return nil, e
+ }
+ m := map[string]string{}
+ for k, val := range v.Metadata {
+ m[k] = val
+ }
+ var text *string
+ if v.Text != nil {
+ s := *v.Text
+ text = &s
+ }
+ out[i] = CanonicalRecord{v.ID, vec, text, m}
+ }
+ return out, nil
+}
+func ValidateFilter(f *Filter) error {
+ if f == nil {
+ return nil
+ }
+ return validateMetadata(f.MetadataEq, MaxFilterPredicates)
+}
+func ValidateDelete(r DeleteRecordsRequest) error {
+ if (r.IDs != nil) == (r.Filter != nil) {
+ return invalid()
+ }
+ if r.IDs != nil {
+ return ValidateIDs(r.IDs)
+ }
+ if len(r.Filter.MetadataEq) == 0 {
+ return invalid()
+ }
+ return ValidateFilter(r.Filter)
+}
+func ValidateLimit(n *int) (int, error) {
+ if n == nil {
+ return DefaultListLimit, nil
+ }
+ if *n < 1 || *n > MaxListLimit {
+ return 0, invalid()
+ }
+ return *n, nil
+}
+func ValidateSearch(d Collection, r SearchRequest) (SearchRequest, error) {
+ if e := ValidateFilter(r.Filter); e != nil {
+ return r, e
+ }
+ k := DefaultTopK
+ if r.TopK != nil {
+ k = *r.TopK
+ }
+ if k < 1 || k > MaxTopK {
+ return r, invalid()
+ }
+ r.TopK = &k
+ switch r.Mode {
+ case "vector":
+ if r.Text != nil {
+ return r, invalid()
+ }
+ case "lexical":
+ if r.Vector != nil || r.EmbeddingSpaceID != nil || r.Accuracy != nil {
+ return r, invalid()
+ }
+ case "hybrid":
+ default:
+ return r, invalid()
+ }
+ if r.Mode != "vector" {
+ if r.Text == nil || !validString(*r.Text, MaxQueryTextBytes) || strings.TrimSpace(*r.Text) == "" {
+ return r, invalid()
+ }
+ if !d.Lexical.Enabled {
+ return r, NewError(CodeUnsupported)
+ }
+ }
+ if r.Mode != "lexical" {
+ if r.EmbeddingSpaceID == nil || *r.EmbeddingSpaceID == "" {
+ return r, invalid()
+ }
+ if *r.EmbeddingSpaceID != d.EmbeddingSpaceID {
+ return r, NewError(CodeSpaceMismatch)
+ }
+ v, e := NormalizeVector(r.Vector, d.Dimensions)
+ if e != nil {
+ return r, e
+ }
+ r.Vector = make([]float64, len(v))
+ for i, x := range v {
+ r.Vector[i] = float64(x)
+ }
+ a := "approximate"
+ if r.Accuracy != nil {
+ a = *r.Accuracy
+ }
+ if a != "approximate" && a != "exact" {
+ return r, invalid()
+ }
+ r.Accuracy = &a
+ }
+ return r, nil
+}
+
+// DecodeRequest is the strict JSON boundary shared by future transports. It
+// rejects duplicate keys at every depth, unknown fields, null scalars, malformed
+// Unicode (including unpaired escaped surrogates), and trailing JSON. Library
+// callers use the validators directly; nil and empty slices remain distinct.
+func DecodeRequest(b []byte, dst any) error {
+ if len(b) > MaxBodyBytes {
+ return NewError(CodeBodyLimit)
+ }
+ if !utf8.Valid(b) || !validEscapes(b) {
+ return invalid()
+ }
+ dec := json.NewDecoder(bytes.NewReader(b))
+ dec.UseNumber()
+ if e := checkJSON(dec, "", 0); e != nil {
+ return invalid()
+ }
+ if _, e := dec.Token(); e != io.EOF {
+ return invalid()
+ }
+ // Text null is allowed only for records, not a search field.
+ if _, ok := dst.(*SearchRequest); ok {
+ var fields map[string]json.RawMessage
+ if json.Unmarshal(b, &fields) != nil {
+ return invalid()
+ }
+ if v, ok := fields["text"]; ok && bytes.Equal(bytes.TrimSpace(v), []byte("null")) {
+ return invalid()
+ }
+ }
+ if !exactFields(b, reflect.TypeOf(dst)) {
+ return invalid()
+ }
+ dec = json.NewDecoder(bytes.NewReader(b))
+ dec.DisallowUnknownFields()
+ if e := dec.Decode(dst); e != nil {
+ return invalid()
+ }
+ return nil
+}
+func checkJSON(d *json.Decoder, key string, depth int) error {
+ if depth > 64 {
+ return invalid()
+ }
+ tok, e := d.Token()
+ if e != nil {
+ return e
+ }
+ if tok == nil {
+ if key != "text" && key != "cursor" {
+ return invalid()
+ }
+ return nil
+ }
+ delim, ok := tok.(json.Delim)
+ if !ok {
+ return nil
+ }
+ switch delim {
+ case '{':
+ seen := map[string]bool{}
+ for d.More() {
+ t, e := d.Token()
+ if e != nil {
+ return e
+ }
+ k, ok := t.(string)
+ if !ok || seen[k] {
+ return invalid()
+ }
+ seen[k] = true
+ if key == "metadata" || key == "metadata_eq" {
+ v, e := d.Token()
+ if e != nil {
+ return e
+ }
+ if _, ok := v.(string); !ok {
+ return invalid()
+ }
+ } else if e := checkJSON(d, k, depth+1); e != nil {
+ return e
+ }
+ }
+ t, e := d.Token()
+ if e != nil || t != json.Delim('}') {
+ return invalid()
+ }
+ case '[':
+ for d.More() {
+ if e := checkJSON(d, "", depth+1); e != nil {
+ return e
+ }
+ }
+ t, e := d.Token()
+ if e != nil || t != json.Delim(']') {
+ return invalid()
+ }
+ default:
+ return invalid()
+ }
+ return nil
+}
+
+// encoding/json replaces lone surrogate escapes; reject them before decoding.
+func validEscapes(b []byte) bool {
+ hex := func(c byte) int {
+ switch {
+ case c >= '0' && c <= '9':
+ return int(c - '0')
+ case c >= 'a' && c <= 'f':
+ return int(c-'a') + 10
+ case c >= 'A' && c <= 'F':
+ return int(c-'A') + 10
+ }
+ return -1
+ }
+ unit := func(i int) int {
+ if i+4 > len(b) {
+ return -1
+ }
+ n := 0
+ for _, c := range b[i : i+4] {
+ v := hex(c)
+ if v < 0 {
+ return -1
+ }
+ n = n*16 + v
+ }
+ return n
+ }
+ for i := 0; i < len(b); i++ {
+ if b[i] != '\\' {
+ continue
+ }
+ i++
+ if i >= len(b) {
+ return false
+ }
+ if b[i] != 'u' {
+ continue
+ }
+ n := unit(i + 1)
+ if n < 0 {
+ return false
+ }
+ i += 4
+ if n >= 0xdc00 && n <= 0xdfff {
+ return false
+ }
+ if n >= 0xd800 && n <= 0xdbff {
+ if i+6 >= len(b) || b[i+1] != '\\' || b[i+2] != 'u' {
+ return false
+ }
+ m := unit(i + 3)
+ if m < 0xdc00 || m > 0xdfff {
+ return false
+ }
+ i += 6
+ }
+ }
+ return true
+}
+
+// encoding/json accepts case-insensitive field aliases; the wire contract does not.
+func exactFields(b []byte, t reflect.Type) bool {
+ if t == nil {
+ return false
+ }
+ for t.Kind() == reflect.Pointer {
+ t = t.Elem()
+ }
+ if bytes.Equal(bytes.TrimSpace(b), []byte("null")) {
+ return true
+ }
+ switch t.Kind() {
+ case reflect.Struct:
+ var m map[string]json.RawMessage
+ if json.Unmarshal(b, &m) != nil {
+ return false
+ }
+ fields := map[string]reflect.Type{}
+ for i := 0; i < t.NumField(); i++ {
+ f := t.Field(i)
+ name := strings.Split(f.Tag.Get("json"), ",")[0]
+ if name != "" && name != "-" {
+ fields[name] = f.Type
+ }
+ }
+ for k, v := range m {
+ ft, ok := fields[k]
+ if !ok || !exactFields(v, ft) {
+ return false
+ }
+ }
+ case reflect.Slice:
+ var a []json.RawMessage
+ if json.Unmarshal(b, &a) != nil {
+ return false
+ }
+ for _, v := range a {
+ if !exactFields(v, t.Elem()) {
+ return false
+ }
+ }
+ }
+ return true
+}
diff --git a/rag/raw/validate_test.go b/rag/raw/validate_test.go
new file mode 100644
index 0000000..583a1a5
--- /dev/null
+++ b/rag/raw/validate_test.go
@@ -0,0 +1,283 @@
+package raw
+
+import (
+ "encoding/json"
+ "fmt"
+ "math"
+ "strings"
+ "testing"
+)
+
+func ptr[T any](v T) *T { return &v }
+func descriptor() Collection {
+ return Collection{CollectionID: "123e4567-e89b-12d3-a456-426614174000", Dimensions: 2, EmbeddingSpaceID: "s", Lexical: LexicalConfig{Enabled: true}}
+}
+func TestRawNormalize(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ v []float64
+ dim int
+ ok bool
+ }{
+ {"unit", []float64{3, 4}, 2, true}, {"tiny", []float64{1e-40, 2e-40}, 2, true}, {"huge", []float64{math.MaxFloat32, math.MaxFloat32}, 2, true}, {"zero", []float64{0, 0}, 2, false}, {"underflow", []float64{1e-300, 0}, 2, false}, {"overflow", []float64{math.MaxFloat64, 0}, 2, false}, {"nan", []float64{math.NaN(), 0}, 2, false}, {"inf", []float64{math.Inf(1), 0}, 2, false}, {"dimension", []float64{1}, 2, false}, {"invalid dimension", nil, 0, false}} {
+ t.Run(tc.name, func(t *testing.T) {
+ v, e := NormalizeVector(tc.v, tc.dim)
+ if (e == nil) != tc.ok {
+ t.Fatalf("%v", e)
+ }
+ if e == nil {
+ var n float64
+ for _, x := range v {
+ n += float64(x) * float64(x)
+ }
+ if math.Abs(n-1) > 1e-6 {
+ t.Fatal(n)
+ }
+ }
+ })
+ }
+ v, _ := NormalizeVector([]float64{3, 4}, 2)
+ if v[0] != float32(.6) || v[1] != float32(.8) {
+ t.Fatal(v)
+ }
+}
+func TestRawCreate(t *testing.T) {
+ base := CreateCollectionRequest{Name: "x", Dimensions: 2, Metric: "cosine", EmbeddingSpaceID: "s"}
+ for _, tc := range []struct {
+ name string
+ edit func(*CreateCollectionRequest)
+ ok bool
+ }{
+ {"valid", func(*CreateCollectionRequest) {}, true}, {"name", func(r *CreateCollectionRequest) { r.Name = strings.Repeat("é", 65) }, false}, {"utf8", func(r *CreateCollectionRequest) { r.Name = "\xff" }, false}, {"dimensions", func(r *CreateCollectionRequest) { r.Dimensions = 2001 }, false}, {"metric", func(r *CreateCollectionRequest) { r.Metric = "dot" }, false}, {"space", func(r *CreateCollectionRequest) { r.EmbeddingSpaceID = "" }, false}, {"space limit", func(r *CreateCollectionRequest) { r.EmbeddingSpaceID = strings.Repeat("x", 257) }, false}} {
+ t.Run(tc.name, func(t *testing.T) {
+ r := base
+ tc.edit(&r)
+ if (ValidateCreate(r) == nil) != tc.ok {
+ t.Fatal(r)
+ }
+ })
+ }
+}
+func TestRawUpsert(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ edit func(*UpsertRequest)
+ ok bool
+ }{
+ {"valid", func(*UpsertRequest) {}, true}, {"duplicate", func(r *UpsertRequest) { r.Records = append(r.Records, r.Records[0]) }, false}, {"empty", func(r *UpsertRequest) { r.Records = nil }, false}, {"batch", func(r *UpsertRequest) { r.Records = make([]Record, 101) }, false}, {"control", func(r *UpsertRequest) { r.Records[0].ID = "a\n" }, false}, {"unicode control", func(r *UpsertRequest) { r.Records[0].ID = "a\u0085" }, false}, {"id limit", func(r *UpsertRequest) { r.Records[0].ID = strings.Repeat("a", 257) }, false}, {"id utf8", func(r *UpsertRequest) { r.Records[0].ID = "\xff" }, false}, {"text nul", func(r *UpsertRequest) { r.Records[0].Text = ptr("a\x00") }, false}, {"text utf8", func(r *UpsertRequest) { r.Records[0].Text = ptr("\xff") }, false}, {"text limit", func(r *UpsertRequest) { r.Records[0].Text = ptr(strings.Repeat("x", 65537)) }, false}, {"metadata key", func(r *UpsertRequest) { r.Records[0].Metadata = map[string]string{strings.Repeat("x", 65): "x"} }, false}, {"metadata value", func(r *UpsertRequest) { r.Records[0].Metadata = map[string]string{"x": strings.Repeat("x", 1025)} }, false}, {"metadata serialized", func(r *UpsertRequest) {
+ m := map[string]string{}
+ for i := 0; i < 32; i++ {
+ m[string(rune('A'+i))] = strings.Repeat("x", 1024)
+ }
+ r.Records[0].Metadata = m
+ }, false}, {"space conflict", func(r *UpsertRequest) { r.EmbeddingSpaceID = "other" }, false}} {
+ t.Run(tc.name, func(t *testing.T) {
+ r := UpsertRequest{EmbeddingSpaceID: "s", Records: []Record{{ID: "x", Vector: []float64{3, 4}}}}
+ tc.edit(&r)
+ out, e := ValidateUpsert(descriptor(), r)
+ if (e == nil) != tc.ok {
+ t.Fatalf("%v", e)
+ }
+ if e == nil && out[0].Metadata == nil {
+ t.Fatal("replacement must produce empty metadata")
+ }
+ })
+ }
+}
+func TestRawSearch(t *testing.T) {
+ for _, tc := range []struct {
+ body string
+ ok bool
+ }{
+ {`{"mode":"vector","vector":[3,4],"embedding_space_id":"s"}`, true},
+ {`{"mode":"vector","vector":[3,4],"embedding_space_id":"s","text":""}`, false},
+ {`{"mode":"vector","vector":[3,4],"embedding_space_id":"s","text":null}`, false},
+ {`{"mode":"lexical","text":"cat"}`, true}, {`{"mode":"lexical","text":"cat","vector":[]}`, false}, {`{"mode":"lexical","text":"cat","vector":null}`, false}, {`{"mode":"lexical","text":"cat","accuracy":""}`, false}, {`{"mode":"lexical","text":"cat","embedding_space_id":""}`, false},
+ {`{"mode":"hybrid","text":"cat","vector":[1,0],"embedding_space_id":"s","accuracy":"exact"}`, true}, {`{"mode":"lexical","text":" "}`, false}, {`{"mode":"lexical","text":"cat","top_k":0}`, false}, {`{"mode":"lexical","text":"cat","top_k":101}`, false}, {`{"text":"cat"}`, false}, {`{"mode":"vector","vector":[1,0],"embedding_space_id":"s","accuracy":""}`, false},
+ } {
+ t.Run(tc.body, func(t *testing.T) {
+ var r SearchRequest
+ e := DecodeRequest([]byte(tc.body), &r)
+ if e == nil {
+ r, e = ValidateSearch(descriptor(), r)
+ }
+ if (e == nil) != tc.ok {
+ t.Fatalf("%v", e)
+ }
+ if e == nil && *r.TopK != 10 {
+ t.Fatal("default top_k")
+ }
+ if e == nil && r.Mode == "vector" && *r.Accuracy != "approximate" {
+ t.Fatal("default accuracy")
+ }
+ })
+ }
+}
+func TestRawSelectorsAndJSON(t *testing.T) {
+ for _, ids := range [][]string{nil, {}, {"a", "a"}, {"\x00"}} {
+ if ValidateIDs(ids) == nil {
+ t.Fatal(ids)
+ }
+ }
+ if ValidateIDs([]string{"a"}) != nil {
+ t.Fatal("valid IDs")
+ }
+ for _, body := range []string{`{}`, `{"ids":[]}`, `{"filter":{"metadata_eq":{}}}`, `{"ids":["a"],"filter":{"metadata_eq":{"x":"y"}}}`} {
+ var r DeleteRecordsRequest
+ _ = DecodeRequest([]byte(body), &r)
+ if ValidateDelete(r) == nil {
+ t.Fatal(body)
+ }
+ }
+ for _, body := range []string{`{"id":"a","id":"b"}`, `{"unknown":1}`, `{} {}`, `{"metadata":{"a":2}}`, `{"metadata":{"a":null}}`, `{"metadata":{"a":{"b":"c"}}}`, `{"vector":null}`} {
+ var r Record
+ if DecodeRequest([]byte(body), &r) == nil {
+ t.Fatal(body)
+ }
+ }
+ f := &Filter{MetadataEq: map[string]string{}}
+ for i := 0; i < 9; i++ {
+ f.MetadataEq[string(rune('a'+i))] = "v"
+ }
+ if ValidateFilter(f) == nil {
+ t.Fatal("filter bound")
+ }
+ for _, n := range []int{0, -1, 101} {
+ if _, e := ValidateLimit(&n); e == nil {
+ t.Fatal(n)
+ }
+ }
+ if n, e := ValidateLimit(nil); e != nil || n != 50 {
+ t.Fatal(n, e)
+ }
+}
+func TestRawCursor(t *testing.T) {
+ cid := descriptor().CollectionID
+ f := &Filter{MetadataEq: map[string]string{"source": "a"}}
+ c, e := EncodeRecordCursor(cid, "last", f, false)
+ if e != nil {
+ t.Fatal(e)
+ }
+ last, e := DecodeRecordCursor(c, cid, f, false)
+ if e != nil || last != "last" {
+ t.Fatal(last, e)
+ }
+ for _, tc := range []struct {
+ c, cid string
+ f *Filter
+ v bool
+ }{{c, cid, nil, false}, {c, cid, f, true}, {c, "123e4567-e89b-12d3-a456-426614174001", f, false}, {"bad", cid, f, false}, {strings.Repeat("a", 2049), cid, f, false}} {
+ if _, e := DecodeRecordCursor(tc.c, tc.cid, tc.f, tc.v); e == nil {
+ t.Fatal(tc)
+ }
+ }
+ c, e = EncodeCollectionCursor(cid)
+ if e != nil {
+ t.Fatal(e)
+ }
+ if id, e := DecodeCollectionCursor(c); e != nil || id != cid {
+ t.Fatal(id, e)
+ }
+}
+func TestRawError(t *testing.T) {
+ e := NewError(CodeSpaceMismatch)
+ if e.Status != 409 || e.Retryable || e.Code != "EMBEDDING_SPACE_MISMATCH" {
+ t.Fatal(e)
+ }
+ b, _ := json.Marshal(e)
+ if !strings.Contains(string(b), `"code"`) {
+ t.Fatal(string(b))
+ }
+}
+
+// Capabilities must describe the same inclusive boundaries as library validation.
+func TestRawContractLimitsParity(t *testing.T) {
+ l := ContractLimits()
+ create := CreateCollectionRequest{Name: "n", Dimensions: 1, Metric: "cosine", EmbeddingSpaceID: "s"}
+ for _, tc := range []struct {
+ name string
+ max int
+ check func(int) error
+ }{
+ {"name", l.NameBytes, func(n int) error { r := create; r.Name = strings.Repeat("n", n); return ValidateCreate(r) }},
+ {"space", l.SpaceIDBytes, func(n int) error { r := create; r.EmbeddingSpaceID = strings.Repeat("s", n); return ValidateCreate(r) }},
+ {"dimensions", l.Dimensions, func(n int) error { r := create; r.Dimensions = n; return ValidateCreate(r) }},
+ {"id", l.IDBytes, func(n int) error { return ValidateID(strings.Repeat("i", n)) }},
+ {"batch", l.Batch, func(n int) error {
+ ids := make([]string, n)
+ for i := range ids {
+ ids[i] = fmt.Sprint(i)
+ }
+ return ValidateIDs(ids)
+ }},
+ {"text", l.TextBytes, func(n int) error {
+ _, err := ValidateUpsert(descriptor(), UpsertRequest{EmbeddingSpaceID: "s", Records: []Record{{ID: "i", Vector: []float64{1, 0}, Text: ptr(strings.Repeat("t", n))}}})
+ return err
+ }},
+ {"metadata keys", l.MetadataKeys, func(n int) error {
+ m := map[string]string{}
+ for i := 0; i < n; i++ {
+ m[fmt.Sprint(i)] = "v"
+ }
+ return validateMetadata(m, l.MetadataKeys)
+ }},
+ {"metadata key", l.MetadataKeyBytes, func(n int) error {
+ return validateMetadata(map[string]string{strings.Repeat("k", n): "v"}, l.MetadataKeys)
+ }},
+ {"metadata value", l.MetadataValueBytes, func(n int) error {
+ return validateMetadata(map[string]string{"k": strings.Repeat("v", n)}, l.MetadataKeys)
+ }},
+ {"filter", l.FilterPredicates, func(n int) error {
+ m := map[string]string{}
+ for i := 0; i < n; i++ {
+ m[fmt.Sprint(i)] = "v"
+ }
+ return ValidateFilter(&Filter{MetadataEq: m})
+ }},
+ {"list", l.ListMax, func(n int) error { _, err := ValidateLimit(&n); return err }},
+ {"top k", l.TopKMax, func(n int) error {
+ _, err := ValidateSearch(descriptor(), SearchRequest{Mode: "lexical", Text: ptr("q"), TopK: &n})
+ return err
+ }},
+ {"query", l.QueryTextBytes, func(n int) error {
+ _, err := ValidateSearch(descriptor(), SearchRequest{Mode: "lexical", Text: ptr(strings.Repeat("q", n))})
+ return err
+ }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ if err := tc.check(tc.max); err != nil {
+ t.Fatalf("advertised maximum rejected: %v", err)
+ }
+ if tc.check(tc.max+1) == nil {
+ t.Fatal("maximum+1 accepted")
+ }
+ })
+ }
+ if n, err := ValidateLimit(nil); err != nil || n != l.ListDefault {
+ t.Fatal("list default parity", n, err)
+ }
+ if r, err := ValidateSearch(descriptor(), SearchRequest{Mode: "lexical", Text: ptr("q")}); err != nil || *r.TopK != l.TopKDefault {
+ t.Fatal("search default parity", r, err)
+ }
+ // An exact serialized metadata boundary, with each individual value within its limit.
+ m := map[string]string{}
+ for i := 0; i < 15; i++ {
+ m[fmt.Sprintf("%02d", i)] = strings.Repeat("v", l.MetadataValueBytes)
+ }
+ m["last"] = ""
+ b, err := json.Marshal(m)
+ if err != nil {
+ t.Fatal(err)
+ }
+ m["last"] = strings.Repeat("v", l.MetadataBytes-len(b))
+ if err := validateMetadata(m, l.MetadataKeys); err != nil {
+ t.Fatal(err)
+ }
+ m["last"] += "v"
+ if validateMetadata(m, l.MetadataKeys) == nil {
+ t.Fatal("serialized metadata maximum+1 accepted")
+ }
+ if l.BodyBytes != MaxBodyBytes || l.CursorBytes != MaxCursorBytes || l.DeleteMatches != MaxDeleteMatches {
+ t.Fatal("shared limit parity")
+ }
+}
diff --git a/rag/sources/router_test.go b/rag/sources/router_test.go
index 49725ac..e1b9b4b 100644
--- a/rag/sources/router_test.go
+++ b/rag/sources/router_test.go
@@ -4,13 +4,27 @@ import (
. "github.com/mudler/localrecall/rag/sources"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
+ "net/http"
+ "net/http/httptest"
)
var _ = Describe("SourceRouter", func() {
+ var server *httptest.Server
+ BeforeEach(func() {
+ server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/sitemap.xml" {
+ w.Header().Set("Content-Type", "application/xml")
+ _, _ = w.Write([]byte(""))
+ return
+ }
+ http.NotFound(w, r)
+ }))
+ })
+ AfterEach(func() { server.Close() })
It("should identify git repository URLs", func() {
config := &Config{}
// Note: This will fail if git is not available, but tests the routing logic
- _, err := SourceRouter("https://example.com/repo.git", config)
+ _, err := SourceRouter(server.URL+"/repo.git", config)
// We expect an error since we're not testing actual git functionality
// Just verify it's trying to route to git
Expect(err).ToNot(BeNil()) // Git operation will fail, but routing works
@@ -19,7 +33,7 @@ var _ = Describe("SourceRouter", func() {
It("should identify sitemap URLs", func() {
config := &Config{}
// This will fail if network is not available, but tests routing
- _, err := SourceRouter("https://example.com/sitemap.xml", config)
+ _, err := SourceRouter(server.URL+"/sitemap.xml", config)
// We expect an error since we're not testing actual network functionality
Expect(err).ToNot(BeNil()) // Network operation will fail, but routing works
})
@@ -27,7 +41,7 @@ var _ = Describe("SourceRouter", func() {
It("should default to web page for regular URLs", func() {
config := &Config{}
// This may succeed or fail depending on network, but tests routing
- _, err := SourceRouter("https://example.com/page", config)
+ _, err := SourceRouter(server.URL+"/page", config)
// The function may succeed or fail, but should not panic
// We just verify the routing logic works (no panic)
_ = err // Accept either success or failure
diff --git a/raw_routes.go b/raw_routes.go
new file mode 100644
index 0000000..8785cb0
--- /dev/null
+++ b/raw_routes.go
@@ -0,0 +1,250 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "github.com/labstack/echo/v4"
+ "github.com/mudler/localrecall/rag/raw"
+ "io"
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
+)
+
+func rawFailure(c echo.Context, err error) error {
+ e := raw.NewError(raw.CodeInternal)
+ var re *raw.Error
+ if errors.As(err, &re) {
+ e = raw.NewError(re.Code)
+ }
+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+ e = raw.NewError(raw.CodeTimeout)
+ // Do not drain an incomplete timed-out body before sending the error.
+ c.Response().Header().Set("Connection", "close")
+ }
+ return c.JSON(e.Status, struct {
+ Success bool `json:"success"`
+ Error *raw.Error `json:"error"`
+ }{false, e})
+}
+func rawBody(c echo.Context, dst any) error {
+ b, err := io.ReadAll(http.MaxBytesReader(c.Response(), c.Request().Body, raw.MaxBodyBytes))
+ if c.Request().Context().Err() != nil {
+ return c.Request().Context().Err()
+ }
+ if err != nil {
+ var limit *http.MaxBytesError
+ if errors.As(err, &limit) {
+ return raw.NewError(raw.CodeBodyLimit)
+ }
+ return raw.NewError(raw.CodeValidation)
+ }
+ if err := raw.DecodeRequest(b, dst); err != nil {
+ return err
+ }
+ // NUL is invalid in every PostgreSQL-bound string, including map keys.
+ decoder := json.NewDecoder(bytes.NewReader(b))
+ for {
+ token, err := decoder.Token()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return raw.NewError(raw.CodeValidation)
+ }
+ if text, ok := token.(string); ok && strings.ContainsRune(text, 0) {
+ return raw.NewError(raw.CodeValidation)
+ }
+ }
+ return nil
+}
+
+// registerRawAPIRoutes inherits the instance middleware installed by registerAPIRoutes.
+// A nil store denotes a failed bootstrap, never a fallback to legacy persistence.
+func registerRawAPIRoutes(e *echo.Echo, store raw.Store) {
+ g := e.Group("/api/raw/v1")
+ g.Use(func(next echo.HandlerFunc) echo.HandlerFunc {
+ return func(c echo.Context) error {
+ ctx, cancel := context.WithTimeout(c.Request().Context(), 30*time.Second)
+ defer cancel()
+ c.SetRequest(c.Request().WithContext(ctx))
+ // Incoming Body.Read does not observe context cancellation. Interrupt
+ // the transport only for raw requests, including early cancellation.
+ controller := http.NewResponseController(c.Response().Writer)
+ interrupted := make(chan struct{})
+ stop := context.AfterFunc(ctx, func() {
+ _ = controller.SetReadDeadline(time.Now())
+ close(interrupted)
+ })
+ defer func() {
+ if !stop() {
+ <-interrupted
+ }
+ // Do not leak a raw deadline into a subsequent keep-alive request.
+ _ = controller.SetReadDeadline(time.Time{})
+ }()
+
+ if store == nil && c.Path() != "/api/raw/v1/capabilities" {
+ if vectorEngine != "postgres" {
+ return rawFailure(c, raw.NewError(raw.CodeUnsupported))
+ }
+ return rawFailure(c, raw.NewError(raw.CodeUnavailable))
+ }
+ return next(c)
+ }
+ })
+ g.GET("/capabilities", func(c echo.Context) error {
+ if store == nil {
+ caps := raw.Capabilities{Backend: vectorEngine, RawCollections: vectorEngine == "postgres", Readiness: raw.Readiness{Reason: "Required dependency unavailable"}, Limits: raw.ContractLimits()}
+ if caps.RawCollections {
+ caps.Modes = []string{"vector", "lexical", "hybrid"}
+ caps.AccuracyOptions = []string{"approximate", "exact"}
+ caps.DefaultAccuracy = "approximate"
+ caps.Metrics = []string{"cosine"}
+ caps.AtomicBatch = true
+ caps.Persistent = true
+ }
+ return c.JSON(200, successResponse("", caps))
+ }
+ data, err := store.Capabilities(c.Request().Context())
+ if err != nil {
+ return rawFailure(c, err)
+ }
+ return c.JSON(200, successResponse("", data))
+ })
+ g.POST("/collections", func(c echo.Context) error {
+ var req raw.CreateCollectionRequest
+ if err := rawBody(c, &req); err != nil {
+ return rawFailure(c, err)
+ }
+ data, created, err := store.CreateCollection(c.Request().Context(), req)
+ if err != nil {
+ return rawFailure(c, err)
+ }
+ status := 200
+ if created {
+ status = 201
+ }
+ return c.JSON(status, successResponse("", data))
+ })
+ g.GET("/collections", func(c echo.Context) error {
+ var req raw.ListCollectionsRequest
+ for k, v := range c.QueryParams() {
+ if len(v) != 1 {
+ return rawFailure(c, raw.NewError(raw.CodeValidation))
+ }
+ switch k {
+ case "limit":
+ n, err := strconv.Atoi(v[0])
+ if err != nil {
+ return rawFailure(c, raw.NewError(raw.CodeValidation))
+ }
+ req.Limit = &n
+ case "cursor":
+ req.Cursor = &v[0]
+ default:
+ return rawFailure(c, raw.NewError(raw.CodeValidation))
+ }
+ }
+ data, err := store.ListCollections(c.Request().Context(), req)
+ if err != nil {
+ return rawFailure(c, err)
+ }
+ return c.JSON(200, successResponse("", data))
+ })
+ g.GET("/collections/:cid", func(c echo.Context) error {
+ data, err := store.GetCollection(c.Request().Context(), c.Param("cid"))
+ if err != nil {
+ return rawFailure(c, err)
+ }
+ return c.JSON(200, successResponse("", data))
+ })
+ g.POST("/collections/:cid/records/upsert", func(c echo.Context) error {
+ var req raw.UpsertRequest
+ if err := rawBody(c, &req); err != nil {
+ return rawFailure(c, err)
+ }
+ data, err := store.Upsert(c.Request().Context(), c.Param("cid"), req)
+ if err != nil {
+ return rawFailure(c, err)
+ }
+ return c.JSON(200, successResponse("", data))
+ })
+ g.POST("/collections/:cid/records/get", func(c echo.Context) error {
+ var req raw.GetRecordsRequest
+ if err := rawBody(c, &req); err != nil {
+ return rawFailure(c, err)
+ }
+ data, err := store.GetRecords(c.Request().Context(), c.Param("cid"), req)
+ if err != nil {
+ return rawFailure(c, err)
+ }
+ return c.JSON(200, successResponse("", data))
+ })
+ g.POST("/collections/:cid/records/list", func(c echo.Context) error {
+ var req raw.ListRecordsRequest
+ if err := rawBody(c, &req); err != nil {
+ return rawFailure(c, err)
+ }
+ data, err := store.ListRecords(c.Request().Context(), c.Param("cid"), req)
+ if err != nil {
+ return rawFailure(c, err)
+ }
+ return c.JSON(200, successResponse("", data))
+ })
+ g.POST("/collections/:cid/records/delete", func(c echo.Context) error {
+ var req raw.DeleteRecordsRequest
+ if err := rawBody(c, &req); err != nil {
+ return rawFailure(c, err)
+ }
+ data, err := store.DeleteRecords(c.Request().Context(), c.Param("cid"), req)
+ if err != nil {
+ return rawFailure(c, err)
+ }
+ return c.JSON(200, successResponse("", data))
+ })
+ g.POST("/collections/:cid/search", func(c echo.Context) error {
+ var req raw.SearchRequest
+ if err := rawBody(c, &req); err != nil {
+ return rawFailure(c, err)
+ }
+ data, err := store.Search(c.Request().Context(), c.Param("cid"), req)
+ if err != nil {
+ return rawFailure(c, err)
+ }
+ return c.JSON(200, successResponse("", data))
+ })
+ g.POST("/collections/:cid/reset", func(c echo.Context) error {
+ var req raw.ResetRequest
+ if err := rawBody(c, &req); err != nil {
+ return rawFailure(c, err)
+ }
+ if err := store.Reset(c.Request().Context(), c.Param("cid")); err != nil {
+ return rawFailure(c, err)
+ }
+ return c.JSON(200, successResponse("", raw.ResetResponse{Reset: true}))
+ })
+ g.DELETE("/collections/:cid", func(c echo.Context) error {
+ b, err := io.ReadAll(http.MaxBytesReader(c.Response(), c.Request().Body, raw.MaxBodyBytes))
+ if c.Request().Context().Err() != nil {
+ return rawFailure(c, c.Request().Context().Err())
+ }
+ if err != nil {
+ var limit *http.MaxBytesError
+ if errors.As(err, &limit) {
+ return rawFailure(c, raw.NewError(raw.CodeBodyLimit))
+ }
+ return rawFailure(c, raw.NewError(raw.CodeValidation))
+ }
+ if len(b) != 0 {
+ return rawFailure(c, raw.NewError(raw.CodeValidation))
+ }
+ if err := store.DeleteCollection(c.Request().Context(), c.Param("cid")); err != nil {
+ return rawFailure(c, err)
+ }
+ return c.JSON(200, successResponse("", raw.DeleteCollectionResponse{Deleted: true}))
+ })
+}
diff --git a/raw_routes_test.go b/raw_routes_test.go
new file mode 100644
index 0000000..3b02708
--- /dev/null
+++ b/raw_routes_test.go
@@ -0,0 +1,486 @@
+package main
+
+import (
+ "bufio"
+ "context"
+ "errors"
+ "fmt"
+ "github.com/jackc/pgx/v5"
+ "github.com/labstack/echo/v4"
+ "github.com/mudler/localrecall/pkg/client"
+ "github.com/mudler/localrecall/rag/engine"
+ "github.com/mudler/localrecall/rag/raw"
+ rawpostgres "github.com/mudler/localrecall/rag/raw/postgres"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "os/exec"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+type rawSpy struct {
+ raw.Store
+ calls int
+ err error
+}
+
+func (s *rawSpy) CreateCollection(context.Context, raw.CreateCollectionRequest) (raw.Collection, bool, error) {
+ s.calls++
+ return raw.Collection{}, true, s.err
+}
+func TestRawHTTPValidation(t *testing.T) {
+ s := &rawSpy{}
+ e := echo.New()
+ registerRawAPIRoutes(e, s)
+ for _, body := range []string{`null`, `{} {}`, `{"name":"a","name":"b"}`, `{"unknown":1}`, `{"lexical":{"enabled":true,"enabled":false}}`, `{"name":"\u0000"}`, "{\"name\":\"\xff\"}"} {
+ r := httptest.NewRecorder()
+ e.ServeHTTP(r, httptest.NewRequest("POST", "/api/raw/v1/collections", strings.NewReader(body)))
+ if r.Code != 400 {
+ t.Fatalf("%q: %d %s", body, r.Code, r.Body)
+ }
+ }
+ r := httptest.NewRecorder()
+ e.ServeHTTP(r, httptest.NewRequest("POST", "/api/raw/v1/collections", strings.NewReader(strings.Repeat(" ", raw.MaxBodyBytes+1))))
+ if r.Code != 413 || s.calls != 0 {
+ t.Fatalf("limit %d calls %d", r.Code, s.calls)
+ }
+ s.err = errors.New("postgres://secret SQL payload")
+ r = httptest.NewRecorder()
+ e.ServeHTTP(r, httptest.NewRequest("POST", "/api/raw/v1/collections", strings.NewReader(`{}`)))
+ if r.Code != 500 || strings.Contains(r.Body.String(), "secret") {
+ t.Fatal(r.Code, r.Body)
+ }
+}
+
+func TestRawHTTPAuthAndReadiness(t *testing.T) {
+ old := vectorEngine
+ defer func() { vectorEngine = old }()
+ for _, backend := range []string{"postgres", "chromem"} {
+ vectorEngine = backend
+ for _, keys := range [][]string{nil, {"key"}} {
+ // Avoid legacy discovery against an external service: register with chromem
+ // and an empty fixture directory, then restore the raw backend.
+ vectorEngine = "chromem"
+ e := echo.New()
+ registerAPIRoutes(e, nil, 400, 0, keys)
+ vectorEngine = backend
+ registerRawAPIRoutes(e, nil)
+ if len(keys) == 0 {
+ assertLegacyEnvelope(t, e)
+ }
+ for _, path := range []string{"/api/raw/v1/capabilities", "/api/raw/v1/collections"} {
+ r := httptest.NewRecorder()
+ q := httptest.NewRequest("GET", path, nil)
+ e.ServeHTTP(r, q)
+ if len(keys) > 0 {
+ if r.Code != 401 {
+ t.Fatal(path, r.Code)
+ }
+ continue
+ }
+ if path == "/api/raw/v1/collections" {
+ want := 501
+ if backend == "postgres" {
+ want = 503
+ }
+ if r.Code != want {
+ t.Fatal(r.Code, r.Body)
+ }
+ } else if path == "/api/raw/v1/capabilities" && r.Code != 200 {
+ t.Fatal(r.Code)
+ }
+ }
+ if len(keys) > 0 {
+ r := httptest.NewRecorder()
+ q := httptest.NewRequest("GET", "/api/raw/v1/capabilities", nil)
+ q.Header.Set("Authorization", "Bearer key")
+ e.ServeHTTP(r, q)
+ if r.Code != 200 {
+ t.Fatal(r.Code)
+ }
+ }
+ }
+ }
+}
+
+func TestRawAPILifecycle(t *testing.T) {
+ if os.Getenv("LOCALRECALL_TEST_DATABASE_OWNER") == "" {
+ t.Fatal("owned database fixture required")
+ }
+ var calls atomic.Int64
+ trap := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(500) }))
+ defer trap.Close()
+ t.Setenv("OPENAI_BASE_URL", trap.URL)
+ t.Setenv("LOCALAI_ENDPOINT", trap.URL)
+ assetTrap := t.TempDir() + "/not-a-directory"
+ if err := os.WriteFile(assetTrap, []byte("trap"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("FILE_ASSETS", assetTrap)
+ t.Setenv("COLLECTION_DB_PATH", assetTrap)
+ ctx := context.Background()
+ cfg := rawpostgres.Config{DatabaseURL: os.Getenv("LOCALRECALL_TEST_DATABASE_URL")}
+ service, err := rawpostgres.New(ctx, cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer service.Close()
+ oldBackend, oldPath := vectorEngine, collectionDBPath
+ vectorEngine, collectionDBPath = "chromem", t.TempDir()
+ defer func() { vectorEngine, collectionDBPath = oldBackend, oldPath }()
+ e := echo.New()
+ registerAPIRoutes(e, nil, 400, 0, nil)
+ registerRawAPIRoutes(e, service)
+ assertLegacyEnvelope(t, e)
+ server := httptest.NewServer(e)
+ defer server.Close()
+ c := client.NewRawClient(server.URL, "", server.Client())
+ caps, err := c.Capabilities(ctx)
+ if err != nil || !caps.Ready {
+ t.Fatal(caps, err)
+ }
+ req := raw.CreateCollectionRequest{Name: "api-owned", Dimensions: 3, Metric: "cosine", EmbeddingSpaceID: "space"}
+ d, created, err := c.CreateCollection(ctx, req)
+ if err != nil || !created {
+ t.Fatal(d, created, err)
+ }
+ _, created, err = c.CreateCollection(ctx, req)
+ if err != nil || created {
+ t.Fatal(created, err)
+ }
+ bad := req
+ bad.Dimensions = 4
+ _, _, err = c.CreateCollection(ctx, bad)
+ assertRawStatus(t, err, 409)
+ if _, err = c.GetCollection(ctx, d.CollectionID); err != nil {
+ t.Fatal(err)
+ }
+ page, err := c.ListCollections(ctx, raw.ListCollectionsRequest{})
+ if err != nil || len(page.Collections) == 0 {
+ t.Fatal(page, err)
+ }
+ assertLiveIsolation := func() {
+ names, err := engine.ListPostgresCollections(ctx, cfg.DatabaseURL)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, name := range names {
+ if name == req.Name {
+ t.Fatal("raw leaked into legacy discovery")
+ }
+ }
+ }
+ assertLiveIsolation()
+ text := "bicycle"
+ _, err = c.Upsert(ctx, d.CollectionID, raw.UpsertRequest{EmbeddingSpaceID: "space", Records: []raw.Record{{ID: "segment", Vector: []float64{1, 0, 0}, Text: &text, Metadata: map[string]string{"source": trap.URL}}}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ got, err := c.GetRecords(ctx, d.CollectionID, raw.GetRecordsRequest{IDs: []string{"segment", "missing"}, IncludeVectors: true})
+ if err != nil || len(got.Records) != 1 || len(got.MissingIDs) != 1 {
+ t.Fatal(got, err)
+ }
+ records, err := c.ListRecords(ctx, d.CollectionID, raw.ListRecordsRequest{})
+ if err != nil || len(records.Records) != 1 {
+ t.Fatal(records, err)
+ }
+ space := "space"
+ exact := "exact"
+ for _, mode := range []string{"vector", "lexical", "hybrid"} {
+ q := raw.SearchRequest{Mode: mode}
+ if mode != "lexical" {
+ q.Vector = []float64{1, 0, 0}
+ q.EmbeddingSpaceID = &space
+ q.Accuracy = &exact
+ }
+ if mode != "vector" {
+ q.Text = &text
+ }
+ result, err := c.Search(ctx, d.CollectionID, q)
+ if err != nil || result.Count != 1 {
+ t.Fatal(result, err)
+ }
+ }
+ service.Close()
+ service, err = rawpostgres.New(ctx, cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer service.Close()
+ reopened := echo.New()
+ registerAPIRoutes(reopened, nil, 400, 0, nil)
+ registerRawAPIRoutes(reopened, service)
+ srv := httptest.NewServer(reopened)
+ defer srv.Close()
+ c = client.NewRawClient(srv.URL, "", srv.Client())
+ if _, err = c.GetCollection(ctx, d.CollectionID); err != nil {
+ t.Fatal(err)
+ }
+ assertLiveIsolation()
+ assertLegacyEnvelope(t, reopened)
+ del, err := c.DeleteRecords(ctx, d.CollectionID, raw.DeleteRecordsRequest{IDs: []string{"segment"}})
+ if err != nil || del.DeletedCount != 1 {
+ t.Fatal(del, err)
+ }
+ if err = c.Reset(ctx, d.CollectionID); err != nil {
+ t.Fatal(err)
+ }
+ if err = c.DeleteCollection(ctx, d.CollectionID); err != nil {
+ t.Fatal(err)
+ }
+ _, err = c.GetCollection(ctx, d.CollectionID)
+ assertRawStatus(t, err, 404)
+ conn, err := pgx.Connect(ctx, cfg.DatabaseURL)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer conn.Close(ctx)
+ if _, err = conn.Exec(ctx, "UPDATE lr_raw_schema SET version=999"); err != nil {
+ t.Fatal(err)
+ }
+ failed, err := rawpostgres.New(ctx, cfg)
+ if err == nil {
+ failed.Close()
+ t.Fatal("unknown schema accepted")
+ }
+ if _, err = conn.Exec(ctx, "UPDATE lr_raw_schema SET version=1"); err != nil {
+ t.Fatal(err)
+ }
+ if calls.Load() != 0 {
+ t.Fatalf("embedding/source calls %d", calls.Load())
+ }
+}
+func assertRawStatus(t *testing.T, err error, status int) {
+ t.Helper()
+ var re *raw.Error
+ if !errors.As(err, &re) || re.Status != status {
+ t.Fatalf("wanted %d: %v", status, err)
+ }
+}
+
+type rawBrokenBody struct{ err error }
+
+func (b rawBrokenBody) Read([]byte) (int, error) { return 0, b.err }
+func (b rawBrokenBody) Close() error { return nil }
+func TestRawHTTPErrors(t *testing.T) {
+ s := &rawSpy{}
+ e := echo.New()
+ registerRawAPIRoutes(e, s)
+ for _, code := range []string{raw.CodeValidation, raw.CodeNotFound, raw.CodeConflict, raw.CodeSpaceMismatch, raw.CodeBodyLimit, raw.CodeDeleteBound, raw.CodeUnsupported, raw.CodeUnavailable, raw.CodeTimeout, raw.CodeInternal} {
+ s.err = raw.NewError(code)
+ r := httptest.NewRecorder()
+ e.ServeHTTP(r, httptest.NewRequest("POST", "/api/raw/v1/collections", strings.NewReader(`{}`)))
+ if r.Code != raw.NewError(code).Status {
+ t.Fatal(code, r.Code)
+ }
+ }
+ s.err = context.Canceled
+ r := httptest.NewRecorder()
+ e.ServeHTTP(r, httptest.NewRequest("POST", "/api/raw/v1/collections", strings.NewReader(`{}`)))
+ if r.Code != 504 {
+ t.Fatal(r.Code)
+ }
+ before := s.calls
+ q := httptest.NewRequest("POST", "/api/raw/v1/collections", nil)
+ q.Body = rawBrokenBody{errors.New("read secret")}
+ r = httptest.NewRecorder()
+ e.ServeHTTP(r, q)
+ if r.Code != 400 || s.calls != before {
+ t.Fatal(r.Code, s.calls)
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ q = httptest.NewRequest("POST", "/api/raw/v1/collections", strings.NewReader(`{}`)).WithContext(ctx)
+ r = httptest.NewRecorder()
+ e.ServeHTTP(r, q)
+ if r.Code != 504 || s.calls != before {
+ t.Fatal(r.Code, s.calls)
+ }
+ for _, body := range []string{`{"records":[{"metadata":{"a":{"nested":"no"}}}]}`, `{"records":[{"metadata":{"a":"one","a":"two"}}]}`, `{"records":null}`, `{"records":[{"metadata":{"a":"\u0000"}}]}`} {
+ r = httptest.NewRecorder()
+ e.ServeHTTP(r, httptest.NewRequest("POST", "/api/raw/v1/collections/cid/records/upsert", strings.NewReader(body)))
+ if r.Code != 400 {
+ t.Fatal(r.Code, r.Body)
+ }
+ }
+}
+
+// A context deadline alone cannot unblock net/http's incoming Body.Read.
+func TestRawStreamingBodyCancellation(t *testing.T) {
+ for _, method := range []string{"POST", "DELETE"} {
+ for _, early := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/cancel=%v", method, early), func(t *testing.T) {
+ s := &rawSpy{}
+ e := echo.New()
+ registerRawAPIRoutes(e, s)
+ ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond)
+ if early {
+ cancel()
+ ctx, cancel = context.WithTimeout(context.Background(), time.Hour)
+ }
+ defer cancel()
+ entered, done := make(chan struct{}), make(chan struct{})
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ close(entered)
+ e.ServeHTTP(w, r.WithContext(ctx))
+ close(done)
+ }))
+ defer server.Close()
+ conn, err := net.Dial("tcp", strings.TrimPrefix(server.URL, "http://"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer conn.Close()
+ path := "/api/raw/v1/collections"
+ if method == "DELETE" {
+ path += "/00000000-0000-0000-0000-000000000000"
+ }
+ fmt.Fprintf(conn, "%s %s HTTP/1.1\r\nHost: fixture\r\nContent-Length: 100\r\n\r\n{", method, path)
+ <-entered
+ if early {
+ cancel()
+ }
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ conn.Close() // unblock and clean up even against the broken implementation
+ <-done
+ t.Fatal("handler remained blocked reading body after cancellation")
+ }
+ if s.calls != 0 {
+ t.Fatalf("store calls: %d", s.calls)
+ }
+ conn.SetReadDeadline(time.Now().Add(time.Second))
+ response, err := http.ReadResponse(bufio.NewReader(conn), nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer response.Body.Close()
+ b, _ := io.ReadAll(response.Body)
+ if response.StatusCode != 504 || !strings.Contains(string(b), `"code":"TIMEOUT"`) {
+ t.Fatalf("%d %s", response.StatusCode, b)
+ }
+ })
+ }
+ }
+}
+
+func TestRawServerBindFailureExit(t *testing.T) {
+ if os.Getenv("RAW_BIND_CHILD") == "1" {
+ vectorEngine = "chromem"
+ listeningAddress = "127.0.0.1:not-a-port"
+ main()
+ return
+ }
+ cmd := exec.Command(os.Args[0], "-test.run=^TestRawServerBindFailureExit$")
+ cmd.Env = append(os.Environ(), "RAW_BIND_CHILD=1")
+ output, err := cmd.CombinedOutput()
+ if err == nil {
+ t.Fatalf("bind failure exited successfully: %s", output)
+ }
+ var exit *exec.ExitError
+ if !errors.As(err, &exit) || exit.ExitCode() != 1 {
+ t.Fatalf("unexpected exit: %v %s", err, output)
+ }
+}
+
+func assertLegacyEnvelope(t *testing.T, e *echo.Echo) {
+ t.Helper()
+ old := vectorEngine
+ vectorEngine = "chromem"
+ defer func() { vectorEngine = old }()
+ for _, tc := range []struct {
+ method, path string
+ status int
+ body string
+ }{
+ {"GET", "/api/collections", 200, `{"success":true,"message":"Collections retrieved successfully","data":{"collections":null,"count":0}}`},
+ {"POST", "/api/collections/absent/reset", 404, `{"success":false,"error":{"code":"NOT_FOUND","message":"Collection not found","details":"Collection 'absent' does not exist"}}`},
+ } {
+ r := httptest.NewRecorder()
+ e.ServeHTTP(r, httptest.NewRequest(tc.method, tc.path, nil))
+ if r.Code != tc.status || strings.TrimSpace(r.Body.String()) != tc.body {
+ t.Fatalf("legacy %s: %d %s", tc.path, r.Code, r.Body)
+ }
+ }
+}
+
+type rawCloseSpy struct {
+ raw.Store
+ closed bool
+}
+
+func (s *rawCloseSpy) Close() { s.closed = true }
+func TestRawServerCleanup(t *testing.T) {
+ for _, fail := range []bool{false, true} {
+ e := echo.New()
+ e.HideBanner = true
+ store := &rawCloseSpy{}
+ stop, cancel := context.WithCancel(context.Background())
+ address := "127.0.0.1:not-a-port"
+ if !fail {
+ listener, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ e.Listener = listener
+ cancel() // injectable signal context, never signal the shared test process
+ }
+ err := serveAPI(e, address, store, stop)
+ cancel()
+ if (err != nil) != fail || !store.closed {
+ t.Fatalf("fail=%v error=%v closed=%v", fail, err, store.closed)
+ }
+ }
+}
+
+func (s *rawSpy) DeleteCollection(context.Context, string) error { s.calls++; return s.err }
+
+func TestRawDeadlineDoesNotLimitLegacyBody(t *testing.T) {
+ e := echo.New()
+ registerRawAPIRoutes(e, &rawSpy{})
+ e.POST("/legacy-stream", func(c echo.Context) error {
+ b, err := io.ReadAll(c.Request().Body)
+ if err != nil {
+ return err
+ }
+ return c.String(200, string(b))
+ })
+ server := httptest.NewServer(e)
+ defer server.Close()
+ conn, err := net.Dial("tcp", strings.TrimPrefix(server.URL, "http://"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer conn.Close()
+ // First exercise successful raw middleware on this same connection.
+ fmt.Fprint(conn, "POST /api/raw/v1/collections HTTP/1.1\r\nHost: fixture\r\nContent-Length: 2\r\n\r\n{}")
+ reader := bufio.NewReader(conn)
+ response, err := http.ReadResponse(reader, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ io.Copy(io.Discard, response.Body)
+ response.Body.Close()
+ fmt.Fprint(conn, "POST /legacy-stream HTTP/1.1\r\nHost: fixture\r\nContent-Length: 2\r\n\r\na")
+ time.Sleep(200 * time.Millisecond)
+ fmt.Fprint(conn, "b")
+ conn.SetReadDeadline(time.Now().Add(time.Second))
+ response, err = http.ReadResponse(reader, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer response.Body.Close()
+ b, _ := io.ReadAll(response.Body)
+ if response.StatusCode != 200 || string(b) != "ab" {
+ t.Fatalf("%d %s", response.StatusCode, b)
+ }
+ if e.Server.ReadTimeout != 0 {
+ t.Fatal("global read timeout changed")
+ }
+}
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
index ade88b4..b6b32fa 100644
--- a/test/e2e/e2e_test.go
+++ b/test/e2e/e2e_test.go
@@ -115,9 +115,11 @@ var _ = Describe("API", func() {
Expect(err).To(BeNil())
docs, err := localRecall.Search(TestCollection, "spiders", 1)
- Expect(err).To(HaveOccurred())
- Expect(docs).To(BeNil())
- Expect(err.Error()).To(ContainSubstring("failed to search collection"))
+ Expect(err).ToNot(HaveOccurred())
+ Expect(docs).To(BeEmpty())
+ entries, err := localRecall.ListEntries(TestCollection)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(entries).To(BeEmpty())
})
It("should be able to delete documents", func() {
diff --git a/test/e2e/raw_legacy_owned_test.go b/test/e2e/raw_legacy_owned_test.go
new file mode 100644
index 0000000..94b1e1a
--- /dev/null
+++ b/test/e2e/raw_legacy_owned_test.go
@@ -0,0 +1,110 @@
+//go:build ownedintegration
+
+package e2e_test
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/mudler/localrecall/pkg/client"
+)
+
+// This release fixture uses the real server, database and embedding provider.
+// Source content is private and deterministic; no public website is fetched.
+func TestOwnedLegacyLifecycle(t *testing.T) {
+ base := os.Getenv("LOCALRECALL_ENDPOINT")
+ if base == "" || os.Getenv("LOCALRECALL_TEST_DATABASE_OWNER") == "" {
+ t.Fatal("owned legacy runner required")
+ }
+ c := client.NewClient(base)
+ name := fmt.Sprintf("owned-%d", time.Now().UnixNano())
+ if err := c.CreateCollection(name); err != nil {
+ t.Fatal(err)
+ }
+ text := "The observatory telescope studies distant galaxies and bright stars."
+ file := filepath.Join(t.TempDir(), "astronomy.txt")
+ if err := os.WriteFile(file, []byte(text), 0600); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := c.Store(name, file); err != nil {
+ t.Fatal(err)
+ }
+ entries, err := c.ListEntries(name)
+ if err != nil || len(entries) != 1 {
+ t.Fatalf("entries %v %v", entries, err)
+ }
+ chunks, err := c.GetEntryContent(name, entries[0])
+ if err != nil || len(chunks) == 0 || !strings.Contains(chunks[0].Content, "galaxies") {
+ t.Fatalf("content %v %v", chunks, err)
+ }
+ results, err := c.Search(name, "telescope galaxies", 3)
+ if err != nil || len(results) == 0 || !strings.Contains(results[0].Content, "galaxies") {
+ t.Fatalf("search %v %v", results, err)
+ }
+ if _, err := c.DeleteEntry(name, entries[0]); err != nil {
+ t.Fatal(err)
+ }
+ src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprint(w, "Private source astronomy observatory telescope galaxies.")
+ }))
+ defer src.Close()
+ request := func(method string, body any) []byte {
+ t.Helper()
+ data, _ := json.Marshal(body)
+ req, err := http.NewRequest(method, base+"/api/collections/"+name+"/sources", bytes.NewReader(data))
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+ resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer resp.Body.Close()
+ b, err := io.ReadAll(resp.Body)
+ if err != nil || resp.StatusCode != 200 {
+ t.Fatalf("source %s: %d %s %v", method, resp.StatusCode, b, err)
+ }
+ return b
+ }
+ request("POST", map[string]any{"url": src.URL, "update_interval": 60})
+ if !bytes.Contains(request("GET", nil), []byte(src.URL)) {
+ t.Fatal("source not listed")
+ }
+ deadline := time.Now().Add(time.Minute)
+ for {
+ entries, err = c.ListEntries(name)
+ if err == nil && len(entries) == 1 {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("source never indexed: %v %v", entries, err)
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ chunks, err = c.GetEntryContent(name, entries[0])
+ if err != nil || len(chunks) == 0 || !strings.Contains(chunks[0].Content, "Private source") {
+ t.Fatalf("source content %v %v", chunks, err)
+ }
+ request("DELETE", map[string]string{"url": src.URL})
+ if bytes.Contains(request("GET", nil), []byte(src.URL)) {
+ t.Fatal("source not removed")
+ }
+ if err = c.Reset(name); err != nil {
+ t.Fatal(err)
+ }
+ entries, err = c.ListEntries(name)
+ if err != nil || len(entries) != 0 {
+ t.Fatalf("reset %v %v", entries, err)
+ }
+ t.Log("real text upload/search/entry/delete/reset and private HTTP source fetch/list/remove passed")
+}
diff --git a/test/e2e/source_manager_test.go b/test/e2e/source_manager_test.go
index 0fb8f88..0783301 100644
--- a/test/e2e/source_manager_test.go
+++ b/test/e2e/source_manager_test.go
@@ -2,6 +2,8 @@ package e2e_test
import (
"fmt"
+ "net/http"
+ "net/http/httptest"
"os"
"path/filepath"
"time"
@@ -250,47 +252,37 @@ var _ = Describe("SourceManager", func() {
})
It("should prevent duplicate content with frequent updates", func() {
- // Add a source with a very short update interval
- err := sourceManager.AddSource(TestCollection, "https://en.wikipedia.org/wiki/Black-crowned_barwing", 1*time.Second)
- Expect(err).To(BeNil())
-
- // Start the background service
+ // Fixed, sub-chunk-size content: public Wikipedia revisions changed the
+ // old expected count. The real fetch/parser/embed/store path stays intact.
+ fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprint(w, "The Black-crowned barwing is a bird of mountain forests. It feeds on insects and fruit.")
+ }))
+ defer fixture.Close()
+ Expect(sourceManager.AddSource(TestCollection, fixture.URL, time.Second)).To(Succeed())
sourceManager.Start()
- // Wait for initial content to be fetched
- Eventually(func() []string {
- return kb.ListDocuments()
- }, 2*time.Minute, 5*time.Second).Should(HaveLen(1))
-
- // Wait for initial content to be fetched
- Eventually(func() int {
- return kb.Count()
- }, 2*time.Minute, 5*time.Second).Should(Equal(25))
-
- // Let it run for 2 minutes and check for duplicates
- Consistently(func() int {
- return kb.Count()
- }, 3*time.Minute, 5*time.Second).Should(Equal(25))
-
- // Verify that search results don't contain duplicates
- Consistently(func() bool {
- results, err := kb.Search("What is the Black-crowned barwing?", 5)
- if err != nil {
- fmt.Println("Error searching for content", err)
- return false
- }
-
- // Check for duplicate content
- seen := make(map[string]bool)
- for _, r := range results {
- if seen[r.Content] {
- fmt.Println("Found a duplicate", r.Content)
- return false // Found a duplicate
+ // StoreOrReplace publishes a fresh UUID document key under the KB lock
+ // only after replacing chunks. A changed key proves a completed update;
+ // LastUpdate is set BEFORE fetching and cannot prove successful storage.
+ previous := ""
+ for cycle := 0; cycle < 3; cycle++ {
+ Eventually(func() string {
+ docs := kb.ListDocuments()
+ if len(docs) != 1 || docs[0] == previous {
+ return ""
}
- seen[r.Content] = true
- }
- return true
- }, 2*time.Minute, 5*time.Second).Should(BeTrue())
+ return docs[0]
+ }, 90*time.Second, TestPollingInterval).ShouldNot(BeEmpty())
+ docs := kb.ListDocuments()
+ Expect(docs).To(HaveLen(1))
+ previous = docs[0]
+ Expect(kb.Count()).To(Equal(1), "chunk count after completed update %d", cycle+1)
+ results, err := kb.Search("What is the Black-crowned barwing?", 1)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(results).To(HaveLen(1))
+ Expect(results[0].Content).To(ContainSubstring("Black-crowned barwing"))
+ }
+
})
})
diff --git a/test/integration/postgres_integration_test.go b/test/integration/postgres_integration_test.go
index 112226c..72f5af6 100644
--- a/test/integration/postgres_integration_test.go
+++ b/test/integration/postgres_integration_test.go
@@ -58,7 +58,10 @@ var _ = Describe("PostgreSQL Integration", func() {
openaiClient = openai.NewClientWithConfig(config)
// Use PostgreSQL from docker-compose
- databaseURL = "postgresql://localrecall:localrecall@localhost:5432/localrecall?sslmode=disable"
+ databaseURL = os.Getenv("LOCALRECALL_TEST_DATABASE_URL")
+ if databaseURL == "" {
+ databaseURL = "postgresql://localrecall:localrecall@localhost:5432/localrecall?sslmode=disable"
+ }
// Verify PostgreSQL is accessible
ctx := context.Background()
diff --git a/test/raw/CI.md b/test/raw/CI.md
new file mode 100644
index 0000000..73dcda0
--- /dev/null
+++ b/test/raw/CI.md
@@ -0,0 +1,30 @@
+# CI fixture selection
+
+`make test-unit` builds `test/raw/Dockerfile.postgres` and executes all `rag`,
+`pkg`, and root tests via `unit.sh`, giving each package a fresh `database.sh` fixture. This includes every raw PostgreSQL
+test (outage/recovery, shutdown, process discovery, search and recall), plus raw
+HTTP tests. Packages run serially (`-p=1`): outage tests intentionally stop their
+owned database and must not interrupt another package. Pristine per-package
+databases also keep legacy BM25 objects and registries out of raw bootstrap tests. No database-dependent
+raw test is skipped or excluded; absent dependencies fail the job.
+
+`make run-e2e` runs the conventional E2E suite and then creates a separate owned
+database and host API for `TestOwnedLegacyLifecycle` via
+`legacy.sh --owned --e2e-only`. That test is classified with `ownedintegration`
+because it must never target the conventional/shared server. The CI invocation
+uses the already warmed real embedding provider and configured embedding model;
+the private HTTP source is reachable by the owned host API. `legacy.sh` enables
+the tag explicitly and checks for a PASS (missing tests/skips fail).
+
+`sh test/raw/verify.sh all` still executes its complete raw, legacy and actual
+LocalAI consumer gates; its existing provisioned native-provider prerequisites
+are unchanged. `legacy.sh` without `--e2e-only` still runs all legacy suites.
+
+On shared development hosts, do not run the compose-based Make targets. Use
+`database.sh` for a disposable PostgreSQL fixture and the documented private
+LocalAI runner for genuine embeddings. CI compose runs are on isolated runners.
+
+The fixture exports both `LOCALRECALL_TEST_DATABASE_URL` and
+`POSTGRES_TEST_URL`, so legacy planning tests never fall back to localhost.
+Race invocations in `legacy.sh` explicitly enable CGO, overriding the Makefile
+default only for those commands.
diff --git a/test/raw/Dockerfile.postgres b/test/raw/Dockerfile.postgres
new file mode 100644
index 0000000..69b1db5
--- /dev/null
+++ b/test/raw/Dockerfile.postgres
@@ -0,0 +1,4 @@
+FROM postgres:18-bookworm
+RUN apt-get update && apt-get install -y --no-install-recommends build-essential git ca-certificates postgresql-server-dev-18 postgresql-18-pgvector && rm -rf /var/lib/apt/lists/*
+RUN git clone --depth 1 --branch v1.3.1 https://github.com/timescale/pg_textsearch /tmp/pg_textsearch && make -C /tmp/pg_textsearch && make -C /tmp/pg_textsearch install && rm -rf /tmp/pg_textsearch
+CMD ["postgres", "-c", "shared_preload_libraries=pg_textsearch"]
diff --git a/test/raw/LOCALAI-SMOKE.md b/test/raw/LOCALAI-SMOKE.md
new file mode 100644
index 0000000..16f1a48
--- /dev/null
+++ b/test/raw/LOCALAI-SMOKE.md
@@ -0,0 +1,78 @@
+# Actual LocalAI consumer smoke
+
+Run from the LocalRecall worktree root:
+
+```sh
+LOCALAI_SMOKE_BACKEND=/absolute/path/to/llama.cpp/grpc-server \
+LOCALAI_SMOKE_LIBS=/absolute/path/to/native/libraries \
+sh test/raw/localai-smoke.sh > .sdd-work/logs/localai-real.log 2>&1
+```
+
+This is a required-dependency test: missing inputs fail, never skip. It requires
+Go matching the pinned consumer, Python 3, Docker, curl for provisioning, and
+approximately 1 GiB of additional headroom with the existing Go caches populated.
+A clean Go cache needs substantially more. Do not blindly pull large images.
+
+## Provisioning
+
+The runner intentionally does not download native dependencies or images:
+
+* Download `https://codeload.github.com/mudler/LocalAI/tar.gz/d4a5af8bfed9c97e8cb01e584f1f9f8a572df844`
+ into `.sdd-work/localai.tar.gz`; extract with `--strip-components=1` into
+ `.sdd-work/localai`. Read its `AGENTS.md` and build instructions. Generate Go
+ protobuf files with `make protogen-go`, using private GOPATH/GOCACHE/GOMODCACHE
+ and TMPDIR under `.sdd-work`. The runner verifies every archive file against
+ the extracted tree, while allowing extra generated files and the smoke fixture.
+* Download `https://huggingface.co/gaianet/All-MiniLM-L6-v2-Embedding-GGUF/resolve/main/all-MiniLM-L6-v2-ggml-model-f16.gguf`
+ into `.sdd-work/model-smoke/minilm.gguf` (44 MiB). The runner requires SHA-256
+ `797b70c4edf85907fe0a49eb85811256f65fa0f7bf52166b147fd16be2be4662`:
+ upstream `main` is mutable, but accepted file content is pinned.
+* Have cached provider image
+ `sha256:f1bc435659b9da49180e868fdd0361915171a740cd947bf081b62a896a7ea8d0`
+ and the PostgreSQL image required by `database.sh`. No running host service is used.
+* Supply a genuine llama.cpp gRPC backend and its native library directory.
+ These are used read-only; backend cwd is the private model directory.
+ The successful local run used `.sdd-bridge-repro/grpc-server` and
+ `.sdd-deps/root/usr/lib/x86_64-linux-gnu` from the existing
+ `mudler-LocalAI/llama-decisions` workspace. Backend SHA-256 was
+ `ce81165dfa8459b3b450402e54c69fd476c2b1ad17d883a8fa85eea84df5fc78`.
+ Its source revision is not attested by this smoke; binary identity is logged.
+
+## What it proves
+
+The provider binary is copied from the cached image into `.sdd-work`, with a
+verified binary digest. Its version is v4.11.0 / source label
+`58830f7ac508845a6f4efa32cfca06af422d4d82`, **not** the consumer revision.
+A private loopback provider and native backend are started and stopped by the
+runner. An actual `/v1/embeddings` request must return 384 finite, nonzero MiniLM
+values before testing. Responses and native/provider logs stay in `.sdd-work/logs`.
+
+The generated test lives in the pinned real `core/services/agentpool` package.
+It calls `NewAgentPoolService`, `buildCollectionsConfig`, the production embedded
+`collections.NewInProcessBackend`, and `shareCollections`, matching the collection
+portion of `startDistributed`. It then exercises the real ForUser upload, list,
+entry-content, search and reset methods. No fake engine, HTTP store adapter or
+standalone reimplementation substitutes for the consumer. Unrelated schedulers,
+skills and NATS are deliberately not started. An alternate modfile replaces
+LocalRecall with this worktree; neither source go.mod/go.sum is edited.
+
+The successful run used PostgreSQL HNSW fallback (vectorscale was unavailable).
+It verifies text embedding compatibility, not EmbeddingGemma2 or multimodal
+quality. The full runner additionally requires concurrent raw zero-embedding accounting,
+legacy suites, cross-process/restart regressions and shutdown cleanup.
+
+## Evidence
+
+Initial run failed before inference because the provider flag was incorrectly
+spelled `--disable-webui`; provider help identified `--disable-web-ui`. After
+that one-line correction, `.sdd-work/logs/localai-real-attempt3.log` reports
+`TestLocalRecallRealConsumer` PASS and owned database removal. The earlier
+compilation-only evidence is `localai-build-generated-attempt1.log`; it is not
+counted as a runtime test.
+
+Attempt5 full acceptance (`.sdd-work/logs/full-attempt5.log`) passed these gates.
+The synchronized consumer test held one embedding request open during raw work:
+28.13 ms overlap, exactly one legacy call, zero extra raw calls, three raw and two
+legacy collections. It sampled three combined database sessions against a ceiling
+of 12, excluding its observer. Catalog objects changed 8 → 30 → 18, with 12
+removed on deletion. See `docs/raw-vector-api.md` for measurement limits.
diff --git a/test/raw/confinement.py b/test/raw/confinement.py
new file mode 100644
index 0000000..35dab25
--- /dev/null
+++ b/test/raw/confinement.py
@@ -0,0 +1,25 @@
+"""Validate output trees before writes or cleanup; native inputs are read-only."""
+import os
+import pathlib
+import sys
+
+root = pathlib.Path(sys.argv[1]).resolve()
+work = root / '.sdd-work'
+try:
+ if work.is_symlink():
+ raise ValueError(str(work))
+ # Check canonical destinations, including dangling links and alternate modfiles.
+ for name in ('tmp', 'cache', 'mod', 'gopath', 'collections', 'assets', 'logs',
+ 'docker', 'legacy', 'model-smoke', 'localai', 'localai-smoke.mod', 'localai-smoke.sum',
+ 'localai.tar.gz'):
+ top = work / name
+ paths = [top]
+ if top.is_dir() and not top.is_symlink():
+ for directory, dirs, files in os.walk(top, followlinks=False):
+ paths.extend(pathlib.Path(directory) / n for n in dirs + files
+ if (pathlib.Path(directory) / n).is_symlink())
+ for path in paths:
+ if not path.resolve().is_relative_to(work):
+ raise ValueError(str(path))
+except (ValueError, OSError, RuntimeError) as error:
+ sys.exit('unsafe output: ' + str(error))
diff --git a/test/raw/confinement_test.py b/test/raw/confinement_test.py
new file mode 100644
index 0000000..5f5ff30
--- /dev/null
+++ b/test/raw/confinement_test.py
@@ -0,0 +1,45 @@
+"""Every public runner must reject escaping outputs before its first write."""
+import os
+import pathlib
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+
+class Confinement(unittest.TestCase):
+ def test_runner_entrypoints(self):
+ source = pathlib.Path('test/raw').resolve()
+ cases = ('model-smoke/backends', 'localai/core/services/agentpool',
+ 'localai-smoke.mod', 'logs/nested', 'docker/contexts',
+ 'cache/nested', 'mod/nested', 'gopath/pkg', 'tmp/nested',
+ 'collections/nested', 'assets/nested', 'legacy/owner/assets')
+ for runner in ('verify.sh', 'database.sh', 'localai-smoke.sh', 'legacy.sh'):
+ for nested in cases:
+ with self.subTest(runner=runner, nested=nested), tempfile.TemporaryDirectory(dir='.sdd-work/tmp') as tmp:
+ root = pathlib.Path(tmp).resolve()
+ shutil.copytree(source, root / 'test/raw')
+ (root / 'go.mod').write_text('module fixture\n')
+ (root / 'rag').mkdir()
+ work = root / '.sdd-work'
+ work.mkdir()
+ outside = root / 'outside'
+ outside.mkdir()
+ sentinel = outside / 'sentinel'
+ sentinel.write_text('unchanged')
+ link = work / nested
+ link.parent.mkdir(parents=True, exist_ok=True)
+ link.symlink_to(outside, target_is_directory=True)
+ before = sorted(str(p.relative_to(root)) for p in root.rglob('*'))
+ env = dict(os.environ)
+ env.pop('LOCALRECALL_TEST_DATABASE_URL', None)
+ result = subprocess.run(['sh', 'test/raw/' + runner], cwd=root,
+ env=env, capture_output=True, timeout=10)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn(b'unsafe output', result.stderr)
+ self.assertEqual(sentinel.read_text(), 'unchanged')
+ self.assertEqual(before, sorted(str(p.relative_to(root)) for p in root.rglob('*')))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/raw/database.sh b/test/raw/database.sh
new file mode 100644
index 0000000..253be2d
--- /dev/null
+++ b/test/raw/database.sh
@@ -0,0 +1,59 @@
+#!/bin/sh
+# Execute a command against a new, private fixture; never accept an external URL.
+set -eu
+[ -f test/raw/Dockerfile.postgres ] || { echo 'run from worktree root' >&2; exit 1; }
+[ -z "${LOCALRECALL_TEST_DATABASE_URL:-}" ] || { echo 'external database targets refused' >&2; exit 1; }
+ROOT=$(pwd -P)
+python3 test/raw/confinement.py "$ROOT"
+[ ! -L .sdd-work ] && [ ! -L .sdd-work/docker ] || exit 1
+mkdir -p .sdd-work/docker .sdd-work/tmp .sdd-work/logs
+export DOCKER_CONFIG="$ROOT/.sdd-work/docker" TMPDIR="$ROOT/.sdd-work/tmp"
+token=$(od -An -N12 -tx1 /dev/urandom | tr -d ' \n')
+name="lr-raw-$token"
+image="localrecall-raw-task2-a5975c8"
+cleanup() {
+ if [ "$(docker inspect -f '{{index .Config.Labels "localrecall.raw.owner"}}' "$name" 2>/dev/null || true)" = "$token" ]; then
+ docker rm -fv "$name" >/dev/null
+ fi
+ printf "Owned fixture %s removed\n" "$name"
+}
+trap cleanup EXIT HUP INT TERM
+docker image inspect "$image" >/dev/null
+# PG18 declares a volume; private tmpfs avoids persistent volumes. A shell PID 1
+# owns the server lifetime but remains alive after pg_ctl stop, preserving tmpfs
+# for the explicit recovery test. Never restart/stop an existing container.
+docker run -d --name "$name" --label "localrecall.raw.owner=$token" --tmpfs /var/lib/postgresql:rw -p 127.0.0.1::5432 -e POSTGRES_PASSWORD="$token" -e POSTGRES_DB=raw_fixture --entrypoint /bin/sh "$image" -c '
+ docker-entrypoint.sh postgres -c shared_preload_libraries=pg_textsearch &
+ server=$!
+ trap "kill -TERM $server 2>/dev/null || true; exit 0" TERM INT
+ wait "$server" || true
+ while :; do sleep 3600 & wait $!; done
+' >/dev/null
+n=0
+until docker exec "$name" pg_isready -U postgres -d raw_fixture >/dev/null 2>&1; do
+ n=$((n+1)); [ "$n" -lt 90 ] || { docker logs "$name"; exit 1; }; sleep 1
+done
+docker exec -i "$name" psql -v ON_ERROR_STOP=1 -U postgres -d raw_fixture <<'SQL'
+CREATE EXTENSION vector;
+CREATE EXTENSION pg_textsearch VERSION '1.3.1';
+CREATE TABLE fixture_probe (id text PRIMARY KEY, vector vector(3), text text);
+CREATE INDEX fixture_hnsw ON fixture_probe USING hnsw (vector vector_cosine_ops);
+CREATE INDEX fixture_bm25 ON fixture_probe USING bm25 (text) WITH (text_config='english');
+INSERT INTO fixture_probe VALUES ('one','[1,0,0]','fixture readiness'), ('two','[0,1,0]','other');
+SELECT vector <=> '[1,0,0]'::vector FROM fixture_probe ORDER BY vector <=> '[1,0,0]'::vector LIMIT 1;
+SELECT text <@> 'readiness' FROM fixture_probe ORDER BY text <@> 'readiness' LIMIT 1;
+DROP TABLE fixture_probe;
+SQL
+port=$(docker port "$name" 5432/tcp | sed -n 's/^127\.0\.0\.1://p')
+case "$port" in ''|*[!0-9]*) exit 1;; esac
+export LOCALRECALL_TEST_DATABASE_URL="postgres://postgres:$token@127.0.0.1:$port/raw_fixture?sslmode=disable"
+export POSTGRES_TEST_URL="$LOCALRECALL_TEST_DATABASE_URL"
+export LOCALRECALL_TEST_DATABASE_OWNER="$token"
+export LOCALRECALL_TEST_DATABASE_CONTAINER="$name"
+printf 'Owned fixture %s at loopback port %s (removed on exit)\n' "$name" "$port"
+if "$@"; then :; else
+ status=$?
+ docker logs "$name"
+ docker exec "$name" cat /tmp/raw-restart.log 2>/dev/null || true
+ exit "$status"
+fi
diff --git a/test/raw/downstream-lock.json b/test/raw/downstream-lock.json
new file mode 100644
index 0000000..b2b7228
--- /dev/null
+++ b/test/raw/downstream-lock.json
@@ -0,0 +1,15 @@
+{
+ "repository": "https://github.com/mudler/LocalAI",
+ "revision": "d4a5af8bfed9c97e8cb01e584f1f9f8a572df844",
+ "localrecall_required_version": "v0.6.7-0.20261006203612-d7c99211a3c7",
+ "localrecall_baseline": "d7c99211a3c7d6721a841fe34938e934f9d8aeee",
+ "consuming_packages": ["github.com/mudler/LocalAI/core/services/agentpool"],
+ "imports": {
+ "core/services/agentpool/collections_shared.go": ["github.com/mudler/localrecall/rag/engine"],
+ "core/services/agentpool/collections_shared_test.go": ["github.com/mudler/localrecall/rag", "github.com/mudler/localrecall/rag/sources"]
+ },
+ "production_calls": ["engine.PostgresCollectionExists", "engine.ListPostgresCollections"],
+ "test_calls": ["rag.NewSourceManager", "rag.PersistentKB", "sources.Config"],
+ "inspection": "Read-only git ls-remote HEAD and immutable GitHub go.mod/full source archive; all Go import call sites inspected. Actual LocalAI, not LocalAI-store. LocalAI also embeds collections through github.com/mudler/LocalAGI/webui/collections.",
+ "downstream_build_verified": false
+}
diff --git a/test/raw/legacy.sh b/test/raw/legacy.sh
new file mode 100644
index 0000000..cfcd697
--- /dev/null
+++ b/test/raw/legacy.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+# Never run legacy destructive suites against an externally selected service.
+set -eu
+python3 test/raw/confinement.py "$(pwd -P)"
+if [ "${1:-}" != --owned ]; then
+ [ "$#" -eq 0 ] || exit 2
+ [ -z "${LOCALRECALL_TEST_DATABASE_URL:-}" ] || exit 1
+ exec sh test/raw/localai-smoke.sh sh test/raw/database.sh sh test/raw/legacy.sh --owned
+fi
+: "${LOCALRECALL_TEST_DATABASE_OWNER:?owned database required}"
+: "${LOCALAI_ENDPOINT:?ready genuine model required}"
+export POSTGRES_TEST_URL="$LOCALRECALL_TEST_DATABASE_URL"
+export COLLECTION_DB_PATH="$PWD/.sdd-work/legacy/$LOCALRECALL_TEST_DATABASE_OWNER/collections" FILE_ASSETS="$PWD/.sdd-work/legacy/$LOCALRECALL_TEST_DATABASE_OWNER/assets"
+mkdir -p "$COLLECTION_DB_PATH" "$FILE_ASSETS"
+mkdir -p "$PWD/.sdd-work/logs"
+if [ "${2:-}" != --e2e-only ]; then
+log="$PWD/.sdd-work/logs/legacy.log"
+if CGO_ENABLED=1 go test -v -mod=readonly -race -count=1 -p=1 -timeout=15m ./rag ./rag/engine ./rag/sources ./pkg/chunk ./pkg/client . ./test/integration >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi
+# Ginkgo's summary includes '0 Skipped'; reject nonzero counts, too.
+! grep -E -- '--- SKIP:|[1-9][0-9]* Skipped|no tests to run|no test files' "$log"
+for suite in TestRag TestEngine TestChunk TestIntegration TestCreateCollectionReusesLoadedCollection; do
+ grep -q -- "--- PASS: $suite" "$log"
+done
+fi
+# Launch an owned real API on a dynamically allocated loopback port.
+go build -mod=readonly -o .sdd-work/legacy/localrecall .
+port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])')
+export LOCALRECALL_ENDPOINT="http://127.0.0.1:$port" LISTENING_ADDRESS="127.0.0.1:$port"
+export DATABASE_URL="$LOCALRECALL_TEST_DATABASE_URL" VECTOR_ENGINE=postgres EMBEDDING_MODEL="${EMBEDDING_MODEL:-minilm}" OPENAI_BASE_URL="$LOCALAI_ENDPOINT"
+.sdd-work/legacy/localrecall >.sdd-work/logs/legacy-server.log 2>&1 &
+pid=$!
+trap 'kill "$pid" 2>/dev/null || true; wait "$pid" 2>/dev/null || true' EXIT HUP INT TERM
+python3 - "$LOCALRECALL_ENDPOINT" "$pid" <<'PY'
+import urllib.request,time,sys,os
+for i in range(60):
+ os.kill(int(sys.argv[2]),0)
+ try:
+ with urllib.request.urlopen(sys.argv[1]+'/api/collections',timeout=2) as r: assert r.status==200
+ break
+ except Exception:
+ if i==59: raise
+ time.sleep(1)
+PY
+log="$PWD/.sdd-work/logs/legacy-e2e.log"
+if CGO_ENABLED=1 go test -tags=ownedintegration -v -mod=readonly -race -count=1 -timeout=5m ./test/e2e -run '^TestOwnedLegacyLifecycle$' >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi
+grep -q -- '--- PASS: TestOwnedLegacyLifecycle' "$log"
+! grep -E -- '--- SKIP:|no tests to run|no test files' "$log"
diff --git a/test/raw/legacy_contract_test.go b/test/raw/legacy_contract_test.go
new file mode 100644
index 0000000..f509ac1
--- /dev/null
+++ b/test/raw/legacy_contract_test.go
@@ -0,0 +1,69 @@
+package raw_test
+
+import (
+ "context"
+ "encoding/json"
+ "io"
+ "testing"
+
+ "github.com/mudler/localrecall/pkg/client"
+ "github.com/mudler/localrecall/rag"
+ "github.com/mudler/localrecall/rag/engine"
+ "github.com/mudler/localrecall/rag/engine/localai"
+ "github.com/mudler/localrecall/rag/types"
+ "github.com/sashabaranov/go-openai"
+)
+
+// Deliberately implements only the frozen legacy interface, without embedding it.
+type customEngine struct{}
+
+func (customEngine) Store(string, map[string]string) (engine.Result, error) {
+ return engine.Result{}, nil
+}
+func (customEngine) StoreDocuments([]string, map[string]string) ([]engine.Result, error) {
+ return nil, nil
+}
+func (customEngine) GetEmbeddingDimensions() (int, error) { return 3, nil }
+func (customEngine) Reset() error { return nil }
+func (customEngine) Search(string, int) ([]types.Result, error) { return nil, nil }
+func (customEngine) Count() int { return 0 }
+func (customEngine) Delete(map[string]string, map[string]string, ...string) error { return nil }
+func (customEngine) GetByID(string) (types.Result, error) { return types.Result{}, nil }
+func (customEngine) GetBySource(string) ([]types.Result, error) { return nil, nil }
+
+var (
+ _ rag.Engine = customEngine{}
+ _ func(string, string, rag.Engine, int, int, *openai.Client, string) (*rag.PersistentKB, error) = rag.NewPersistentCollectionKB
+ _ func(string, string, *openai.Client, string) (*engine.PostgresDB, error) = engine.NewPostgresDBCollection
+ _ func(string, string, *openai.Client, string) (*engine.ChromemDB, error) = engine.NewChromemDBCollection
+ _ func(*localai.StoreClient, *openai.Client, string) *engine.LocalAIRAGDB = engine.NewLocalAIRAGDB
+ _ func(string) *client.Client = client.NewClient
+ _ func(*client.Client, string) error = (*client.Client).CreateCollection
+ _ func(*client.Client) ([]string, error) = (*client.Client).ListCollections
+ _ func(*client.Client, string) ([]string, error) = (*client.Client).ListEntries
+ _ func(*client.Client, string, string) ([]client.EntryChunk, error) = (*client.Client).GetEntryContent
+ _ func(*client.Client, string, string) (io.ReadCloser, error) = (*client.Client).GetEntryRawFile
+ _ func(*client.Client, string, string) ([]string, error) = (*client.Client).DeleteEntry
+ _ func(*client.Client, string, string, int) ([]types.Result, error) = (*client.Client).Search
+ _ func(*client.Client, string) error = (*client.Client).Reset
+ _ func(*client.Client, string, string) (string, error) = (*client.Client).Store
+ _ func(context.Context, string, string) (bool, error) = engine.PostgresCollectionExists
+ _ func(context.Context, string) ([]string, error) = engine.ListPostgresCollections
+)
+
+func TestLegacyContract(t *testing.T) {
+ // Unkeyed literal freezes field order/types as well as names and JSON shape.
+ r := types.Result{"id", map[string]string{"source": "s"}, []float32{1}, "text", float32(.5)}
+ b, e := json.Marshal(r)
+ if e != nil {
+ t.Fatal(e)
+ }
+ if string(b) != `{"ID":"id","Metadata":{"source":"s"},"Embedding":[1],"Content":"text","Similarity":0.5}` {
+ t.Fatal(string(b))
+ }
+ var kb rag.PersistentKB
+ kb.Engine = customEngine{}
+ if kb.Engine.Count() != 0 {
+ t.Fatal("legacy engine")
+ }
+}
diff --git a/test/raw/localai-smoke.sh b/test/raw/localai-smoke.sh
new file mode 100644
index 0000000..672bcd3
--- /dev/null
+++ b/test/raw/localai-smoke.sh
@@ -0,0 +1,105 @@
+#!/bin/sh
+# Prerequisites and provider provenance: see test/raw/LOCALAI-SMOKE.md.
+set -eu
+ROOT=$(pwd -P)
+python3 test/raw/confinement.py "$ROOT"
+[ -f test/raw/downstream-lock.json ] || exit 1
+W="$ROOT/.sdd-work"
+[ ! -L "$W" ] || exit 1
+mkdir -p "$W/logs" "$W/tmp" "$W/model-smoke"
+export TMPDIR="$W/tmp" GOCACHE="$W/cache" GOMODCACHE="$W/mod" GOPATH="$W/gopath" GOENV=off GOTOOLCHAIN=local
+export DOCKER_CONFIG="$W/docker"
+: "${LOCALAI_SMOKE_BACKEND:?set path to genuine llama.cpp grpc-server}"
+: "${LOCALAI_SMOKE_LIBS:?set native library directory}"
+export LD_LIBRARY_PATH="$LOCALAI_SMOKE_LIBS${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+python3 - "$ROOT" <<'PY'
+import hashlib,json,pathlib,tarfile,sys
+r=pathlib.Path(sys.argv[1]); w=r/'.sdd-work'
+lock=json.loads((r/'test/raw/downstream-lock.json').read_text())
+assert hashlib.sha256((w/'localai.tar.gz').read_bytes()).hexdigest() == '99e86e821f0094a3176926cfc1508deb88d229b8ab1d292e52a361df08fdd92e', 'unexpected archive digest'
+assert hashlib.sha256((w/'model-smoke/minilm.gguf').read_bytes()).hexdigest() == '797b70c4edf85907fe0a49eb85811256f65fa0f7bf52166b147fd16be2be4662', 'unexpected model digest'
+with tarfile.open(w/'localai.tar.gz') as tar:
+ members=tar.getmembers(); prefix=members[0].name.split('/')[0]
+ assert prefix=='LocalAI-'+lock['revision'], prefix
+ n=0
+ for m in members:
+ rel=pathlib.PurePosixPath(m.name).relative_to(prefix)
+ p=w/'localai'/rel
+ if m.isfile():
+ assert p.is_file() and hashlib.sha256(p.read_bytes()).digest()==hashlib.sha256(tar.extractfile(m).read()).digest(), str(rel)
+ n+=1
+ elif m.issym(): assert p.is_symlink() and str(p.readlink())==m.linkname,str(rel)
+ print('Verified archive source files:',n,'consumer revision:',lock['revision'])
+PY
+# The archive must have been downloaded from the immutable codeload revision;
+# canonical digest recorded independently, never use the enclosing repo's HEAD.
+sha256sum "$W/localai.tar.gz" "$W/model-smoke/minilm.gguf" "$LOCALAI_SMOKE_BACKEND"
+image=sha256:f1bc435659b9da49180e868fdd0361915171a740cd947bf081b62a896a7ea8d0
+docker image inspect "$image" --format 'provider image {{.Id}} {{json .Config.Labels}}'
+name="lr-embedding-$$"
+created=
+backend_pid= provider_pid=
+cleanup() {
+ [ -z "$created" ] || docker rm -v "$name" >/dev/null 2>&1 || true
+ [ -z "$provider_pid" ] || { kill "$provider_pid" 2>/dev/null || true; wait "$provider_pid" 2>/dev/null || true; }
+ [ -z "$backend_pid" ] || { kill "$backend_pid" 2>/dev/null || true; wait "$backend_pid" 2>/dev/null || true; }
+}
+trap cleanup EXIT HUP INT TERM
+if [ ! -f "$W/model-smoke/local-ai" ]; then
+ docker create --name "$name" --network none --tmpfs /models --tmpfs /backends --tmpfs /configuration --tmpfs /data "$image" >/dev/null
+ created=yes
+ docker cp "$name:/local-ai" "$W/model-smoke/local-ai"
+ docker rm -v "$name" >/dev/null
+ created=
+fi
+echo "3c03e310467418dec59d6c391e395a8e4cdfcee6dd450e66f74eeeefa21a53f6 $W/model-smoke/local-ai" | sha256sum -c -
+ports=$(python3 - <<'PY'
+import socket
+s=[socket.socket(),socket.socket()]
+for x in s: x.bind(('127.0.0.1',0))
+print(*(x.getsockname()[1] for x in s))
+PY
+)
+backend_port=${ports%% *}; api_port=${ports##* }
+cat > "$W/model-smoke/minilm.yaml" < "$W/model-smoke/legacy.yaml"
+(cd "$W/model-smoke"; exec "$LOCALAI_SMOKE_BACKEND" --addr "127.0.0.1:$backend_port") > "$W/logs/localai-native-smoke.log" 2>&1 &
+backend_pid=$!
+(cd "$W/model-smoke"; exec ./local-ai run --address "127.0.0.1:$api_port" --models-path "$W/model-smoke" --backends-path "$W/model-smoke/backends" --backends-system-path "$W/model-smoke/system" --data-path "$W/model-smoke/data" --localai-config-dir "$W/model-smoke/config" --external-grpc-backends "smoke-llama:127.0.0.1:$backend_port" --disable-web-ui --disable-agents) > "$W/logs/localai-provider-smoke.log" 2>&1 &
+provider_pid=$!
+export LOCALRECALL_SMOKE_API="http://127.0.0.1:$api_port/v1"
+python3 - "$LOCALRECALL_SMOKE_API" "$W/logs/localai-embeddings.json" "$provider_pid" <<'PY'
+import urllib.request,time,json,sys,math,os
+for i in range(90):
+ os.kill(int(sys.argv[3]),0)
+ try:
+ req=urllib.request.Request(sys.argv[1]+'/embeddings',data=json.dumps({'model':'minilm','input':'The telescope studies distant galaxies.'}).encode(),headers={'Content-Type':'application/json'})
+ with urllib.request.urlopen(req,timeout=10) as resp: data=json.load(resp)
+ v=data['data'][0]['embedding']; assert len(v)==384 and all(math.isfinite(x) for x in v) and any(x!=0 for x in v)
+ open(sys.argv[2],'w').write(json.dumps(data)); print('Genuine MiniLM /embeddings ready, dimensions',len(v)); break
+ except Exception as e:
+ if i==89: raise
+ time.sleep(1)
+PY
+# An optional command runs only after a genuine model response, and remains
+# inside this script's owned provider lifetime. No externally supplied endpoint.
+if [ "$#" -gt 0 ]; then
+ export LOCALAI_ENDPOINT="$LOCALRECALL_SMOKE_API"
+ "$@"
+ exit $?
+fi
+cp test/raw/templates/localai_consumer_test.go.txt "$W/localai/core/services/agentpool/localrecall_smoke_test.go"
+# Preserve upstream mod/sum; resolve only in private alternate files.
+cp "$W/localai/go.mod" "$W/localai-smoke.mod"
+cp "$W/localai/go.sum" "$W/localai-smoke.sum"
+(cd "$W/localai"; go mod edit -modfile="$W/localai-smoke.mod" -replace="github.com/mudler/localrecall=$ROOT")
+echo "consumer: go test -mod=mod -modfile=$W/localai-smoke.mod -count=1 -v ./core/services/agentpool -run ^TestLocalRecallRealConsumer$ (no build tags)"
+sh test/raw/database.sh sh -c 'cd "$1/localai"; go test -mod=mod -modfile="$1/localai-smoke.mod" -count=1 -v -timeout=5m ./core/services/agentpool -run "^TestLocalRecallRealConsumer$"' sh "$W"
diff --git a/test/raw/templates/localai_consumer_test.go.txt b/test/raw/templates/localai_consumer_test.go.txt
new file mode 100644
index 0000000..d2e18ef
--- /dev/null
+++ b/test/raw/templates/localai_consumer_test.go.txt
@@ -0,0 +1,244 @@
+package agentpool
+
+import (
+ "context"
+ "fmt"
+ "github.com/jackc/pgx/v5"
+ "github.com/mudler/localrecall/rag/raw"
+ rawpostgres "github.com/mudler/localrecall/rag/raw/postgres"
+ "net/http"
+ "net/http/httptest"
+ "net/http/httputil"
+ urlpkg "net/url"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/mudler/LocalAGI/webui/collections"
+ "github.com/mudler/LocalAI/core/config"
+)
+
+// This fixture is copied into the pinned actual consumer package. It uses
+// the same collection construction as startDistributed, without starting
+// unrelated agent schedulers, messaging or skills services.
+func TestLocalRecallRealConsumer(t *testing.T) {
+ url, db := os.Getenv("LOCALRECALL_SMOKE_API"), os.Getenv("LOCALRECALL_TEST_DATABASE_URL")
+ if url == "" || db == "" || os.Getenv("LOCALRECALL_TEST_DATABASE_OWNER") == "" {
+ t.Fatal("owned provider/database required")
+ }
+ target, err := urlpkg.Parse(url)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var calls atomic.Int64
+ var hold atomic.Bool
+ entered, release := make(chan struct{}, 1), make(chan struct{})
+ defer func() {
+ select {
+ case <-release:
+ default:
+ close(release)
+ }
+ }()
+ proxy := httputil.NewSingleHostReverseProxy(target)
+ counting := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, "/embeddings") {
+ calls.Add(1)
+ if hold.Load() {
+ select {
+ case entered <- struct{}{}:
+ default:
+ }
+ select {
+ case <-release:
+ case <-r.Context().Done():
+ return
+ }
+ }
+ }
+ proxy.ServeHTTP(w, r)
+ }))
+ defer func() {
+ select {
+ case <-release:
+ default:
+ close(release)
+ }
+ counting.Close()
+ }()
+ url = counting.URL
+ dir := t.TempDir()
+ svc, err := NewAgentPoolService(&config.ApplicationConfig{AgentPool: config.AgentPoolConfig{
+ VectorEngine: "postgres", DatabaseURL: db, EmbeddingModel: "minilm", MaxChunkingSize: 400,
+ }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ cfg := svc.buildCollectionsConfig(url, "", filepath.Join(dir, "collections"), filepath.Join(dir, "assets"))
+ inner, state := collections.NewInProcessBackend(cfg)
+ svc.collectionsBackend = svc.shareCollections(inner, state, cfg)
+ defer func() {
+ svc.collectionsBackend.(*sharedCollections).Close()
+ state.SourceManager.Stop()
+ for _, kb := range state.Collections {
+ if kb != nil {
+ kb.Close()
+ }
+ }
+ }()
+ const name = "real-consumer"
+ if err := svc.CreateCollectionForUser("", name); err != nil {
+ t.Fatal(err)
+ }
+ const text = "The observatory telescope studies distant galaxies and bright stars."
+ entry, err := svc.UploadToCollectionForUser("", name, "astronomy.txt", strings.NewReader(text))
+ if err != nil {
+ t.Fatal(err)
+ }
+ entries, err := svc.ListCollectionEntriesForUser("", name)
+ if err != nil || len(entries) != 1 {
+ t.Fatalf("entries=%v err=%v", entries, err)
+ }
+ content, chunks, err := svc.GetCollectionEntryContentForUser("", name, entry)
+ if err != nil || !strings.Contains(content, "galaxies") || chunks < 1 {
+ t.Fatalf("content=%q chunks=%d err=%v", content, chunks, err)
+ }
+ results, err := svc.SearchCollectionForUser("", name, "telescope galaxies", 3)
+ if err != nil || len(results) == 0 || !strings.Contains(results[0].Content, "galaxies") {
+ t.Fatalf("search=%v err=%v", results, err)
+ }
+ if calls.Load() == 0 {
+ t.Fatal("consumer bypassed embedding counter")
+ }
+ before := calls.Load()
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer cancel()
+ admin, err := pgx.Connect(ctx, db)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer admin.Close(ctx)
+ peak := 0
+ sample := func() {
+ var n int
+ if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid()").Scan(&n); err != nil {
+ t.Fatal(err)
+ }
+ if n > peak {
+ peak = n
+ }
+ }
+ catalog := func() int {
+ var n int
+ if err := admin.QueryRow(ctx, "SELECT count(*) FROM pg_class WHERE relnamespace='public'::regnamespace").Scan(&n); err != nil {
+ t.Fatal(err)
+ }
+ return n
+ }
+ baselineCatalog := catalog()
+ start := time.Now()
+ store, err := rawpostgres.New(ctx, rawpostgres.Config{DatabaseURL: db})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer store.Close()
+ rc, _, err := store.CreateCollection(ctx, raw.CreateCollectionRequest{Name: name, Dimensions: 3, Metric: "cosine", EmbeddingSpaceID: "synthetic-v1"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer store.DeleteCollection(ctx, rc.CollectionID)
+ if _, err = store.Upsert(ctx, rc.CollectionID, raw.UpsertRequest{EmbeddingSpaceID: "synthetic-v1", Records: []raw.Record{{ID: "raw", Vector: []float64{1, 0, 0}}}}); err != nil {
+ t.Fatal(err)
+ }
+ space, accuracy := "synthetic-v1", "exact"
+ if _, err = store.Search(ctx, rc.CollectionID, raw.SearchRequest{Mode: "vector", Accuracy: &accuracy, EmbeddingSpaceID: &space, Vector: []float64{1, 0, 0}}); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = store.GetRecords(ctx, rc.CollectionID, raw.GetRecordsRequest{IDs: []string{"raw"}}); err != nil {
+ t.Fatal(err)
+ }
+ if err = store.Reset(ctx, rc.CollectionID); err != nil {
+ t.Fatal(err)
+ }
+ if err = store.DeleteCollection(ctx, rc.CollectionID); err != nil {
+ t.Fatal(err)
+ }
+ if calls.Load() != before {
+ t.Fatalf("raw embedding traffic: before=%d after=%d", before, calls.Load())
+ }
+ t.Logf("serial consumer embedding calls=%d; raw lifecycle/search attributed calls=0", before)
+ // Hold an actual consumer embedding HTTP request while raw database work
+ // completes. Only the consumer receives this dedicated forwarding endpoint;
+ // raw Service has no HTTP client or endpoint. Exact delta attributes overlap.
+ if err := svc.CreateCollectionForUser("", "real-consumer-second"); err != nil {
+ t.Fatal(err)
+ }
+ before = calls.Load()
+ hold.Store(true)
+ legacyDone := make(chan error, 1)
+ go func() { _, e := svc.SearchCollectionForUser("", name, "telescope galaxies", 3); legacyDone <- e }()
+ select {
+ case <-entered:
+ case <-time.After(10 * time.Second):
+ t.Fatal("consumer never entered embedding proxy")
+ }
+ overlapStart := time.Now()
+ rawIDs := []string{}
+ for i := 0; i < 3; i++ {
+ rc, _, e := store.CreateCollection(ctx, raw.CreateCollectionRequest{Name: fmt.Sprintf("overlap-%d", i), Dimensions: 3, Metric: "cosine", EmbeddingSpaceID: "synthetic-v1"})
+ if e != nil {
+ t.Fatal(e)
+ }
+ rawIDs = append(rawIDs, rc.CollectionID)
+ if _, e = store.Upsert(ctx, rc.CollectionID, raw.UpsertRequest{EmbeddingSpaceID: "synthetic-v1", Records: []raw.Record{{ID: "raw", Vector: []float64{1, 0, 0}}}}); e != nil {
+ t.Fatal(e)
+ }
+ if _, e = store.Search(ctx, rc.CollectionID, raw.SearchRequest{Mode: "vector", Accuracy: &accuracy, EmbeddingSpaceID: &space, Vector: []float64{1, 0, 0}}); e != nil {
+ t.Fatal(e)
+ }
+ sample()
+ }
+ highCatalog := catalog()
+ if calls.Load() != before+1 {
+ t.Fatalf("overlap attribution: %d -> %d", before, calls.Load())
+ }
+ for _, id := range rawIDs {
+ if e := store.DeleteCollection(ctx, id); e != nil {
+ t.Fatal(e)
+ }
+ }
+ finalCatalog := catalog()
+ overlapElapsed := time.Since(overlapStart)
+ close(release)
+ hold.Store(false)
+ select {
+ case e := <-legacyDone:
+ if e != nil {
+ t.Fatal(e)
+ }
+ case <-time.After(10 * time.Second):
+ t.Fatal("consumer did not finish")
+ }
+ if calls.Load() != before+1 {
+ t.Fatal("unexpected embedding traffic")
+ }
+ // Default four per pool: one raw plus two loaded legacy collections.
+ if peak < 3 || peak > 12 {
+ t.Fatalf("combined sampled pool peak=%d budget=12", peak)
+ }
+ if highCatalog <= finalCatalog {
+ t.Fatal("no index/catalog lifecycle observed")
+ }
+ t.Logf("operational fixture raw_collections=3 legacy_collections=2 overlap_requests=1 overlap_duration=%s legacy_calls=1 raw_calls=0 combined_sampled_pool_peak=%d configured_budget=12 catalog_before=%d catalog_high=%d catalog_after=%d catalog_removed=%d elapsed=%s", overlapElapsed, peak, baselineCatalog, highCatalog, finalCatalog, highCatalog-finalCatalog, time.Since(start))
+ if err := svc.ResetCollectionForUser("", name); err != nil {
+ t.Fatal(err)
+ }
+ entries, err = svc.ListCollectionEntriesForUser("", name)
+ if err != nil || len(entries) != 0 {
+ t.Fatalf("reset entries=%v err=%v", entries, err)
+ }
+ t.Log("actual agentpool construction, upload, entry, real-embedding search and reset passed")
+}
diff --git a/test/raw/unit.sh b/test/raw/unit.sh
new file mode 100644
index 0000000..76fee3c
--- /dev/null
+++ b/test/raw/unit.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+# Each package owns a pristine database: legacy catalogs must not contaminate
+# raw bootstrap/extension tests, and outage tests cannot disrupt other packages.
+set -eu
+python3 test/raw/confinement.py "$(pwd -P)"
+packages=$(go list ./rag/... ./pkg/... .)
+for package in $packages; do
+ sh test/raw/database.sh go test -v -count=1 -p=1 "$package"
+done
diff --git a/test/raw/verify.sh b/test/raw/verify.sh
new file mode 100644
index 0000000..2bdcd69
--- /dev/null
+++ b/test/raw/verify.sh
@@ -0,0 +1,84 @@
+#!/bin/sh
+set -eu
+ROOT=$(pwd -P)
+python3 test/raw/confinement.py "$ROOT"
+[ -f "$ROOT/go.mod" ] && [ -d "$ROOT/rag" ] || { echo 'run from worktree root' >&2; exit 1; }
+# Reject symlinks before creating any output, including nested cache paths.
+for path in .sdd-work .sdd-work/tmp .sdd-work/cache .sdd-work/mod .sdd-work/gopath .sdd-work/collections .sdd-work/assets .sdd-work/logs; do
+ if [ -L "$path" ] || { [ -d "$path" ] && [ "$path" != .sdd-work ] && [ -n "$(find "$path" -type l -print -quit)" ]; }; then
+ echo "symlinks in output path $path are not permitted" >&2; exit 1
+ fi
+done
+export TMPDIR="$ROOT/.sdd-work/tmp" GOCACHE="$ROOT/.sdd-work/cache" GOMODCACHE="$ROOT/.sdd-work/mod" GOPATH="$ROOT/.sdd-work/gopath"
+export COLLECTION_DB_PATH="$ROOT/.sdd-work/collections" FILE_ASSETS="$ROOT/.sdd-work/assets"
+export GOENV=off GOTOOLCHAIN=local
+mkdir -p "$TMPDIR" "$GOCACHE" "$GOMODCACHE" "$GOPATH" "$COLLECTION_DB_PATH" "$FILE_ASSETS" "$ROOT/.sdd-work/logs"
+case "${1:-}" in
+ all)
+ python3 test/raw/confinement_test.py
+ sh test/raw/verify.sh api
+ log="$ROOT/.sdd-work/logs/race10.log"
+ if sh test/raw/database.sh go test -v -mod=readonly -race -count=10 -timeout=30m ./rag/raw/... ./pkg/client/... >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi
+ ! grep -E 'SKIP|no tests to run|no test files' "$log"
+ for suite in TestRawProcessDiscovery TestRawRecordsReplacement TestRawOwnedOutage TestRawActiveShutdown TestRawClient; do
+ [ "$(grep -c -- "^--- PASS: $suite (" "$log")" -eq 10 ]
+ done
+ go vet -mod=readonly ./rag/raw/... ./pkg/client/...
+ sh test/raw/legacy.sh
+ log="$ROOT/.sdd-work/logs/consumer-all.log"
+ if sh test/raw/localai-smoke.sh >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi
+ grep -q -- '--- PASS: TestLocalRecallRealConsumer' "$log"
+ ! grep -E 'SKIP|no tests to run|no test files' "$log"
+ git diff --check
+ ;;
+ api)
+ sh test/raw/verify.sh search
+ log="$ROOT/.sdd-work/logs/api-$(date +%s)-$$.log"
+ if sh test/raw/database.sh go test -v -mod=readonly -race -count=1 -timeout=10m -run '^TestRaw' . >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi
+ for suite in HTTPValidation HTTPAuthAndReadiness APILifecycle; do grep -q -- "--- PASS: TestRaw${suite}" "$log"; done
+ ! grep -E "SKIP|no tests to run|no test files" "$log"
+ log="$ROOT/.sdd-work/logs/api-client-$(date +%s)-$$.log"
+ if go test -v -mod=readonly -race -count=1 ./pkg/client/... >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi
+ grep -q -- '--- PASS: TestRawClient' "$log"
+ ! grep -E "SKIP|no tests to run|no test files" "$log"
+ git diff --check
+ ;;
+ search)
+ sh test/raw/verify.sh records
+ log="$ROOT/.sdd-work/logs/search-$(date +%s)-$$.log"
+ if sh test/raw/database.sh go test -v -mod=readonly -race -count=1 -timeout=10m ./rag/raw/postgres/... -run "^(TestRaw(Search|Explain|Recall))" >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi
+ for suite in Search Explain Recall; do
+ grep -q -- "--- PASS: TestRaw${suite}" "$log"
+ done
+ ! grep -E "SKIP|no tests to run|no test files" "$log"
+ git diff --check
+ ;;
+ records)
+ sh test/raw/verify.sh lifecycle
+ log="$ROOT/.sdd-work/logs/records-$(date +%s)-$$.log"
+ if sh test/raw/database.sh go test -v -mod=readonly -race -count=1 -timeout=10m ./rag/raw/postgres/... -run "^(TestRaw(Records|Concurrency|Cursor|DeleteBound))" >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi
+ for suite in Records Concurrency Cursor DeleteBound; do
+ grep -q -- "--- PASS: TestRaw${suite}" "$log"
+ done
+ ! grep -E "SKIP|no tests to run|no test files" "$log"
+ git diff --check
+ ;;
+ lifecycle)
+ sh test/raw/verify.sh contracts
+ go test -v -mod=readonly -race -count=1 ./rag/internal/pgutil/...
+ log="$ROOT/.sdd-work/logs/lifecycle-$(date +%s)-$$.log"
+ if sh test/raw/database.sh go test -v -mod=readonly -race -count=1 -timeout=10m ./rag/raw/postgres/... -run "^(TestRaw(Lifecycle|Bootstrap|Capabilities|Pool|LegacyHelpers))" >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi
+ for suite in Lifecycle Bootstrap Capabilities Pool LegacyHelpers; do
+ grep -q -- "--- PASS: TestRaw${suite}" "$log"
+ done
+ ! grep -E "SKIP|no tests to run|no test files" "$log"
+ git diff --check
+ ;;
+ contracts)
+ log="$ROOT/.sdd-work/logs/contracts-$(date +%s)-$$.log"
+ if go test -mod=readonly -race -count=1 ./rag/raw ./test/raw >"$log" 2>&1; then cat "$log"; else cat "$log"; exit 1; fi
+ [ "$(grep -c '^ok[[:space:]]' "$log")" -eq 2 ] && ! grep -E '\[no test files\]|\[no tests to run\]' "$log"
+ git diff --check
+ ;;
+ *) echo 'usage: sh test/raw/verify.sh contracts|lifecycle|records|search|api|all' >&2; exit 2;;
+esac