diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 14eee78..c6a2558 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -6,11 +6,45 @@ on: branches: [ main ] jobs: - readme: + validate: runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: aurekai + POSTGRES_PASSWORD: aurekai + POSTGRES_DB: aurekai + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U aurekai -d aurekai" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + env: + DATABASE_URL: postgresql://aurekai:aurekai@localhost:5432/aurekai steps: - uses: actions/checkout@v4 - - name: Ensure scaffold files exist + + - name: Ensure required files exist run: | test -f README.md test -f LICENSE + test -f .release-versions.env + test -f scripts/init-db.sh + test -f tests/validate-schemas.sh + test -f tests/validate-scripts.sh + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y postgresql-client + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements.txt + + - name: Validate schemas + run: bash tests/validate-schemas.sh "$DATABASE_URL" + + - name: Validate scripts + run: bash tests/validate-scripts.sh "$DATABASE_URL" diff --git a/.release-versions.env b/.release-versions.env new file mode 100644 index 0000000..e8f31ad --- /dev/null +++ b/.release-versions.env @@ -0,0 +1,4 @@ +AUREKAI_VERSION=0.8.0-alpha.4 +AKAI_PACKAGE_VERSION=0.8.0-alpha.4 +AUREKAI_MANIFEST_SCHEMA=aurekai.deploy.v1 +HELM_CHART_VERSION=0.8.1 diff --git a/README.md b/README.md index e88fd04..8cb963a 100644 --- a/README.md +++ b/README.md @@ -4,20 +4,55 @@ # aurekai-postgres -Aurekai integration surface for Postgres. +Aurekai integration surface for PostgreSQL - robust, transactional, multi-tenant data-ml storage for model memory, SAE dictionaries, semantic cache, manifests, and proof bundles. -Status: planned -Type: data +Status: active +Type: data-ml + +## Overview + +`aurekai-postgres` provides PostgreSQL DDL schemas and operational scripts to persist and query Aurekai artifacts with production-grade guarantees: ACID transactions, row-level security compatibility, indexing, and JSONB-native querying. ## Core Template Set -- doctor-deep -- manifest-verify -- model-memory-pack -- sae-audit -- semantic-cache-bench -- proof-bundle-export -- release-gate +| Template | Description | +|---|---| +| `doctor-deep` | Connection checks, schema integrity, and optional `akai doctor --deep` | +| `manifest-verify` | Validate `aurekai.deploy.v1` manifest record in Postgres | +| `model-memory-pack` | Insert `.akmodel` / `.bfmodel` / `.akfpqx` / `.bffpqx` metadata + payload | +| `sae-audit` | Audit SAE dictionary records by extension/operator/layer | +| `semantic-cache-bench` | Benchmark cache write/read throughput on Postgres | +| `proof-bundle-export` | Export proof bundle rows to `out/proof-bundle.json` | +| `release-gate` | Verify populated tables and release constraints before publish | + +## Quick Start + +```bash +# 1) Start local postgres (optional) +docker compose -f examples/docker-compose.postgres.yaml up -d + +# 2) Export connection string +export DATABASE_URL=postgresql://aurekai:aurekai@localhost:5432/aurekai + +# 3) Initialize schema +bash scripts/init-db.sh "$DATABASE_URL" + +# 4) Run deep checks +bash scripts/doctor-deep.sh "$DATABASE_URL" + +# 5) Run release gate +bash scripts/release-gate.sh "$DATABASE_URL" +``` + +## Directory Layout + +``` +sql/ PostgreSQL schema files +scripts/ Operational template scripts +examples/ Local compose + sample payloads +tests/ Validation scripts for CI/local +docs/ Quickstart and reference docs +``` ## Canonical References diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 0000000..50c57d7 --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,40 @@ +# Quickstart + +## 1) Start PostgreSQL + +Use Docker for local development: + +```bash +docker compose -f examples/docker-compose.postgres.yaml up -d +``` + +## 2) Set DATABASE_URL + +```bash +export DATABASE_URL=postgresql://aurekai:aurekai@localhost:5432/aurekai +``` + +## 3) Install Python dependency for benchmark scripts + +```bash +python3 -m pip install -r requirements.txt +``` + +## 4) Initialize schema + +```bash +bash scripts/init-db.sh "$DATABASE_URL" +``` + +## 5) Run diagnostics + +```bash +bash scripts/doctor-deep.sh "$DATABASE_URL" +``` + +## 6) Run full validation + +```bash +bash tests/validate-schemas.sh "$DATABASE_URL" +bash tests/validate-scripts.sh "$DATABASE_URL" +``` diff --git a/docs/schema.md b/docs/schema.md new file mode 100644 index 0000000..272cccc --- /dev/null +++ b/docs/schema.md @@ -0,0 +1,21 @@ +# Schema Reference + +## Tables + +1. `model_memory`: metadata + optional payload for `.akmodel`, `.bfmodel`, `.akfpqx`, `.bffpqx` +2. `sae_dictionaries`: metadata + optional payload for `.aksae`, `.bfsae` +3. `semantic_cache`: key/value cache with optional embedding byte payload +4. `manifests`: release manifests (`aurekai.deploy.v1`) with JSONB raw payload +5. `proof_bundles`: proof outputs and status for recipe runs + +## Postgres Types Used + +1. `BYTEA` for binary artifacts and embeddings +2. `JSONB` for metadata/raw payload and proof records +3. `TIMESTAMPTZ` for UTC event timestamps + +## Indexing Strategy + +1. B-tree indexes on high-cardinality lookup fields (`artifact_name`, `operator_id`, `version`, `status`) +2. GIN indexes on JSONB columns for fast metadata and payload filtering +3. Unique constraint on `manifests(name, version)` diff --git a/docs/scripts.md b/docs/scripts.md new file mode 100644 index 0000000..c6d022d --- /dev/null +++ b/docs/scripts.md @@ -0,0 +1,12 @@ +# Scripts Reference + +Each script accepts `DATABASE_URL` as first positional arg, or via environment variable. + +1. `scripts/init-db.sh`: apply all schema files +2. `scripts/doctor-deep.sh`: verify connectivity, required tables, row counts, optional `akai doctor --deep` +3. `scripts/manifest-verify.sh`: ensure specific manifest exists +4. `scripts/model-memory-pack.sh`: insert binary artifact into `model_memory` +5. `scripts/sae-audit.sh`: summarize SAE records and checksum health +6. `scripts/semantic-cache-bench.sh`: benchmark cache write/read operations +7. `scripts/proof-bundle-export.sh`: export `proof_bundles` rows to JSON +8. `scripts/release-gate.sh`: enforce release readiness checks diff --git a/examples/docker-compose.postgres.yaml b/examples/docker-compose.postgres.yaml new file mode 100644 index 0000000..6c2f30a --- /dev/null +++ b/examples/docker-compose.postgres.yaml @@ -0,0 +1,15 @@ +version: '3.9' +services: + postgres: + image: postgres:16 + environment: + POSTGRES_USER: aurekai + POSTGRES_PASSWORD: aurekai + POSTGRES_DB: aurekai + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U aurekai -d aurekai"] + interval: 5s + timeout: 5s + retries: 20 diff --git a/examples/sample-akmodel.json b/examples/sample-akmodel.json new file mode 100644 index 0000000..dedd227 --- /dev/null +++ b/examples/sample-akmodel.json @@ -0,0 +1,9 @@ +{ + "name": "my-operator-v1", + "ext": ".akmodel", + "operator_id": "akai_run_operator", + "version": "0.8.0-alpha.4", + "schema_version": "aurekai.deploy.v1", + "size_bytes": 204800, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +} diff --git a/examples/sample-aurekai.manifest.json b/examples/sample-aurekai.manifest.json new file mode 100644 index 0000000..ddcfc6f --- /dev/null +++ b/examples/sample-aurekai.manifest.json @@ -0,0 +1,11 @@ +{ + "schema_version": "aurekai.deploy.v1", + "name": "aurekai", + "version": "0.8.0-alpha.4", + "operator_count": 89, + "runtime_ref": "git:1ba1f19", + "helm_chart_ver": "0.8.1", + "pypi_ver": "0.8.0a3", + "npm_ver": "0.8.0-alpha.4", + "jsr_ver": "0.8.0-alpha.3" +} diff --git a/examples/sample-db-init.sh b/examples/sample-db-init.sh new file mode 100755 index 0000000..aa5c053 --- /dev/null +++ b/examples/sample-db-init.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$SCRIPT_DIR/.." +DB_URL="${DATABASE_URL:-postgresql://aurekai:aurekai@localhost:5432/aurekai}" + +echo "Using DATABASE_URL=$DB_URL" +bash "$REPO_ROOT/scripts/init-db.sh" "$DB_URL" + +python3 - <=3.2,<3.4 diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100755 index 0000000..09b2f4b --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +DATABASE_URL="${1:-${DATABASE_URL:-}}" +if [ -z "$DATABASE_URL" ]; then + echo "DATABASE_URL not provided. Pass as first arg or export DATABASE_URL." >&2 + exit 1 +fi + +psql_exec() { + psql "$DATABASE_URL" -v ON_ERROR_STOP=1 "$@" +} diff --git a/scripts/doctor-deep.sh b/scripts/doctor-deep.sh new file mode 100755 index 0000000..df44474 --- /dev/null +++ b/scripts/doctor-deep.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Usage: bash scripts/doctor-deep.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" "${1:-}" + +echo "[doctor-deep] Checking Postgres connectivity" +psql_exec -t -A -c "SELECT 'ok'" | grep -q ok + +echo "[doctor-deep] Checking required tables" +for tbl in model_memory sae_dictionaries semantic_cache manifests proof_bundles; do + count=$(psql_exec -t -A -c "SELECT count(*) FROM pg_tables WHERE schemaname='public' AND tablename='$tbl';") + if [ "$count" != "1" ]; then + echo "[doctor-deep] Missing table: $tbl" >&2 + exit 1 + fi + rows=$(psql_exec -t -A -c "SELECT count(*) FROM $tbl;") + echo " $tbl: $rows rows" +done + +if command -v akai >/dev/null 2>&1; then + echo "[doctor-deep] Running akai doctor --deep" + akai doctor --deep +else + echo "[doctor-deep] akai not installed; skipping akai doctor --deep" +fi + +echo "[doctor-deep] PASS" diff --git a/scripts/init-db.sh b/scripts/init-db.sh new file mode 100755 index 0000000..a81fab7 --- /dev/null +++ b/scripts/init-db.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Usage: bash scripts/init-db.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" "${1:-}" + +echo "[init-db] Applying schemas to $DATABASE_URL" +for schema in \ + schema-model-memory.sql \ + schema-sae.sql \ + schema-semantic-cache.sql \ + schema-manifests.sql \ + schema-proof-bundles.sql; do + echo " Applying $schema" + psql_exec -f "$SCRIPT_DIR/../sql/$schema" >/dev/null +done + +echo "[init-db] Tables:" +psql_exec -t -A -c "SELECT tablename FROM pg_tables WHERE schemaname='public' ORDER BY tablename;" diff --git a/scripts/manifest-verify.sh b/scripts/manifest-verify.sh new file mode 100755 index 0000000..35bc3d3 --- /dev/null +++ b/scripts/manifest-verify.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Usage: bash scripts/manifest-verify.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" "${1:-}" +NAME="${2:?manifest name required}" +VERSION="${3:?manifest version required}" + +count=$(psql_exec -t -A -c "SELECT count(*) FROM manifests WHERE name='$NAME' AND version='$VERSION' AND schema_version='aurekai.deploy.v1';") +if [ "$count" -lt 1 ]; then + echo "[manifest-verify] FAIL: manifest not found" >&2 + exit 1 +fi + +psql_exec -P pager=off -c "SELECT id, name, version, schema_version, operator_count, created_at FROM manifests WHERE name='$NAME' AND version='$VERSION';" +echo "[manifest-verify] PASS" diff --git a/scripts/model-memory-pack.sh b/scripts/model-memory-pack.sh new file mode 100755 index 0000000..4d36a6e --- /dev/null +++ b/scripts/model-memory-pack.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Usage: bash scripts/model-memory-pack.sh [operator_id] +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" "${1:-}" +ARTIFACT="${2:?artifact path required}" +OPERATOR_ID="${3:-}" + +if [ ! -f "$ARTIFACT" ]; then + echo "[model-memory-pack] Artifact not found: $ARTIFACT" >&2 + exit 1 +fi + +EXT=".${ARTIFACT##*.}" +case "$EXT" in + .akmodel|.bfmodel|.akfpqx|.bffpqx) ;; + *) echo "[model-memory-pack] Unsupported extension: $EXT" >&2; exit 1 ;; +esac + +NAME=$(basename "$ARTIFACT") +SIZE=$(wc -c < "$ARTIFACT" | tr -d ' ') +SHA=$(shasum -a 256 "$ARTIFACT" | awk '{print $1}') +ESC_PATH=$(printf "%s" "$ARTIFACT" | sed "s/'/''/g") +ESC_NAME=$(printf "%s" "$NAME" | sed "s/'/''/g") +ESC_OP=$(printf "%s" "$OPERATOR_ID" | sed "s/'/''/g") + +if [ -n "$OPERATOR_ID" ]; then + OPVAL="'$ESC_OP'" +else + OPVAL="NULL" +fi + +psql_exec -c "INSERT INTO model_memory (artifact_name, artifact_ext, operator_id, size_bytes, sha256, payload) VALUES ('$ESC_NAME', '$EXT', $OPVAL, $SIZE, '$SHA', pg_read_binary_file('$ESC_PATH'));" + +echo "[model-memory-pack] Inserted $NAME" diff --git a/scripts/proof-bundle-export.sh b/scripts/proof-bundle-export.sh new file mode 100755 index 0000000..3194cec --- /dev/null +++ b/scripts/proof-bundle-export.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Usage: bash scripts/proof-bundle-export.sh [output_dir] +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" "${1:-}" +OUT_DIR="${2:-out}" +mkdir -p "$OUT_DIR" +OUT_FILE="$OUT_DIR/proof-bundle.json" + +echo "[proof-bundle-export] Exporting proof bundles to $OUT_FILE" + +psql_exec -t -A -F $'\t' -c "SELECT id, recipe_name, status, coalesce(operator_id,''), version, schema_version, proof_json::text, exported_at::text FROM proof_bundles ORDER BY exported_at DESC" | \ +python3 - "$OUT_FILE" <<'PYEOF' +import json +import sys + +out_file = sys.argv[1] +rows = [] +for line in sys.stdin: + line = line.rstrip("\n") + if not line: + continue + parts = line.split("\t") + if len(parts) != 8: + continue + row = { + "id": int(parts[0]), + "recipe_name": parts[1], + "status": parts[2], + "operator_id": parts[3] or None, + "version": parts[4], + "schema_version": parts[5], + "proof": json.loads(parts[6]), + "exported_at": parts[7], + } + rows.append(row) + +with open(out_file, "w", encoding="utf-8") as f: + json.dump({"status": "ok", "count": len(rows), "bundles": rows}, f, indent=2) + +print(f"[proof-bundle-export] Wrote {len(rows)} bundle(s)") +print("[proof-bundle-export] PASS") +PYEOF diff --git a/scripts/release-gate.sh b/scripts/release-gate.sh new file mode 100755 index 0000000..0ea0e8d --- /dev/null +++ b/scripts/release-gate.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Usage: bash scripts/release-gate.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" "${1:-}" + +fail=0 +for tbl in model_memory sae_dictionaries semantic_cache manifests proof_bundles; do + count=$(psql_exec -t -A -c "SELECT count(*) FROM $tbl;") + if [ "$count" -gt 0 ]; then + echo "[release-gate] PASS $tbl: $count row(s)" + else + echo "[release-gate] FAIL $tbl is empty" >&2 + fail=1 + fi +done + +bad_manifest=$(psql_exec -t -A -c "SELECT count(*) FROM manifests WHERE schema_version != 'aurekai.deploy.v1';") +if [ "$bad_manifest" -gt 0 ]; then + echo "[release-gate] FAIL manifests with non-standard schema_version: $bad_manifest" >&2 + fail=1 +fi + +failed_proof=$(psql_exec -t -A -c "SELECT count(*) FROM proof_bundles WHERE status = 'fail';") +if [ "$failed_proof" -gt 0 ]; then + echo "[release-gate] FAIL proof bundles in failed state: $failed_proof" >&2 + fail=1 +fi + +if [ "$fail" -ne 0 ]; then + echo "[release-gate] FAIL" >&2 + exit 1 +fi + +echo "[release-gate] PASS" diff --git a/scripts/sae-audit.sh b/scripts/sae-audit.sh new file mode 100755 index 0000000..148cedb --- /dev/null +++ b/scripts/sae-audit.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Usage: bash scripts/sae-audit.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" "${1:-}" + +echo "[sae-audit] Totals by extension" +psql_exec -P pager=off -c "SELECT artifact_ext, count(*) AS count, coalesce(sum(size_bytes),0) AS total_bytes FROM sae_dictionaries GROUP BY artifact_ext ORDER BY artifact_ext;" + +echo "[sae-audit] Totals by operator" +psql_exec -P pager=off -c "SELECT coalesce(operator_id,'') AS operator_id, count(*) AS count FROM sae_dictionaries GROUP BY operator_id ORDER BY count DESC;" + +nullsha=$(psql_exec -t -A -c "SELECT count(*) FROM sae_dictionaries WHERE sha256 IS NULL;") +if [ "$nullsha" -gt 0 ]; then + echo "[sae-audit] WARNING: $nullsha row(s) missing sha256" >&2 +fi + +echo "[sae-audit] PASS" diff --git a/scripts/semantic-cache-bench.sh b/scripts/semantic-cache-bench.sh new file mode 100755 index 0000000..39b902e --- /dev/null +++ b/scripts/semantic-cache-bench.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Usage: bash scripts/semantic-cache-bench.sh [iterations] +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" "${1:-}" +ITERATIONS="${2:-1000}" + +echo "[semantic-cache-bench] Running benchmark for $ITERATIONS iterations" +python3 - <" >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$SCRIPT_DIR/.." + +bash "$REPO_ROOT/scripts/init-db.sh" "$DB_URL" + +for tbl in model_memory sae_dictionaries semantic_cache manifests proof_bundles; do + count=$(psql "$DB_URL" -t -A -c "SELECT count(*) FROM pg_tables WHERE schemaname='public' AND tablename='$tbl';") + if [ "$count" != "1" ]; then + echo "[validate-schemas] FAIL missing $tbl" >&2 + exit 1 + fi + echo "[validate-schemas] OK $tbl" +done + +echo "[validate-schemas] PASS" diff --git a/tests/validate-scripts.sh b/tests/validate-scripts.sh new file mode 100755 index 0000000..3a136d4 --- /dev/null +++ b/tests/validate-scripts.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +DB_URL="${1:-${DATABASE_URL:-}}" +if [ -z "$DB_URL" ]; then + echo "Usage: bash tests/validate-scripts.sh " >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$SCRIPT_DIR/.." +export DATABASE_URL="$DB_URL" + +bash "$REPO_ROOT/scripts/init-db.sh" "$DB_URL" + +python3 - <