diff --git a/.agents/skills/neon-postgres/SKILL.md b/.agents/skills/neon-postgres/SKILL.md new file mode 100644 index 0000000..2c30c9f --- /dev/null +++ b/.agents/skills/neon-postgres/SKILL.md @@ -0,0 +1,278 @@ +--- +name: neon-postgres +description: >- + Guides and best practices for working with Lakebase Postgres, the database + behind Neon. Covers setup, connection methods and drivers, pooled vs direct + connections, branching, schema migrations, autoscaling, scale-to-zero, instant + restore, read replicas, connection pooling, IP allow lists, and logical + replication. Also covers Lakebase Search: semantic vector search, full-text + search with BM25 ranking, and hybrid search. + Use when users ask about "Lakebase Postgres", "Neon setup", "connect to Neon", + "Neon project", "DATABASE_URL", "serverless Postgres", "Neon CLI", "neon", "Neon MCP", + "Neon Auth", "@neondatabase/serverless", "@neondatabase/neon-js", + "scale to zero", "Neon autoscaling", "Neon read replica", + "Neon connection pooling", "schema migrations", "database troubleshooting", + "Postgres performance", "neon inspect db", "semantic search", "vector + search", "full-text search", "BM25", or "hybrid search". +metadata: + parent: neon + source: https://github.com/neondatabase/agent-skills/tree/main/skills/neon-postgres +--- + +**FIRST**: Use the parent `neon` skill for a Neon overview, getting started with Neon, Neon development best practices, and more. + +If the `neon` skill is not installed, fetch it from https://neon.com/docs/ai/skills/neon/SKILL.md or install it with: + +```bash +npx skills add neondatabase/agent-skills --skill neon +``` + +# Lakebase Postgres + +Lakebase Postgres is the database at the core of Neon. It runs on the lakebase architecture — OLTP built directly on cloud object storage — which decouples storage from compute to offer autoscaling, branching, instant restore, and scale-to-zero. It's fully compatible with Postgres and works with any language, framework, or ORM that supports Postgres. + +It is the same database whether you reach it through Neon or through Databricks; this skill covers the Neon access path. + +## Setup Flow + +### 1. Select the organization and project + +Use the CLI (default) or MCP server to list organizations and projects. Let the user select an existing project or create a new one. Check the `.neon` file for an existing linked project or branch. + +### 2. Get the connection string + +Use the CLI (default), `neon env pull`, or the MCP server to get the connection string. Store it in `.env` as `DATABASE_URL`. Read the file first before modifying it, to avoid overwriting existing values. + +#### When to use pooled vs direct connections + +| Use case | Connection type | +| ---------------------------------------- | ---------------- | +| Web applications, serverless functions | Pooled (-pooler) | +| Schema migrations | Direct | +| pg_dump / pg_restore | Direct | +| Logical replication | Direct | +| Long-running analytics with temp tables | Direct | +| Admin tasks needing SET or session state | Direct | +| LISTEN / NOTIFY | Direct | + +### 3. Pick the connection method and driver + +Always pair Neon with an ORM such as **Drizzle** for easy schema management and migrations. Refer to the connection methods guide to pick the correct driver based on how the runtime treats your code: https://neon.com/docs/connect/choose-connection.md. + +Recommendations: + +- Drizzle as ORM (see https://neon.com/docs/guides/drizzle.md) +- On Vercel, use `node-postgres` (`npm install pg`) with Vercel Fluid compute and `import { attachDatabasePool } from "@vercel/functions";` +- On Cloudflare, use `node-postgres` with Cloudflare Hyperdrive +- On Neon Functions, use `node-postgres`, as the functions are long-running and reuse the pool across requests. +- Use the `@neondatabase/serverless` driver for serverless and edge environments (for example, when using Netlify) — HTTP transport for one-shot queries, WebSocket for transaction support. Link: https://neon.com/docs/serverless/serverless-driver.md + +### 4. Set up the schema + +Manage schemas and migrations as code. Avoid running ad hoc schema migrations against your database, since they're hard to manage. + +If you're using an ORM, follow your ORM's best practices to manage schemas and migrations. For example, if using Drizzle, only use Drizzle for schema and migration management unless instructed otherwise. + +## Branching + +Use this when the user is planning isolated environments, schema migration testing, preview deployments, or branch lifecycle automation. + +Key points: + +- Branches are instant, copy-on-write clones (no full data copy). +- Each branch has its own compute endpoint. +- Use the neon CLI or MCP server to create, inspect, and compare branches. + +Link: https://neon.com/docs/introduction/branching.md + +For detailed branch creation workflows (normal vs schema-only branches, reset-from-parent, CLI/MCP selection), use the `neon-postgres-branches` skill. If it isn't installed, fetch it from https://neon.com/docs/ai/skills/neon-postgres-branches/SKILL.md or install it with: + +```bash +npx skills add neondatabase/agent-skills --skill neon-postgres-branches +``` + +## Migrations + +Test a migration on a branch of production, against production-like data, before applying it to production. + +Use a **direct (non-pooled)** connection string when you run the migration, not a pooled one. `neon connection-string` returns the direct string by default; make sure the hostname does not include the `-pooler` suffix. + +## Troubleshooting and Neon-Specific Performance + +Use Neon's predefined, read-only diagnostics before writing catalog queries by hand. The Neon CLI `neon inspect db` subcommands and the Neon MCP server's `inspect_database` tool run the same checks. + +This section covers Neon-specific diagnostic tools, compute cache behavior, and platform signals. When the evidence points to generic Postgres work such as rewriting a query, choosing an index, changing a schema, or interpreting plan nodes, load the [`postgres-best-practices`](https://github.com/neondatabase/postgres-skills/tree/main/skills/postgres-best-practices) skill and carry the diagnostic evidence into that workflow. + +Docs: + +- CLI: https://neon.com/docs/cli/inspect.md +- Query performance: https://neon.com/docs/postgresql/query-performance.md +- `pg_stat_statements`: https://neon.com/docs/extensions/pg_stat_statements.md +- Neon Local File Cache: https://neon.com/docs/extensions/neon.md + +### Choose CLI or MCP + +Prefer the Neon CLI when terminal access and authentication are available: + +```bash +neon inspect db +``` + +The CLI resolves the project and branch from the current Neon context. Use `--project-id`, `--branch`, and `--database-name` to override it. Omit `--database-name` to inspect every database on the branch. Use `--db-url` only when inspecting a Postgres database directly instead of resolving it through the Neon API. + +When using Neon MCP, call `inspect_database` with `projectId` and one `check`. Pass `branchId`, `databaseName`, or `computeId` only when needed. Omit `databaseName` to inspect all databases on the branch. Increase `limit` only when the result says it was truncated. + +### Pick the Diagnostic + +| Symptom or question | Checks | +| ---------------------------------------------- | ------------------------------------ | +| Which relations consume storage? | `table-sizes`, `index-sizes` | +| Is an index unused or a table scanned heavily? | `unused-indexes`, `seq-scans` | +| What has run for 5+ minutes or holds locks? | `long-running-queries`, `locks` | +| Which queries consume the most total time? | `outliers` | +| Which queries run most often? | `calls` | +| Does the active data fit in compute cache? | `lfc-hit-rate`, `working-set` | +| Is autovacuum behind or is space wasted? | `vacuum-stats`, `bloat` | +| Is logical replication healthy? | `replication-slots`, `subscriptions` | + +Do not confuse these checks: + +- `long-running-queries` reports statements running **right now** for more than five minutes. +- `outliers` ranks the top queries by cumulative execution time since statistics were reset. It does not rank by mean latency. +- `calls` ranks by execution count over the same statistics history. + +`outliers` and `calls` require `pg_stat_statements`. `lfc-hit-rate` and `working-set` require the `neon` extension. If a check reports a missing extension, ask before running the suggested `CREATE EXTENSION` statement because installing an extension modifies the database. + +### Interpret Results Safely + +- Treat `unused-indexes` as a candidate list, not permission to drop indexes. Confirm the observation window, constraints, and workload before removal. +- A sequential scan can be correct for a small table or a query reading much of a table. Check table size, selectivity, and the query plan before adding an index. +- `bloat` is a statistical estimate. Confirm the impact and plan locks or maintenance before `VACUUM FULL`, `REINDEX`, or similar remediation. +- Cache and Postgres statistics reset when compute restarts, including scale-to-zero suspension. Run a representative workload before interpreting fresh `lfc-hit-rate`, `working-set`, `vacuum-stats`, or `pg_stat_statements` results. +- Compute-wide checks (`lfc-hit-rate`, `working-set`, and `replication-slots`) run once even when inspecting every database. +- One failing database can fail an all-databases inspection; retry the relevant check with an explicit `databaseName` to isolate it. + +### Inspect Neon Cache Behavior Per Query + +Standard `EXPLAIN (ANALYZE, BUFFERS)` reports Postgres shared-buffer activity, but it does not show Neon's Local File Cache (LFC) or page prefetching. For a safe read-only query, add Neon's `FILECACHE` and `PREFETCH` options: + +```sql +EXPLAIN (ANALYZE, BUFFERS, PREFETCH, FILECACHE) +SELECT ...; +``` + +- `File cache: hits` counts pages found in the compute's LFC. +- `File cache: misses` counts pages not found in the LFC and fetched from database storage. +- `Prefetch: hits`, `misses`, `expired`, and `duplicates` show how effectively Neon fetched pages before the executor requested them. + +`FILECACHE` and `PREFETCH` provide metrics for this query and do not require the `neon` extension. By contrast, `neon inspect db lfc-hit-rate` and `working-set` provide compute-wide statistics and do require the extension. + +The MCP `explain_sql_statement` tool can produce a standard plan but does not expose `FILECACHE` or `PREFETCH` options. To collect those Neon-specific metrics through MCP, use `run_sql` with the explicit, read-only `EXPLAIN` statement above. + +Because `ANALYZE` executes the statement, use it only when execution is safe; do not run it autonomously for mutating SQL. Compare cold- and warm-cache runs carefully because the first execution can populate the cache and materially change later results. + +### Performance Workflow + +1. Reproduce the symptom and note its time window. +2. Run the smallest relevant `inspect` checks from the table above. +3. Identify a specific query before changing schema or compute. Use MCP `explain_sql_statement` for a standard plan, or the Neon-specific `EXPLAIN` above when LFC or prefetch behavior matters. +4. If the bottleneck is query shape, indexing, schema, locking, or vacuum behavior, load `postgres-best-practices` and carry forward the inspection results and query plan. Keep Neon compute, cache, connection, and platform decisions in this skill. +5. Re-run the same check and workload to verify the change. + +Use MCP `list_slow_queries` instead of `inspect_database` when the user specifically needs queries ranked by average execution time with a custom threshold and limit. Outside the explicit `EXPLAIN` case above, use `run_sql` only for read-only diagnostic SQL when the predefined checks do not answer the question. + +## Autoscaling + +Use this when the user needs compute to scale automatically with workload and wants guidance on CU sizing and runtime behavior. + +Link: https://neon.com/docs/introduction/autoscaling.md + +## Scale to Zero + +Use this when optimizing idle costs and discussing suspend/resume behavior, including cold-start trade-offs. + +Key points: + +- Idle computes suspend automatically after a default of 5 minutes; the timeout is configurable, and suspension can only be disabled on the Launch and Scale plans. +- First query after suspend typically has a cold-start penalty (around hundreds of ms) +- Storage remains active while compute is suspended. + +Link: https://neon.com/docs/introduction/scale-to-zero.md + +## Instant Restore + +Use this when the user needs point-in-time recovery or wants to restore data state without traditional backup restore workflows. + +Key points: + +- History windows for instant restore depend on plan limits. +- Users can create branches from historical points-in-time. +- Time Travel queries can be used for historical inspection workflows. + +Link: https://neon.com/docs/introduction/branch-restore.md + +## Read Replicas + +Use this for read-heavy workloads where the user needs dedicated read-only compute without duplicating storage. + +Key points: + +- Replicas are read-only compute endpoints sharing the same storage. +- Creation is fast and scaling is independent from primary compute. +- Typical use cases: analytics, reporting, and read-heavy APIs. + +Link: https://neon.com/docs/introduction/read-replicas.md + +## Connection Pooling + +Use this when the user is in serverless or high-concurrency environments and needs safe, scalable Postgres connection management. + +Key points: + +- Neon pooling uses PgBouncer. +- Add `-pooler` to endpoint hostnames to use pooled connections. +- Pooling is especially important in serverless runtimes with bursty concurrency. + +Link: https://neon.com/docs/connect/connection-pooling.md + +## IP Allow Lists + +Use this when the user needs to restrict database access by trusted networks, IPs, or CIDR ranges. + +Link: https://neon.com/docs/introduction/ip-allow.md + +## Logical Replication + +Use this when integrating CDC pipelines, external Postgres sync, or replication-based data movement. + +Key points: + +- Neon supports native logical replication workflows. +- Useful for replicating to/from external Postgres systems. + +Link: https://neon.com/docs/guides/logical-replication-guide.md + +## Lakebase Search + +Use Lakebase Search for semantic, full-text, and hybrid search: + +- For semantic search, read [Vector search](references/vector-search.md). +- For full-text search with BM25 ranking, read [Full-text search](references/full-text-search.md). +- For combining semantic and lexical results, read [Hybrid search](references/hybrid-search.md). + +Links: + +- [Get started with Lakebase Search](https://neon.com/docs/ai/lakebase-search-get-started) +- [`lakebase_vector` reference](https://neon.com/docs/extensions/lakebase-vector) +- [`lakebase_text` reference](https://neon.com/docs/extensions/lakebase-text) + +## Gotchas + +### Pooled vs direct connections: use the direct URL for migrations, dumps, and replication + +Neon gives you two connection strings for the same database: a **pooled** one (hostname with the `-pooler` suffix) and a **direct/unpooled** one (no `-pooler` suffix). `neon env pull` writes them as `DATABASE_URL` and `DATABASE_URL_UNPOOLED`. The pooled connection routes through PgBouncer in transaction mode, which doesn't support session-level operations. Choose the right one: + +- **Pooled (`DATABASE_URL`)** — your application's normal query traffic, especially serverless and connection-per-request workloads. +- **Direct (`DATABASE_URL_UNPOOLED`)** — schema migrations (Prisma Migrate, Drizzle Kit, Alembic, and others), `pg_dump` / `pg_restore`, logical replication, `LISTEN`/`NOTIFY`, and anything relying on `SET` or other session state. + +Running migrations, dumps, or replication over the pooled connection can fail, and never in a way that names pooling: `prepared statement "s0" already exists` from Prisma Migrate, a `SET search_path` that doesn't persist past its own transaction so the next query reports `relation "mytable" does not exist`, or a write intermittently hitting a read-only transaction (`SQLSTATE 25006`) that a pooled backend inherited from an earlier client. Migration tools generally take both strings at once — Prisma's `directUrl` alongside `url` — so point that at the direct one rather than swapping `DATABASE_URL` and losing pooling for the application. See https://neon.com/docs/connect/connection-pooling.md. diff --git a/.agents/skills/neon-postgres/references/full-text-search.md b/.agents/skills/neon-postgres/references/full-text-search.md new file mode 100644 index 0000000..9f90b7f --- /dev/null +++ b/.agents/skills/neon-postgres/references/full-text-search.md @@ -0,0 +1,99 @@ +# Full-Text Search with BM25 Ranking + +Use `lakebase_text` for BM25 relevance ranking with PostgreSQL's standard `tsvector` type. The `lakebase_bm25` index adds corpus-aware ranking and top-K pushdown. + +Lakebase Search requires Postgres 16 or later. Enable the extension before creating the index: + +```sql +CREATE EXTENSION IF NOT EXISTS lakebase_text; +``` + +`lakebase_text` has no extension dependency. It relies on a preloaded library that Neon enables by default; if the project customized its preloaded-library list, confirm the library remains enabled. + +## Prepare and Index Text + +Prefer a stored generated `tsvector` when search text comes from stable table columns: + +```sql +CREATE TABLE documents ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + title text NOT NULL, + body text NOT NULL, + body_tsv tsvector GENERATED ALWAYS AS + (to_tsvector('english', body)) STORED +); +``` + +Create the index after the initial corpus has been inserted so build-time corpus statistics are meaningful. BM25 scoring is tuned by two storage parameters set at index-build time: + +- `k1` controls term-frequency saturation (default `1.2`, range `1.2`–`2.0`): higher values let repeated terms keep adding relevance. + +- `b` controls document-length normalization (default `0.75`, range `0.0`–`1.0`): higher values penalize longer documents more. + +Both can only be set in the `WITH` clause, and updating them rebuilds the index: + +```sql +CREATE INDEX documents_body_bm25 ON documents + USING lakebase_bm25 (body_tsv) + WITH (k1 = 1.2, b = 0.75); +``` + +After a large bulk load, run `VACUUM` to refresh the statistics used by BM25 scoring. + +## Query and Interpret Scores + +`to_bm25query` binds the query `tsvector` to the BM25 index whose corpus statistics should be used. The `<@>` operator returns a negative BM25 score, so lower (more negative) values are more relevant and must sort ascending: + +```sql +SELECT + id, + title, + body_tsv <@> to_bm25query( + to_tsvector('english', $1), + 'documents_body_bm25'::regclass + ) AS score +FROM documents +ORDER BY score +LIMIT $2; +``` + +Use the same text-search configuration for document and query vectors. Select a language-specific or custom configuration that matches the corpus. + +## Set the Candidate Limit + +`lakebase_bm25.default_limit` controls how many rows the index returns before PostgreSQL applies the SQL `LIMIT`. Its default is `1000`; setting it close to the requested top-K avoids unnecessary scoring. + +## Use Prefilter Selectively + +Enable prefilter when a `WHERE` condition is strict or unpredictable and cheap to evaluate. It lets the index prune rows before BM25 scoring. A loose or expensive filter can be slower with prefilter enabled. + +```sql +BEGIN; +SET LOCAL lakebase_bm25.default_limit = 20; +SET LOCAL lakebase_bm25.prefilter = on; + +SELECT + id, + title, + body_tsv <@> to_bm25query( + to_tsvector('english', $1), + 'documents_body_bm25'::regclass + ) AS score +FROM documents +WHERE id % 1000 = 0 +ORDER BY score +LIMIT $2; +COMMIT; +``` + +## Set Parameters at Build Time or Per Query + +Several BM25 parameters can be set in two places. As an index storage parameter in the `CREATE INDEX` `WITH` clause, a value is fed into the index as its build-time default. As a session GUC via `SET` (or `SET LOCAL`), it applies per query and takes precedence over the stored default when both are present. + +- `default_limit` and `prefilter` exist in both forms: set an index default that fits the common case, then override it per query with a GUC without rebuilding. +- `k1` (default `1.2`) and `b` (default `0.75`) are storage parameters only. There is no GUC for them. +- `enable_scan` (default `on`) is a GUC only. + +The examples use `SET LOCAL` so each override is scoped to its own transaction, which is required behind a connection pool or stateless driver. + +Source: [`lakebase_text` documentation](https://neon.com/docs/extensions/lakebase-text). diff --git a/.agents/skills/neon-postgres/references/hybrid-search.md b/.agents/skills/neon-postgres/references/hybrid-search.md new file mode 100644 index 0000000..309bd34 --- /dev/null +++ b/.agents/skills/neon-postgres/references/hybrid-search.md @@ -0,0 +1,90 @@ +# Hybrid Search + +Use hybrid search when either semantic similarity or exact vocabulary can identify a relevant document. Lakebase Search does not provide a built-in hybrid function: run vector and BM25 retrieval separately, then combine their results with a fusion strategy suited to the workload. + +Lakebase Search requires Postgres 16 or later. Hybrid search uses both extensions: + +```sql +CREATE EXTENSION IF NOT EXISTS lakebase_vector CASCADE; +CREATE EXTENSION IF NOT EXISTS lakebase_text; +``` + +`lakebase_vector` installs `pgvector` through `CASCADE`; `lakebase_text` has no extension dependency. Both rely on preloaded libraries that Neon enables by default. If the project customized its preloaded-library list, confirm both libraries remain enabled. + +Prepare a table with both vector and text-search columns: + +```sql +CREATE TABLE documents ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + title text NOT NULL, + body text NOT NULL, + embedding vector(1536), + body_tsv tsvector GENERATED ALWAYS AS + (to_tsvector('english', body)) STORED +); +``` + +Replace `1536` with the embedding model's dimension. Use the same model and preprocessing for stored-document and query embeddings, and choose a PostgreSQL text-search configuration appropriate for the corpus. + +Create and validate each retriever independently before combining them. Follow [Vector search](vector-search.md) and [Full-text search](full-text-search.md) for their indexes, query operators, and tuning. + +Reciprocal Rank Fusion (RRF) is the approach in the Lakebase Search get-started guide and a useful default because it combines ranks instead of incomparable raw distances and scores. It is not the only option: weighted rank fusion, normalized score fusion, or a reranker may fit applications with different relevance signals. + +## RRF Example + +For rank `r` and constant `k`, each retriever contributes `1 / (k + r)`. The documented starting point uses 40 candidates per retriever and `k = 60`; tune both for the corpus and workload. + +Bind the query embedding as `$1`, query text as `$2`, and final result count as `$3`: + +```sql +WITH vector_ranked AS ( + SELECT id, RANK() OVER (ORDER BY distance) AS rank + FROM ( + SELECT id, embedding <=> $1::vector AS distance + FROM documents + ORDER BY distance + FETCH FIRST 40 ROWS WITH TIES + ) AS vector_candidates +), +keyword_ranked AS ( + SELECT id, RANK() OVER (ORDER BY score) AS rank + FROM ( + SELECT + id, + body_tsv <@> to_bm25query( + to_tsvector('english', $2), + 'documents_body_bm25'::regclass + ) AS score + FROM documents + ORDER BY score + FETCH FIRST 40 ROWS WITH TIES + ) AS keyword_candidates +) +SELECT + d.id, + d.title, + COALESCE(1.0 / (60 + v.rank), 0) + + COALESCE(1.0 / (60 + k.rank), 0) AS rrf_score +FROM documents AS d +LEFT JOIN vector_ranked AS v ON v.id = d.id +LEFT JOIN keyword_ranked AS k ON k.id = d.id +WHERE v.id IS NOT NULL OR k.id IS NOT NULL +ORDER BY rrf_score DESC, d.id +LIMIT $3; +``` + +`RANK()` gives tied retrieval scores the same rank. Sort by `rrf_score` descending and use the stable ID as a final tie-breaker. + +`FETCH FIRST ... ROWS WITH TIES` keeps every candidate tied at the cutoff, so `RANK()` receives the complete boundary tie group. The candidate set can therefore exceed 40 rows. `lakebase_bm25.default_limit` defaults to `1000`; increase it only when the BM25 candidate set needs to exceed that value. + +## Adapt the Hybrid Search + +- Retrieve more candidates from each source than the final result count; otherwise one retriever can dominate before fusion has enough overlap. Keep `lakebase_bm25.default_limit` above the BM25 candidate target and allow room for boundary ties. +- Keep each retriever's operator and index configuration correct independently before tuning RRF. +- Tune candidate counts and the RRF constant with judged or behavioral relevance data, plus latency measurements. +- Add weights only when product evidence shows one retriever should contribute more. Weight the reciprocal-rank contributions, not the raw vector distance and negative BM25 score. +- Apply the same access-control and tenant filters to both candidate CTEs. If BM25 filters are strict and cheap, evaluate whether `lakebase_bm25.prefilter` improves the filtered query. + +Source: [Lakebase Search get-started guide][lakebase-search-guide]. + +[lakebase-search-guide]: https://neon.com/docs/ai/lakebase-search-get-started#combine-results-with-hybrid-search diff --git a/.agents/skills/neon-postgres/references/vector-search.md b/.agents/skills/neon-postgres/references/vector-search.md new file mode 100644 index 0000000..813f067 --- /dev/null +++ b/.agents/skills/neon-postgres/references/vector-search.md @@ -0,0 +1,137 @@ +# Semantic Vector Search + +Use `lakebase_vector` for approximate nearest-neighbor retrieval over embeddings. It retains pgvector's vector types, distance operators, and query syntax; the index access method is `lakebase_ann`. + +## Contents + +- [Create the extension](#create-the-extension) — enable `lakebase_vector` and its `pgvector` dependency +- [Prepare embeddings](#prepare-embeddings) — define the vector column and keep embedding dimensions consistent +- [Build the index](#build-the-index) — match the distance metric, operator class, and query operator +- [Tune the index](#tune-the-index) — configure index-build options and concurrent index management +- [Query](#query) — rank by vector distance or filter by a similarity radius +- [Tune search](#tune-search) — inspect the index and tune recall against latency + - [Use prefilter selectively](#use-prefilter-selectively) — apply selective filters before ANN scoring + +## Create the Extension + +Lakebase Search requires Postgres 16 or later. Enable the extension before creating vector columns or indexes: + +```sql +CREATE EXTENSION IF NOT EXISTS lakebase_vector CASCADE; +``` + +`lakebase_vector` installs `pgvector` through `CASCADE`. It relies on a preloaded library that Neon enables by default; if the project customized its preloaded-library list, confirm the library remains enabled. + +## Prepare Embeddings + +Use any embedding provider whose vector dimensions and distance metric match the schema and index: + +```sql +CREATE TABLE documents ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + title text NOT NULL, + body text NOT NULL, + embedding vector(1536) +); +``` + +Replace `1536` with the embedding model's dimension. Generate stored-document and query embeddings with the same model and preprocessing. Keep embedding generation outside SQL unless the architecture already provides an in-database embedding function. + +## Build the Index + +Choose the operator class and query operator as a matched pair: + +| Metric | Common use | Operator class | Distance operator | +| --- | --- | --- | --- | +| Cosine | Most text embeddings | `vector_cosine_ops` | `<=>` | +| L2 / Euclidean | Absolute distance matters; vectors do not need normalization | `vector_l2_ops` | `<->` | +| Inner product | Unit-normalized vectors; matches cosine for unit vectors | `vector_ip_ops` | `<#>` | + +```sql +CREATE INDEX documents_embedding_ann ON documents + USING lakebase_ann (embedding vector_cosine_ops); +``` + +## Tune the Index + +The default index options suit most workloads: + +- `build_mode = 'standard'` balances recall and index build time. Use `quality` for better recall when a longer build is acceptable. +- `lists = 'auto'` chooses the IVF partition layout from the number of indexed vectors. Choose between `auto` and a manual value case by case: test both on the target dataset and use the value that better meets recall and performance targets. + +To prioritize recall over index build time: + +```sql +CREATE INDEX documents_embedding_ann_quality ON documents + USING lakebase_ann (embedding vector_cosine_ops) + WITH (build_mode = 'quality'); +``` + +To override the automatic partition layout instead: + +```sql +CREATE INDEX documents_embedding_ann_lists ON documents + USING lakebase_ann (embedding vector_cosine_ops) + WITH (lists = '1024'); +``` + +For a large table, use `CREATE INDEX CONCURRENTLY` to avoid locking out writes while creating the index. For a frequently changing table, periodically use `REINDEX INDEX CONCURRENTLY` to rebuild the index with minimal write locking. + +## Query + +Generate the query embedding with the same model and preprocessing used for stored documents, then bind it as a parameter: + +```sql +SELECT id, title, embedding <=> $1::vector AS distance +FROM documents +ORDER BY distance +LIMIT $2; +``` + +Distance sorts ascending: a smaller value is a closer match. Keep the query operator consistent with the index operator class. + +To filter by a similarity radius, use the matching boolean range operator in `WHERE` and the distance operator in `ORDER BY`: + +```sql +SELECT id, title +FROM documents +WHERE embedding <<=>> sphere($1::vector, 0.5) +ORDER BY embedding <=> $1::vector +LIMIT $2; +``` + +The cosine range operator `<<=>>` returns a boolean; do not use it as the ranking expression. + +## Tune Search + +Inspect the index before overriding defaults: + +```sql +SELECT lakebase_ann_index_info('documents_embedding_ann'); +``` + +This reports `lists`, `default_probes`, and `default_epsilon`. Small datasets use exact flat search before IVF lists are built. In that state, `lists` and `default_probes` are empty. Leave `lakebase_ann.probes` set to its default of `'auto'`; `lakebase_ann.epsilon` still controls full-precision reranking during flat search. + +For an IVF index, `lakebase_ann.probes` controls how many partitions are searched at each level. Higher values generally improve recall at the cost of speed. Its default is `'auto'`. When `lists` is not empty, the shape of `probes` must match the shape of `lists`: use one value for a one-level index or two comma-separated values for a two-level index. At each level, the `probes` value must be no larger than the corresponding `lists` value. A mismatched or out-of-range value causes an error. + +`lakebase_ann.epsilon` controls how many candidates are reranked using full-precision distances. Higher values rerank more candidates and take longer. Its default is `'auto'`, which works well for most workloads. + +### Use Prefilter Selectively + +By default, PostgreSQL applies non-vector filters after the ANN index returns candidate rows. Enable prefilter when a filter is cheap to evaluate and removes most rows. Leave it off for loose or expensive filters. + +```sql +BEGIN; +SET LOCAL lakebase_ann.prefilter = on; + +SELECT id, title +FROM documents +WHERE id % 100 = 0 +ORDER BY embedding <=> $1::vector +LIMIT $2; +COMMIT; +``` + +Start with `probes` and `epsilon` set to `'auto'`. Benchmark manual probe values against representative query embeddings and choose the smallest values that satisfy recall and tail-latency targets. Keep session settings and the query in the same transaction when using a connection pool or stateless driver. + +Source: [`lakebase_vector` documentation](https://neon.com/docs/extensions/lakebase-vector). diff --git a/.agents/skills/neon/SKILL.md b/.agents/skills/neon/SKILL.md new file mode 100644 index 0000000..aa74ef6 --- /dev/null +++ b/.agents/skills/neon/SKILL.md @@ -0,0 +1,490 @@ +--- +name: neon +description: >- + Overview of Neon, a complete set of cloud backend primitives for apps and + agents, spanning Lakebase Postgres, Auth, the Data API, Object Storage, Compute + Functions, and the AI Gateway. Start here to route to the right Neon skill, set + up the CLI or MCP server, and follow the branch-first workflow. Use when "Neon" + or "Lakebase Postgres" is mentioned, or when any of its individual capabilities + are the trigger: "object storage" or "S3", "buckets", "serverless functions", + "AI gateway", "call an LLM", "logs", "branch logs", "query logs", + "log export", "Loki", "Grafana", "observability", "telemetry", "postgres", + "database", or "backend". Also use when there is no Neon account yet, the + user cannot sign in or provide an API key right now and needs a project they + can claim later, or the user asks for a throwaway DATABASE_URL, Claimable Neon, + Claimable Postgres, neon.new, claimable.neon.tech, instant Postgres, a no-signup + database, temporary postgres, quick postgres, a no credit card database, or + npx neon-new. +metadata: + source: https://github.com/neondatabase/agent-skills/tree/main/skills/neon +--- + +# Neon + +Important: Neon is no longer just a Postgres provider. Neon is a complete set of cloud backend primitives built around Lakebase Postgres, for developers, startups, and agent platforms, from Databricks — bundling Lakebase Postgres, Auth (managed Better Auth), long-running Functions, Object Storage, and an AI Gateway, all instant, branchable, and serverless. + +**Lakebase Postgres** is the database itself. It is one product, reached two ways: through Neon, with a free plan and the full set of backend primitives above, or through Databricks, alongside the rest of the Databricks suite. Same infrastructure, same features, same engineering team. Call the database Lakebase Postgres, and use "Neon" for the brand and for this access path — not as a name for the database. + +A Neon branch is an isolated, copy-on-write clone of your data, taken from its current state or from a past state within your project's retained history window. You can freely modify or delete it. It shares data with its parent until writes cause it to diverge, and those writes are stored independently as deltas. + +## Backend Primitives + +Neon bundles several backend primitives for building apps and agents that all branch together: + +- **Lakebase Postgres** — Postgres that scales and branches with your app, built on the lakebase architecture: OLTP directly on cloud object storage, with storage decoupled from compute. _Generally available._ +- **Auth** — Managed Better Auth with users and sessions stored in Postgres. _Generally available._ +- **Object Storage** — S3-compatible object storage that branches with your projects. _Public beta._ +- **Functions** — Neon's compute offering: long-running serverless functions that run close to your database, for WebSocket servers, long agent HTTP streams, APIs, and server-sent event servers. _Public beta._ +- **AI Gateway** — One API for frontier and open-source models, supporting the chat completions API and the responses API, powered by Databricks Unity AI Gateway. _Public beta._ + +### Public Beta Service Availability + +Object Storage, Functions, and AI Gateway are in public beta. + +Beta access features are currently available on projects in `us-east-2` and `eu-central-1`. Before guiding a user through any of these services, confirm they are working in one of these regions. If not, they will need to create a new project in a supported region. + +## Architecture: How to Use Neon + +Neon is **not** a place to host your app frontend. Neon provides the backend primitives (Lakebase Postgres, Auth, Object Storage, Functions, AI Gateway) that **compose with** the application platform you already use. + +Recommended architectures: + +**Full-stack app on Vercel** (or Netlify) augmented with Neon — the app framework (Next.js, TanStack Start, etc.) owns your UI and routes and talks directly to your Neon services (Lakebase Postgres, Auth, Object Storage, Functions, AI Gateway). + +**Reach for Neon Functions when you outgrow the host's limits** — a WebSocket or SSE server, long-running agents, or an MCP server that risks timing out on short, lambda-style serverless functions. As long as there is an active connection, a Neon Function can run up to 24 hours without interruption, with the added benefit of running close to your data. + +**Move your whole backend control plane onto Neon Functions** — especially useful when the frontend is **client-only** rather than full-stack: TanStack Router, React Router in client mode, and similar SPAs hosted on Vercel or Netlify. The client talks **directly to Neon Functions**, where you build REST APIs and request/response agents. Secure these functions like any standalone REST API — verify a JWT or API key at the top of each handler (see the `neon-functions` skill). + +Because Functions are just your backend, they compose with a full-stack app that already has one (Next.js route handlers, etc.), too. + +## Neon Documentation + +The Neon documentation is the source of truth for all Neon-related information. Always verify claims against the official docs before responding. Neon features and APIs evolve, so prefer fetching current docs over relying on training data. + +### Finding the Right Page + +Look the page up before you fetch it — **don't guess URLs!** The docs index lists every available page with its URL and a short description: + +``` +https://neon.com/docs/llms.txt +``` + +### Fetching Docs as Markdown + +Any Neon doc page can be fetched as markdown in two ways: + +1. **Append `.md` to the URL** (simplest): https://neon.com/docs/introduction/branching.md +2. **Request `text/markdown`** on the standard URL: `curl -H "Accept: text/markdown" https://neon.com/docs/introduction/branching` + +Both return the same markdown content. Use whichever method your tools support. + +## Choosing the Right Skill + +Neon provides a set of agent skills in addition to the official documentation. When a task matches one of the rows below, work from that skill rather than from this overview. You may have some of these skills already installed, or you may need to install them. + +The skills below live in the [`neondatabase/agent-skills`](https://github.com/neondatabase/agent-skills) repo: + +| Skill | Use it for | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `neon-postgres` | Working with databases, including connections, schemas, queries, search, and autoscaling: SQL development, schema design, performance optimization, and scaling decisions. | +| `neon-postgres-branches` | Choosing or creating the right branch type for dev, preview, test, or CI workflows. Use this skill as a slash command. | +| `neon-object-storage` | Storing and serving files (uploads, images, blobs), including branching them with the database. | +| `neon-functions` | Deploying long-running or streaming serverless functions — APIs, agents, SSE/WebSocket servers. | +| `neon-ai-gateway` | Calling an LLM or routing across model providers with one credential, including discovering the branch's servable models at runtime via the OpenAI-compatible `/v1/models` endpoint. | +| `neon-postgres-egress-optimizer` | Diagnosing or fixing excessive Postgres egress (network data-transfer) costs in a codebase. | + +For guidance on agent platforms that provision and operate Lakebase Postgres on Neon at scale, use `neon-postgres-agent-platforms`, which lives in a separate repo: [`neondatabase/neon-for-agent-platforms`](https://github.com/neondatabase/neon-for-agent-platforms). + +### Installing the Right Skill + +First check whether the target skill is already installed and accessible (for example, it appears in the available skills list or its `SKILL.md` is present). If it is, use it directly. If it is not installed, install it via the `skills` CLI, if available, with `npx`/`bunx`: + +```bash +npx skills add neondatabase/agent-skills -s +``` + +Replace `` with the skill you need (for example, `neon-object-storage`, `neon-functions`, or `neon-ai-gateway`). Useful flags: + +- `-g` — install globally instead of into the current project. +- `-y` — non-interactive mode (skip prompts). +- `-a ` — pick the target agent(s) for non-interactive mode. + +For example, to install the object storage skill globally for a specific agent without prompts: + +```bash +npx skills add neondatabase/agent-skills -s neon-object-storage -g -y -a +``` + +If you don't have access to the `skills` CLI, you can visit https://neon.com/.well-known/agent-skills for a registry of all available Neon skills and fetch them manually. + +### Updating Skills + +Keep the skills up to date: for every new session, update them so you are working with the latest best practices. + +Use the same method that was used to install them. With the `skills` CLI, run the install command above with `update` in place of `add`, or run `npx skills update` to update all Neon skills. If the skills were installed via a plugin, they are updated automatically. + +## Getting Started with Neon + +Before `npx neon@latest init --agent`, check whether the CLI is already authenticated: + +- `NEON_API_KEY` is set +- `npx neon@latest profile list -o json` lists a profile whose `account` is not `-` + +A `DEFAULT` row with `account: "-"` and `file: "missing"` is not an account. If `neon` is not installed, or `npx neon@latest profile list` cannot run, that is not an account. + +If none of those hold, follow [Starting without a Neon account](#starting-without-a-neon-account). + +The easiest way to get started with Neon is to use our CLI and the project bootstrap wizard: + +```bash +npx neon@latest init --agent +``` + +Use the `--agent` flag to run in a non-interactive, state-machine mode. + +This init command will guide you through installation of suggested Neon development tools. Everything is customizable. The defaults are: + +- Neon CLI installed globally +- Neon MCP server installed globally +- Neon Agent skills installed into the project + +If `init` is run in an empty project, it will run the `bootstrap` command, offering to install one of our project templates. + +### Getting Started with the Neon CLI + +**Prefer the CLI over the MCP server** unless the user instructs otherwise, the CLI is unavailable or blocked in your environment, or it is not authenticated, since it provides more capabilities, including deploying Neon Functions. + +The above `init` command will install the Neon CLI, but the CLI can also be installed manually with `npm i -g neon` or `bun i -g neon`. For full CLI installation options, see https://neon.com/docs/cli/install.md + +#### Useful CLI Commands + +These commands are included in the `init` command but can be run manually as needed. + +1. `neon link` — Interactively links the workspace to a Neon org, project, and branch, writing the IDs to a git-ignored `.neon` file. Run once per project. Once linked, project- and branch-scoped commands no longer need `--project-id` or `--branch` (for example, `neon branch list`). +2. `neon checkout ` — Pins a different branch in `.neon`, creating it if it doesn't exist yet, and pulls that branch's env. It drives the [Branch-First Dev Flow](#branch-first-dev-flow) described below. +3. `neon config init` — Initializes a `neon.ts` file, which declares how you provision and manage Neon services, in the root of the project. +4. `neon env pull` — Fetches the current branch's Neon environment variables (`DATABASE_URL`, …) into your existing `.env`, or `.env.local` if you don't have one (override the target with `--file`). No branch ID needed; it reads `.neon`. **`link` and `checkout` run this for you by default**, so you rarely call it directly. + + Without `neon.ts` it pulls the vars of every service the branch actually has (Postgres, plus Neon Auth, the Data API, and bucket `AWS_*` once provisioned); with `neon.ts` it pulls only the services declared there and errors if the branch is missing one — and the AI Gateway vars are never pulled unless `neon.ts` declares `aiGateway`. + +### Getting Started with the Neon MCP Server + +The above `init` command will install the Neon MCP server globally, but it can also be installed manually using: `npx -y add-mcp https://mcp.neon.tech/mcp -g -n Neon -y -a ` or through your IDE plugin. + +For all available plugins, see: https://neon.com/docs/ai/ai-agents-tools.md + +For full MCP server installation options, see https://neon.com/docs/ai/connect-mcp-clients-to-neon.md + +Useful MCP tools to initialize a project: + +- `list_projects` — Lists the first 10 Neon projects in your account, providing a summary of each project. If you can't find a specific project, increase the limit by passing a higher value to the `limit` parameter. +- `create_project` — Creates a new Neon project in your Neon account. A project acts as a container for branches, databases, roles, and computes. +- `get_connection_string` — Returns your database connection string. + +## Starting without a Neon account + +If the Getting Started account check found credentials, use them. If a command waits on a browser (`Awaiting authentication in web browser`) or authentication fails, stop and ask the user to sign in (`neon auth`) or mint an API key. Prefer that over Claimable Neon unless they say otherwise. + +If they cannot sign in or provide a key right now, ask before using Claimable Neon. Continue only after they say yes. That is a temporary workaround. + +If there is no Neon account yet, follow [references/claimable-neon.md](https://neon.com/docs/ai/skills/neon/references/claimable-neon.md). Do not run `neon init --agent` or `neon auth` on this path; those need a human Neon account. If `neon claim` is missing, the reference has the REST fallback. Unclaimed projects expire at `project_expires_at` (72 hours today). Claim codes expire in `expires_in` (15 minutes today). Add Auth or the Data API with `neon.ts` and `neon deploy` before or after claim. + +Requests for neon.new, Claimable Postgres, claimable.neon.tech, instant Postgres, or a no-signup database are the same path. + +## Neon Infrastructure as Code + +`neon.ts` is Neon's branch config and infrastructure-as-code file: declare which Neon services your project's branches should have, get type-safe env vars, and program branch settings — all in TypeScript. It's the config layer for your Neon services, and it composes with the branch-first loop below. Add it with `@neon/config`: + +```bash +npm i @neon/config +``` + +```typescript +// neon.ts +import { defineConfig } from "@neon/config/v1"; + +export default defineConfig({ + preview: { + aiGateway: true, + buckets: { + images: { + access: "private", + }, + }, + functions: { + imagegen: { + name: "AI SDK image agent", + source: "src/index.ts", + }, + }, + }, +}); +``` + +### Provision services with neon config + +Every project ships with Lakebase Postgres; `neon.ts` lets you also declare Neon Auth and the Data API today, with Functions, buckets, and the AI Gateway under a `preview` block — every service for the branch composes in one file: + +```typescript +// neon.ts +export default defineConfig({ + auth: true, + dataApi: true, + preview: { + functions: {}, + buckets: {}, + aiGateway: true, // see the neon-ai-gateway skill + }, +}); +``` + +Reconcile the declaration from the CLI — the Neon equivalent of `terraform status` / `plan` / `apply`: + +```bash +neon status # print the branch's live config (read-only). Alias for `neon config status`. +neon config plan # dry-run diff of what apply would change (read-only) +neon deploy --env # apply neon.ts. Pass --env when Function env reads process.env. Alias for `neon config apply` +``` + +`apply` / `deploy` provision the declared services **and then pull the branch's env into your local `.env.local`** (e.g. `Pulled 5 Neon variables into .env.local: DATABASE_URL, …`), so your local env always matches what's deployed. + +### Function env and `neon deploy` + +`neon deploy` is the preferred full deployment: it applies `neon.ts` (services and functions) to the linked branch. `neon deploy --env ` loads that file into `process.env` before evaluating `neon.ts`, then uploads those values as Function env. Use it every time Function env reads `process.env`. + +`` is the gitignored file `neon env pull` already writes (`.env` if that file exists, otherwise `.env.local`). Env pull writes Neon-managed vars only (`DATABASE_URL`, `NEON_AI_GATEWAY_*`, …). Add every key under `preview.functions.*.env` to that file yourself, then pass the same path to `--env`. + +Every declared Function env key must be a defined string. `undefined` (an unset `process.env.X`) means you listed a key you want written but the value is missing: `defineConfig` throws. Omit the key from `neon.ts` if you do not want to write it. Never coerce a missing `process.env` value to an empty string: that uploads `""` and deletes the live key. An empty assignment in the file (`KEY=`) is also `""`. If TypeScript needs a type assertion, use `process.env.X!` and make sure the file actually has the value. + +Use `neon functions deploy` when you are not applying `neon.ts`: a single function by slug, or a targeted `--env KEY=VALUE` update (that flag is not a file path). + +### Type-safe env vars with parseEnv + +`@neon/env`'s `parseEnv` takes your `neon.ts` config object and returns a parsed, typed env object, validated against the services you declared. The shape of `env` follows your config, and missing variables are flagged with clear errors. + +```bash +npm i @neon/env +``` + +```typescript +import { parseEnv } from "@neon/env"; +import config from "./neon"; + +const env = parseEnv(config); + +console.log(env.postgres.databaseUrl); +console.log(env.auth.baseUrl); +``` + +By default `parseEnv` requires _every_ variable your config implies. When one of your apps only uses a subset, for example when you need to read `DATABASE_URL` but never the unpooled URL, pass an array of env-var keys to require and validate only those. The keys are typesafe: autocomplete only offers variables your config enables, and the returned shape is narrowed to exactly what you selected (so unselected variables are neither enforced nor present). + +```typescript +import { parseEnv } from "@neon/env"; +import config from "./neon"; + +// Only DATABASE_URL is required and returned; DATABASE_URL_UNPOOLED is not enforced. +const { postgres } = parseEnv(config, ["DATABASE_URL"]); +console.log(postgres.databaseUrl); + +// Selecting across services — only these keys are validated. +const env = parseEnv(config, ["DATABASE_URL", "NEON_AUTH_BASE_URL"]); +console.log(env.postgres.databaseUrl, env.auth.baseUrl); +``` + +### Branch configuration + +Beyond services, `neon.ts` can program what configuration _new_ branches receive via the `branch` property — a function of the branch being evaluated that returns its settings: + +```typescript +// neon.ts +import { defineConfig } from "@neon/config/v1"; + +export default defineConfig({ + auth: true, + dataApi: true, + branch: (branch) => { + if (branch.exists) { + // leave existing branches untouched + return {}; + } + if (branch.name.startsWith("dev")) { + return { + ttl: "7d", // clean up the branch after 7 days + postgres: { + computeSettings: { + autoscalingLimitMinCu: 0.25, // scale to zero + autoscalingLimitMaxCu: 1, // keep it cheap + suspendTimeout: "5m", + }, + }, + }; + } + return {}; + }, +}); +``` + +The `branch` function receives the target branch (its `name`, whether it `exists` yet, whether it's the default, and more) and returns the tuning you want. Here new `dev-*` branches get a 7-day TTL so they clean themselves up, plus a cheap scale-to-zero compute profile, while existing branches and everything else fall through to the defaults. Because `neon checkout` applies this policy on create, a fresh `dev-*` branch comes up with these settings already in place. + +### Type-safe config: invalid setups don't compile + +Because `neon.ts` is TypeScript, the compiler catches invalid infrastructure before you ever deploy — and Neon encodes the actual rules (and their fixes) into the types, so the error tells you what to do rather than failing with a useless `Type 'true' is not assignable to type 'never'`. The canonical case: the Data API verifies requests with Neon Auth by default, so enabling it on its own is a type error _on_ `dataApi`: + +```typescript +export default defineConfig({ + dataApi: true, // type error: `dataApi` (default authProvider 'neon') requires Neon Auth +}); +``` + +The message names both fixes, so pick one: + +```typescript +// 1. Enable Neon Auth (the default Data API auth provider): +export default defineConfig({ auth: true, dataApi: true }); + +// 2. Or verify a third-party IdP instead of Neon Auth: +export default defineConfig({ + dataApi: { + authProvider: "external", + jwksUrl: "https://your-idp/.well-known/jwks.json", + }, +}); +``` + +Treat a `neon.ts` type error as the config telling you which services must go together — read the message, it spells out the valid combinations. + +See https://neon.com/docs/reference/neon-ts.md for documentation on the `neon.ts` file. + +## Branch-First Dev Flow + +Neon branches enable a branch-first development flow, which we recommend when using Neon services. This and `neon.ts` above are the two halves of the recommended setup — `neon.ts` declares what every branch should have, and the branch-first loop is how you move between those branches day to day. Each works on its own, and they compose. + +Create a Neon branch any time you would create a git branch. Use the following commands if you have CLI access: + +- `neon checkout ` — Creates the branch if it doesn't exist, or checks out the existing one, by updating only the branch pointer in `.neon`. Run without a name for an interactive picker. It does not touch code or local Postgres. +- `neon env pull` — Fetches the current branch's Neon environment variables into your `.env` (see [Useful CLI Commands](#useful-cli-commands) above). **`link` and `checkout` run this for you by default**, so you rarely call it directly. +- `neon diff` — Shows the schema diff between the child branch and its parent. Run this to see what changes have been made to the schema since the last branch was created and before you commit your changes. + +```bash +neon link # once; also pulls the linked branch's env +neon checkout dev-add-search # per feature; also pulls the branch's env +``` + +Because `link` and `checkout` pull env by default, the branch's `DATABASE_URL` lands in your local `.env` automatically — build against it, then `checkout` the next branch and repeat. As the agent, drive this loop yourself: run `checkout` between tasks. + +### How checkout composes with neon.ts + +When a `neon.ts` is present, `neon checkout` applies your policy as it **creates** a branch, so a fresh branch comes up with its declared settings and services already in place. That create-apply does not load `--env`; if Function env reads `process.env`, run `neon deploy --env ` after checkout (add `--update-existing` if checkout already created the branch). Checking out an _existing_ branch never reconciles it — apply config changes to it explicitly with `neon deploy --env ` (alias for `neon config apply`). The bundled `env pull` also checks `neon.ts` against the linked branch and fails fast if the branch is missing a declared service, pointing you at `neon deploy --env ` to provision it, so your local env and the remote branch never drift apart silently. + +### Opting out of local env vars + +If env vars are injected at runtime instead of written to disk — or you simply don't want secrets in the working tree — pass `--no-env-pull` to `link` / `checkout` and supply the env another way: + +- `neon-env run -- ` (from `@neon/env`) fetches the branch's vars from your `neon.ts` and injects them into the child process at runtime — no `.env` file needed. This is the runtime counterpart to the on-disk `env pull`. +- `neon-env export` (from `@neon/env`) prints the branch's env to stdout as dotenv lines or, with `--format json`, JSON — for piping into another env manager rather than running a command. For example, [varlock](https://varlock.dev) can bulk-load it from a `.env.schema` with `@setValuesBulk(exec("neon-env export --format json"), format=json)`. +- `fetchEnv` from `@neon/env` is the programmatic version of the same thing: resolve the branch's env in code at runtime instead of shelling out to `neon-env run`. +- `neon dev` injects the same vars into your local dev server — it's part of Neon Functions local development (a public beta feature). + +When an agent should not write a local `.env`, instruct it (for example in your `AGENTS.md`) to run `neon checkout --no-env-pull` and rely on runtime injection. + +For reading env you _already_ have on disk (typed and validated against your `neon.ts`), use `parseEnv` — see [Type-safe env vars with parseEnv](#type-safe-env-vars-with-parseenv) above. + +## Observability + +Neon exposes branch-scoped logs. **Today they cover Neon Functions and Object Storage only.** Postgres computes and the AI Gateway are coming; until then, neither emits records. Logs are region-gated like the other beta services above. `us-east-2` and `eu-central-1` are enabled today. A branch that can't serve logs at all answers `404` with `reason: telemetry_not_enabled` (the message says whether it's the wrong region or a branch not collecting telemetry yet), versus a `200` empty result when the branch is enabled but has no records in the window; an unknown branch answers `reason: branch_not_found`. + +Use Neon CLI 3.1 or newer first. **Decide which branch you are querying.** Without `--branch`, the CLI uses the branch pinned in `.neon`, or the project's default branch when the workspace isn't linked. A deployed function or bucket usually lives on a different branch than the one checked out for development, so an empty result is more often the wrong branch than a missing log. + +```bash +neon logs query --since 1h +neon logs query --branch production --source function --minimum-severity error --since 6h +neon logs query --source storage --since 1h --output json +neon logs fields +neon logs field-values service_name --since 1h +``` + +`--source` accepts `function`, `storage`, and `pg_endpoint`, but only `function` and `storage` return records today — `pg_endpoint` is accepted and comes back empty until Postgres logs ship. The window defaults to 1h on `query` and 6h on `field-values`, and cannot exceed 7d on either. If Neon reports `--minimum-severity` as unsupported on a branch, use `--severity-text` instead (an exact, case-sensitive match, e.g. `ERROR`); severities vary by source, so confirm what a branch carries with `neon logs field-values severity_text`. Run `neon logs --help` for the full filter and pagination interface. + +`--logql` replaces the structured filters with a raw stream selector or line filter. Its stream label is `entity_type`, not `source`: + +```bash +neon logs query --since 1h --logql '{entity_type="function"} |= "timeout"' +``` + +If the CLI is unavailable, fall back to the Neon MCP server's read-only `query_logs`, `list_log_fields`, and `list_log_field_values` tools. + +In TypeScript applications, use `@neon/sdk`. Project and branch are positional, and `query` returns a lazy paginated iterable rather than a promise: + +```typescript +for await (const record of neon.logs.query(projectId, branchId, { + since: "1h", + source: "function", +})) { + console.log(record.timestamp, record.severity_text, record.message); +} + +const { data: fields } = await neon.logs.fields(projectId, branchId); +const { data: serviceNames } = await neon.logs.fieldValues( + projectId, + branchId, + "service_name", +); +``` + +`query`'s iterator always throws on error, but `fields` and `fieldValues` follow the client's `throwOnError`, which defaults to `false` and hands back `{ data, error }`. `fieldValues` resolves to the whole response, not a bare array: read `serviceNames.values`, and treat them as an arbitrary subset whenever `serviceNames.is_truncated` is true. + +### Loki-compatible read API + +For direct HTTP reads, authenticate with `Authorization: Bearer ` and use this branch-scoped base URL: + +```text +https://console.neon.tech/telemetry/v1/projects/{projectId}/branches/{branchId}/loki +``` + +The available endpoints are: + +- `GET /api/v1/query_range` +- `GET /api/v1/labels` +- `GET /api/v1/label/{name}/values` + +This is a read-only Loki-compatible subset, not a push endpoint or complete Loki deployment. `query_range` supports LogQL stream selectors and line filters, plus `since` or `start`/`end`, `limit`, and `direction`; it does not support aggregations, parsers, or formatting stages. + +The paths above are the ones to call directly. A Loki client that builds its own paths — a Grafana data source appends `/loki/api/v1` to whatever URL it is given — may need a different root, so confirm the data-source URL against the Neon docs rather than pasting this base. + +## Manage Neon Resources + +Recommended: Use `@neon/sdk` to manage Neon resources programmatically, such as creating projects, branches, and snapshots for dev scripts, CI/CD automations, and platforms building on top of Neon. + +`@neon/sdk` is the official TypeScript client for the [Neon API](https://neon.com/docs/reference/api-reference.md): **Fetch-based, zero-dependency, ESM-only**, generated from Neon's [OpenAPI spec](https://neon.com/api_spec/release/v2.json) with an ergonomic layer on top. It is the successor to [`@neondatabase/api-client`](https://www.npmjs.com/package/@neondatabase/api-client) (axios-based, generated-only). The old client is **not deprecated** and is safe to keep using, but new code should prefer `@neon/sdk`. + +### Neon for (Agentic) Platforms + +If you're building agents that generate apps from prompts, your users want to build apps, not manage databases. Industry-leading platforms like Replit and V0 create databases on Neon because it aligns with how agents work: an instant, branchable, serverless Lakebase Postgres data layer, invisible to users. + +Neon features for agents: + +- Instant Provisioning: your users never wait for infrastructure. +- Snapshots: let users toggle between checkpoints of code and state together. +- Low cost-per-Database: automatic scale to zero and 350ms cold starts. +- Full-Stack, Batteries-Included: Neon Auth, Data API included at no added charge. +- Granular API Controls: Track and control usage for flexible limits and invoicing. + +All details here: https://neon.com/programs/agents.md + +Apply for the Neon Agent Program for special program pricing here: https://neon.com/programs/agents + +## Gotchas + +### Neon Auth: "invalid domain" + +Neon Auth only redirects back to domains on its trusted-domains list. Anytime the domain your app runs on changes — a new production custom domain, a new deploy/preview URL, moving from `localhost` to a hosted environment, and so on — you must register the new domain with Neon Auth. Otherwise sign-in and OAuth callbacks fail with an **`invalid domain`** error because the redirect target isn't trusted. + +The easiest way to fix this is the CLI. With the workspace linked to the project (see the branch-first flow above), add the new domain to the trusted list: + +```bash +neon neon-auth domain add # e.g. neon neon-auth domain add https://app.example.com +neon neon-auth domain list # verify what's currently trusted +neon neon-auth domain delete # remove one you no longer use +``` + +If the workspace isn't linked, pass `--project-id ` (and `--branch `) explicitly. For local development, `neon neon-auth domain allow-localhost` manages whether `localhost` is permitted. Register the domain before pointing users at the new URL, so they never hit the `invalid domain` error. diff --git a/.agents/skills/neon/references/claimable-neon.md b/.agents/skills/neon/references/claimable-neon.md new file mode 100644 index 0000000..d0f8df7 --- /dev/null +++ b/.agents/skills/neon/references/claimable-neon.md @@ -0,0 +1,121 @@ +# Claimable Neon + +Claimable Neon provisions a temporary Neon project — Lakebase Postgres, and optionally the Data API and Managed Better Auth — before a human creates an account. The agent holds an identity assertion, not a Neon API key. A human can later claim the project into their organization. + +This flow follows the [auth.md](https://claimable.neon.tech/auth.md) protocol. Fetch `https://claimable.neon.tech/auth.md` for request and response fields. REST is on `https://claimable.neon.tech`. Use the table below; do not invent other identity paths. + +Use this after the neon skill account check found no account. + +## Path + +1. Install the CLI: `npm i -g neon@latest` +2. If `neon claim --help` does not list `create`, skip to [If neon claim is missing](#if-neon-claim-is-missing). +3. Write a `neon.ts` that declares the services you need, or skip the file and pass `--service` on create. Postgres is always requested. +4. Create the project: `neon claim create --env-pull` (add `--service data-api --service auth` if there is no `neon.ts`) +5. If create did not write env, pull it: `neon env pull` +6. Use the `neon-postgres` skill for connections, schemas, and queries. Install it if it is missing: `neon skills -s neon-postgres` + +Do not run `neon init --agent` or `neon auth` on this path; those need a human Neon account. `--api-key` and `--profile` are refused on `neon claim`. + +```bash +npm i -g neon@latest +neon claim --help +``` + +If that help lists `create` and you need Auth or the Data API, `npm i @neon/config` and write `neon.ts`. Then `neon claim create --env-pull`. + +```typescript +import { defineConfig } from "@neon/config/v1"; + +export default defineConfig({ + auth: true, + dataApi: true, +}); +``` + +`neon claim create` reads `neon.ts` when it is present. It writes provisioned vars to an existing `.env`, otherwise `.env.local`, and gitignores that file. If `.env` or `.env.local` already has a `DATABASE_URL` (or other Neon-managed keys), pass `--file ` or `--no-env-pull`. The identity assertion is the pre-claim credential. + +Before claim, Postgres is always granted; Auth and the Data API are granted when requested. Functions, Object Storage, and AI Gateway come back with `granted: false` and `reason: "requires_claim"`. The CLI prints those as `denied_capabilities`. Report what you were given. Do not retry or strip them. + +After create, report the `project_id`, `project_expires_at`, and any denied capabilities. Do not invent the window. Unclaimed projects expire at `project_expires_at` (72 hours today). That clock is independent of the claim code. + +## Claim + +Do not mint a claim URL until the human is ready. Opening the URL does not freeze access. Continuing to Neon starts the transfer and rotates `DATABASE_URL`. Existing access tokens are revoked. Auth and the Data API stay enabled when they were granted. + +A claim code expires in `expires_in` seconds (15 minutes / 900 today). If the unused code expires, mint another: `neon claim accept --no-open` or `POST /v1/projects/{id}/claim`. Each mint cancels the previous unused code. You can mint several times; only the latest unused code works. Re-issue only while `project_expires_at` is still in the future. + +Continuing to Neon starts a transfer with a new 15-minute window and leaves the project key and database password revoked. If that window expires before the human accepts, mint again. Do not restore pre-claim `DATABASE_URL`. + +When `reconciled` is true, the pre-claim `DATABASE_URL` no longer works. Auth and Data API URLs stay if they were granted. The human signs in with `neon auth`. Then the agent runs `neon link` and `neon env pull` to write the new `DATABASE_URL`. `neon link` discovers the project after that sign-in. + +Auth and the Data API stay off unless requested at create or enabled later. On the unclaimed project, `neon.ts` plus `neon deploy` enables them. After claim, the same config talks to Neon directly. An external JWKS is only accepted after claim. Data API with the default auth provider requires Auth: + +```typescript +import { defineConfig } from "@neon/config/v1"; + +export default defineConfig({ + auth: true, + dataApi: true, +}); +``` + +```bash +neon deploy +``` + +```typescript +export default defineConfig({ + dataApi: { + authProvider: "external", + jwksUrl: "https://example.com/.well-known/jwks.json", + }, +}); +``` + +`neon checkout` does not apply this to an existing branch. `neon deploy` (alias of `neon config apply`) does. + +### With the CLI + +When the human is ready, run `neon claim accept --no-open`. Bare `neon claim accept` opens a browser. Report the `verification_url`, `user_code`, and `expires_in_seconds` the CLI printed (HTTP names: `verification_uri_complete`, `user_code`, `expires_in`). If the code expires, run `neon claim accept --no-open` again. Poll with `neon claim status`. The CLI re-exchanges the assertion; do not call the token endpoint yourself. + +```bash +neon claim accept --no-open +neon claim status +``` + +Permanently delete the unclaimed project (this does not cancel a claim): + +```bash +neon claim delete --yes +``` + +### With REST + +An agent must not complete the claim. Do not `POST /v1/projects/{id}/claim` until the human is ready. The human opens `verification_uri_complete` and accepts the transfer. If the claim code expires, `POST /v1/projects/{id}/claim` again. Each POST replaces the unused previous code. If the human continued to Neon and that transfer expired, POST again for a new code. The live claim response also includes `user_code` and `expires_in`. `auth.md` documents `verification_uri_complete` and the polling `interval`. + +After the human continues to Neon, existing access tokens are revoked: re-exchange the identity assertion, then poll `GET /v1/projects/{id}/claim` with that token at the interval `auth.md` returns. `claim_in_progress` on a new mint means the transfer window is still live: poll, do not mint. After that window expires, POST claim again. Report `verification_uri_complete`, `user_code`, and `expires_in`. + +When `error.code` is `capability_requires_claim`, preserve the denied capability and give the human a claim link instead of retrying or silently omitting it. + +Only `invalid_grant`, `project_expired`, and `project_claimed` mean the stored identity assertion is dead. `token_expired` means re-exchange the assertion. + +## If neon claim is missing + +Fall back to the REST API. Fetch `https://claimable.neon.tech/auth.md` for request and response fields. The claimable resource is `/v1/projects/{id}` on `https://claimable.neon.tech`, not `/v1/databases/{id}`. Follow [Claim](#claim) for when to mint, what rotates, and what to do after `reconciled`. + +```http +POST https://claimable.neon.tech/v1/agent/identity +POST https://claimable.neon.tech/v1/oauth2/token +GET https://claimable.neon.tech/v1/projects/{id}/credentials +POST https://claimable.neon.tech/v1/projects/{id}/claim +GET https://claimable.neon.tech/v1/projects/{id}/claim +DELETE https://claimable.neon.tech/v1/projects/{id} +``` + +| CLI | REST | +| ----------------------------- | ------------------------------------------------------------------------------------------------- | +| `neon claim create` | `POST /v1/agent/identity`, then `POST /v1/oauth2/token`, then `GET /v1/projects/{id}/credentials` | +| `neon claim accept --no-open` | `POST /v1/projects/{id}/claim` | +| `neon claim status` | `GET /v1/projects/{id}/claim` | +| `neon claim delete --yes` | `DELETE /v1/projects/{id}` | diff --git a/.secrets.baseline b/.secrets.baseline index 9838a54..ac9550d 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -206,15 +206,6 @@ "line_number": 276 } ], - "docs/architecture.md": [ - { - "type": "Secret Keyword", - "filename": "docs/architecture.md", - "hashed_secret": "9352df1c0850d2e36c7f44567e1c8e124d7b0d92", - "is_verified": false, - "line_number": 280 - } - ], "docs/inventory/services.yaml": [ { "type": "Secret Keyword", @@ -282,6 +273,22 @@ "line_number": 13 } ], + "skills-lock.json": [ + { + "type": "Hex High Entropy String", + "filename": "skills-lock.json", + "hashed_secret": "c7884a9b2aaa88f23ab62317cc86760c71eb8b24", + "is_verified": false, + "line_number": 8 + }, + { + "type": "Hex High Entropy String", + "filename": "skills-lock.json", + "hashed_secret": "9a1592503753151fd8ba441bd5e591c3d49eba46", + "is_verified": false, + "line_number": 14 + } + ], "tests/conftest.py": [ { "type": "Hex High Entropy String", @@ -459,5 +466,5 @@ } ] }, - "generated_at": "2026-07-24T09:09:27Z" + "generated_at": "2026-09-05T09:15:14Z" } diff --git a/skills-lock.json b/skills-lock.json new file mode 100644 index 0000000..4da0295 --- /dev/null +++ b/skills-lock.json @@ -0,0 +1,17 @@ +{ + "version": 1, + "skills": { + "neon": { + "source": "neondatabase/agent-skills", + "sourceType": "github", + "skillPath": "skills/neon/SKILL.md", + "computedHash": "cd40530aa01dce7af2b9b77afb5d9452b13c3d6d1b6690af8b38605f17d1eeaf" + }, + "neon-postgres": { + "source": "neondatabase/agent-skills", + "sourceType": "github", + "skillPath": "skills/neon-postgres/SKILL.md", + "computedHash": "64150370145c18f563fbc652c8d8bf273c4943d2a9f2039ddd51af9fd08d96f3" + } + } +}