feat: external IVF-PQ vector index over parquet (Phase 1) — RFC draft

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "7.0.0-beta.15"
+current_version = "7.1.0-beta.1"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(-(?P<prerelease>(beta|rc))\\.(?P<prerelease_num>\\d+))?"
 serialize = [
     "{major}.{minor}.{patch}-{prerelease}.{prerelease_num}",

AGENTS.md

@@ -130,6 +130,11 @@ AWS_DEFAULT_REGION=us-east-1 pytest --run-integration python/tests/test_s3_ddb.p
 - Indent content under MkDocs admonition directives (`!!! note`, etc.) with 4 spaces.
 - Proofread comments and docs for typos before committing.
 
+## Pull Requests
+
+- PR titles must follow the Conventional Commits specification because `.github/workflows/pr-title.yml` validates the PR title and body with commitlint. Use prefixes like `feat:`, `fix:`, `docs:`, `perf:`, `ci:`, `test:`, `build:`, `style:`, or `chore:`; add a scope when useful.
+- Before creating or updating a PR, run the lint checks for every touched language surface, even when they are expensive. For Rust changes, run `cargo fmt --all` and `cargo clippy --all --tests --benches -- -D warnings`. For Python changes, follow the environment workflow in `python/AGENTS.md` and run `uv run make lint` from `python/`. If a required lint check cannot be run, state the blocker explicitly in the PR summary.
+
 ## Review Guidelines
 
 Contributor and maintainer attention is the most valuable resource. Less is more.

Cargo.toml

@@ -30,7 +30,7 @@ resolver = "3"
 
 
 [workspace.package]
-version = "7.0.0-beta.15"
+version = "7.1.0-beta.1"
 edition = "2024"
 authors = ["Lance Devs <dev@lance.org>"]
 license = "Apache-2.0"
@@ -55,25 +55,25 @@ rust-version = "1.91.0"
 [workspace.dependencies]
 arc-swap = "1.7"
 libc = "0.2.176"
-lance = { version = "=7.0.0-beta.15", path = "./rust/lance", default-features = false }
-lance-arrow = { version = "=7.0.0-beta.15", path = "./rust/lance-arrow" }
-lance-core = { version = "=7.0.0-beta.15", path = "./rust/lance-core" }
-lance-datafusion = { version = "=7.0.0-beta.15", path = "./rust/lance-datafusion" }
-lance-datagen = { version = "=7.0.0-beta.15", path = "./rust/lance-datagen" }
-lance-encoding = { version = "=7.0.0-beta.15", path = "./rust/lance-encoding" }
-lance-file = { version = "=7.0.0-beta.15", path = "./rust/lance-file" }
-lance-geo = { version = "=7.0.0-beta.15", path = "./rust/lance-geo" }
-lance-index = { version = "=7.0.0-beta.15", path = "./rust/lance-index" }
-lance-io = { version = "=7.0.0-beta.15", path = "./rust/lance-io", default-features = false }
-lance-linalg = { version = "=7.0.0-beta.15", path = "./rust/lance-linalg" }
-lance-namespace = { version = "=7.0.0-beta.15", path = "./rust/lance-namespace" }
-lance-namespace-impls = { version = "=7.0.0-beta.15", path = "./rust/lance-namespace-impls" }
+lance = { version = "=7.1.0-beta.1", path = "./rust/lance", default-features = false }
+lance-arrow = { version = "=7.1.0-beta.1", path = "./rust/lance-arrow" }
+lance-core = { version = "=7.1.0-beta.1", path = "./rust/lance-core" }
+lance-datafusion = { version = "=7.1.0-beta.1", path = "./rust/lance-datafusion" }
+lance-datagen = { version = "=7.1.0-beta.1", path = "./rust/lance-datagen" }
+lance-encoding = { version = "=7.1.0-beta.1", path = "./rust/lance-encoding" }
+lance-file = { version = "=7.1.0-beta.1", path = "./rust/lance-file" }
+lance-geo = { version = "=7.1.0-beta.1", path = "./rust/lance-geo" }
+lance-index = { version = "=7.1.0-beta.1", path = "./rust/lance-index" }
+lance-io = { version = "=7.1.0-beta.1", path = "./rust/lance-io", default-features = false }
+lance-linalg = { version = "=7.1.0-beta.1", path = "./rust/lance-linalg" }
+lance-namespace = { version = "=7.1.0-beta.1", path = "./rust/lance-namespace" }
+lance-namespace-impls = { version = "=7.1.0-beta.1", path = "./rust/lance-namespace-impls" }
 lance-namespace-datafusion = { version = "=7.0.0-beta.9", path = "./rust/lance-namespace-datafusion" }
-lance-namespace-reqwest-client = "0.7.5"
-lance-tokenizer = { version = "=7.0.0-beta.15", path = "./rust/lance-tokenizer" }
-lance-table = { version = "=7.0.0-beta.15", path = "./rust/lance-table" }
-lance-test-macros = { version = "=7.0.0-beta.15", path = "./rust/lance-test-macros" }
-lance-testing = { version = "=7.0.0-beta.15", path = "./rust/lance-testing" }
+lance-namespace-reqwest-client = "0.7.7"
+lance-tokenizer = { version = "=7.1.0-beta.1", path = "./rust/lance-tokenizer" }
+lance-table = { version = "=7.1.0-beta.1", path = "./rust/lance-table" }
+lance-test-macros = { version = "=7.1.0-beta.1", path = "./rust/lance-test-macros" }
+lance-testing = { version = "=7.1.0-beta.1", path = "./rust/lance-testing" }
 approx = "0.5.1"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false, features = ["prettyprint"] }
@@ -99,7 +99,7 @@ half = { "version" = "2.1", default-features = false, features = [
     "num-traits",
     "std",
 ] }
-lance-bitpacking = { version = "=7.0.0-beta.15", path = "./rust/compression/bitpacking" }
+lance-bitpacking = { version = "=7.1.0-beta.1", path = "./rust/compression/bitpacking" }
 bitpacking = "0.9"
 bitvec = "1"
 bytes = "1.11.1"
@@ -139,7 +139,7 @@ deepsize = "0.2.0"
 dirs = "6.0.0"
 either = "1.0"
 fst = { version = "0.4.7", features = ["levenshtein"] }
-fsst = { version = "=7.0.0-beta.15", path = "./rust/compression/fsst" }
+fsst = { version = "=7.1.0-beta.1", path = "./rust/compression/fsst" }
 futures = "0.3"
 geoarrow-array = "0.8"
 geoarrow-schema = "0.8"

docs/pyproject.toml

@@ -3,7 +3,7 @@ name = "lance-docs"
 version = "0.1.0"
 description = "Documentation for Lance - Modern columnar data format for ML and LLMs"
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.10,<3.11"
 dependencies = [
     "mkdocs>=1.5.0",
     "mkdocs-material>=9.4.0",

docs/src/format/index/system/mem_wal.md

@@ -4,6 +4,8 @@ The MemWAL Index is a system index that serves as the centralized structure for
 It stores configuration (shard specs, indexes to maintain), merge progress, and shard state snapshots.
 
 A table has at most one MemWAL index.
+The table may be a primary-key table or an append-only table without primary-key metadata.
+Primary-key-dependent lookup and deduplication semantics only apply when a primary key is defined.
 
 For the complete specification, see:
 

docs/src/format/table/layout.md

@@ -20,7 +20,8 @@ A Lance dataset in its basic form stores all files within the dataset root direc
     data/
         *.lance           -- Data files containing column data
     _versions/
-        *.manifest        -- Manifest files (one per version)
+        *.manifest                -- Manifest files (one per version)
+        latest_version_hint.json  -- Optional hint of the latest version (see below)
     _transactions/
         *.txn             -- Transaction files for commit coordination
     _deletions/
@@ -201,3 +202,15 @@ Manifest files are stored in the `_versions/` directory with naming schemes that
 
 See [Manifest Naming Schemes](transaction.md#manifest-naming-schemes) for details on the V1 and V2 patterns and their implications for version discovery.
 
+### Version Hint
+
+The optional file `_versions/latest_version_hint.json` records the latest committed version as JSON:
+
+```json
+{"version": 42}
+```
+
+It exists to accelerate latest-version discovery on stores where listing `_versions/` is expensive: a reader can read the hint and probe higher versions with HEAD requests instead of listing the whole directory, falling back to a full listing if the hint is missing or stale.
+
+The hint is purely an optimization. It is always safe to delete, never affects correctness, and can be ignored by readers that don't understand it. Writers may choose not to write it.
+