\1", text)
- text = re.sub(r"\*\*([^*]+)\*\*", r"\1", text)
- text = re.sub(r'\[([^\]]+)\]\((https?://[^\s)"<>]+)\)', r'\1', text)
- return text
-
-
-def convert_table(lines: list[str]) -> str:
- rows = [l.strip().strip("|").split("|") for l in lines if l.strip().startswith("|")]
- if len(rows) < 2:
- return "".join(f"{inline_format(l)}
" for l in lines) - header = [c.strip() for c in rows[0]] - body = [[c.strip() for c in r] for r in rows[2:]] if len(rows) > 2 else [] - out = "| {inline_format(h)} | " - out += "
|---|
| {inline_format(cell)} | " - out += "
{code}{content}{text}
\n") - i += 1 - - close_list() - return "".join(out), toc - - -def build_sidebar(toc: list[dict]) -> str: - lines = ['', "") - return "\n".join(lines) - - -def build(source: Path = SOURCE) -> None: - if not source.exists(): - print(f"error: {source} not found", file=sys.stderr) - sys.exit(1) - - md = source.read_text(encoding="utf-8") - output = source.with_suffix(".html") - title_match = re.search(r"^# (.+)$", md, re.MULTILINE) - title = escape(title_match.group(1) if title_match else "CROWDB User Guide") - body, toc = convert_markdown(md) - sidebar = build_sidebar(toc) - - doc = f""" - - - - -Run an Iceberg REST catalog and its storage on one Linux amd64 host.
-For evaluation only; no production or upgrade guarantee.
-docker run -d --name crowdb-iceberg \
- -p 127.0.0.1:80:80 \
- crowdb/crowdb-iceberg:v0.1.0-devThe image repository is crowdb/crowdb-iceberg; examples use the version tag
v0.1.0-dev. Docker pulls the image if it is not present locally. Docker creates
an anonymous volume for the data.
-Check startup, then retrieve your client credentials:
-docker inspect --format '{{.State.Health.Status}}' crowdb-iceberg
-docker exec crowdb-iceberg crowdb-monitor credentials show --format envWait for healthy. Connect your Iceberg client to http://localhost using
ICEBERG_TOKEN from the credential output. Port 80 serves both the REST catalog
and Iceberg FileIO; no separate S3 port is needed for Iceberg.
-Stop and start the same container without losing its data:
-docker stop --time 120 crowdb-iceberg
-docker start crowdb-icebergThe quick start exposes only Iceberg to the host. The GUI is not ready for use
-and is not published. For data you want to reuse after deleting and recreating
-the container, use a named volume as shown below.
--d: run in the background.--name crowdb-iceberg: give the container a convenient name for later commands.-p 127.0.0.1:80:80: publish Iceberg on host loopback. Fixed host ports requireexplicit mapping; the image cannot publish them automatically.
--v crowdb-data:/opt/crowdb/data: keep data in a named volume. Docker createsit if needed; reuse it with the same image when recreating the container.
---restart unless-stopped: automatically restart after failure or Dockerdaemon restart, unless you explicitly stopped the container.
---stop-timeout 120: allow 120 seconds for shutdown before Docker sends SIGKILL.--log-driver json-file --log-opt max-size=30m --log-opt max-file=5:rotate Docker's captured logs separately from CROWDB's internal logs.
-Use this instead of the quick-start command:
-docker run -d --name crowdb-iceberg \
- -p 127.0.0.1:80:80 \
- -v crowdb-data:/opt/crowdb/data \
- --restart unless-stopped --stop-timeout 120 \
- --log-driver json-file --log-opt max-size=30m --log-opt max-file=5 \
- crowdb/crowdb-iceberg:v0.1.0-devDo not attach the same data volume to two running containers. A bind mount can
-replace the named volume, but its directory must be writable by UID/GID 10001.
-The credential command prints ICEBERG_URI and ICEBERG_TOKEN, plus credentials
for the optional independent S3 service. Keep this output private. Credentials
-remain the same when the data volume is reused.
-Set ICEBERG_URI and ICEBERG_TOKEN in your client environment using the values
from the credential command. With PyIceberg installed:
-import os
-from pyiceberg.catalog import load_catalog
-
-catalog = load_catalog(
- "crowdb",
- type="rest",
- uri=os.environ["ICEBERG_URI"],
- token=os.environ["ICEBERG_TOKEN"],
-)
-catalog.create_namespace_if_not_exists("demo")
-print(catalog.list_namespaces())Add -p 127.0.0.1:81:81 when creating the container if you also need the general
S3 API. Connect to http://localhost:81 with path-style addressing, region
us-east-1, and the printed AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY.
Uploading an S3 object does not register an Iceberg table.
-The examples provide local HTTP access. Changing host ports or using a remote
-client also requires reachable Iceberg FileIO URLs; changing only the client's
-catalog URI is insufficient. The container's ports are independent of bare-metal
-service defaults.
-docker logs --tail 100 crowdb-iceberg
-docker exec crowdb-iceberg crowdb-monitor liveness
-docker exec crowdb-iceberg crowdb-monitor readinessstarting: initialization or recovery is still running; inspect the logs.unhealthy: a required service is unavailable. Liveness checks the monitor;readiness checks the complete deployment. Successful probes exit zero.
-exited: inspect logs before restarting. Repeated child failures exhaust therestart budget. Docker's restart policy handles container exits; an unhealthy
-health check alone does not restart the container.
-configure a different host mapping and reachable client/FileIO endpoints.
-bootstrap manifests or secrets to bypass compatibility checks.
-CROWDB logs are under /opt/crowdb/data/log. Monitor events are in
monitor/monitor.log; child logs have service-specific directories. Log rotation
uses a 30 MiB target and five retained files per channel across child restarts.
-Docker's own log retention is configured separately, as in the extended example.
-volumes are not automatically reused by a newly created container; record the
-volume identity before deleting the original container.
-private permissions and sparse files. Restore with the exact image version;
-copying only disk images is insufficient.
-no host fault tolerance. Monitor actual filesystem space; configured disk
-capacity is not a usable-capacity guarantee.
-is disabled in this profile. Cross-version volume migration is not promised.
-acceptance coverage. Spark, Flink, Trino, dataframe workflows and selected ORC
-data are not certified by those checks.
-Core dumps can contain credentials and user data. Linux host kernel.core_pattern
controls their destination; the image does not change it. A Docker core ulimit
-alone does not guarantee a dump.
-working directory. The destination must be writable.
-support is absent. This image does not include an Apport agent; do not assume
- a report will appear under the host's /var/crash.
/var/lib/systemd/coredump; inspect with coredumpctl on the host.
The image retains function symbols but currently has no DWARF source-line
-information. Do not assume that the data volume contains usable core dumps.
-See the Linux core manual
-for host collection rules.
- -CROWDB is a distributed storage platform with an S3-compatible object data
-plane and a multi-group Multi-Paxos key-value foundation. This guide starts
-with the shortest usable path: create a persistent local S3 cluster and use
-bucket and object operations. The later sections cover the underlying KV
-cluster, physical topology, individual servers, upgrades, and recovery.
-CROWDB provides three user-facing interfaces:
-crowdb-web service provides a visual dashboardwith cluster topology, group health, a KV Operator panel (store/group
-selector, paginated scan, inline CRUD, demo data injection), and
-Swagger UI for browsing the OpenAPI spec of any registered
- crowdb-kv-server instance.
crowdb-cli s3 owns the local S3 cluster lifecycle and sends bucket/object requests directly to the cluster recorded by --root.
Lower-level management commands discover services through group 0 and call
-them directly. Output is a human-readable console transcript.
-console service exposes the lower-level cluster management API documented
-in §8.
-S3 examples use the loopback access endpoint. The first local cluster normally
-receives 127.0.0.1:16000; when that port is already assigned, use the
endpoint printed by s3 cluster start or `s3 cluster status.
S3_ENDPOINT=http://127.0.0.1:16000Before following the steps below:
-pixi run build from the repository root.target/release to PATH, or invoke the binaries by their full paths.cluster identity and contains configuration, logs, and data files.
-Choose a directory and start the cluster:
-S3_ROOT="$PWD/.crowdb-runtime/persistent/s3-local"
-crowdb-cli s3 cluster start --root "$S3_ROOT"An absent or empty directory creates a new file-backed cluster. A recognized
-cluster directory restarts the same cluster with its existing data and port
-assignments. A non-empty directory that is not a CROWDB cluster is rejected
-without modification.
-The command prints the S3 endpoint, Web management URL, running service count,
-and cluster root. The default local endpoints are:
-S3_ENDPOINT=http://127.0.0.1:16000
-WEB_URL=http://127.0.0.1:14000If the command prints different endpoints because a default port is already
-assigned, use the printed values. Open WEB_URL for the topology, service, and
cluster management console.
-Inspect the recorded processes without changing them:
-crowdb-cli s3 cluster status --root "$S3_ROOT"The CLI owns S3 mini-cluster lifecycle. The bundled Web service loads that
-mini-cluster's console registry and presents its running services.
-Create a bucket:
-crowdb-cli s3 bucket put --root "$S3_ROOT" photos
-curl -X PUT "$S3_ENDPOINT/photos"List buckets:
-crowdb-cli s3 bucket list --root "$S3_ROOT"
-curl "$S3_ENDPOINT/"GET one bucket and show its XML result:
-crowdb-cli s3 bucket get --root "$S3_ROOT" photos
-curl "$S3_ENDPOINT/photos"Remove an empty bucket:
-crowdb-cli s3 bucket delete --root "$S3_ROOT" photos
-curl -X DELETE "$S3_ENDPOINT/photos"Removing a non-empty bucket returns the S3 error and leaves its objects intact.
-Create the bucket used by the following examples, then upload an object from a
-file, literal text, generated random bytes, or standard input:
-crowdb-cli s3 bucket put --root "$S3_ROOT" documents
-
-crowdb-cli s3 object put --root "$S3_ROOT" \
- documents reports/hello.txt --file ./hello.txt
-
-crowdb-cli s3 object put --root "$S3_ROOT" \
- documents reports/text.txt --text 'object content'
-
-crowdb-cli s3 object put --root "$S3_ROOT" \
- documents reports/random.bin --random-size 1048576
-
-printf 'hello from CROWDB\n' | crowdb-cli s3 object put \
- --root "$S3_ROOT" documents reports/stdin.txt
-
-curl -X PUT --data-binary @hello.txt \
- "$S3_ENDPOINT/documents/reports/hello.txt"put creates a new object or replaces the bytes of an existing key.
Read an object to standard output or a file:
-crowdb-cli s3 object get --root "$S3_ROOT" \
- documents reports/hello.txt
-
-crowdb-cli s3 object get --root "$S3_ROOT" \
- documents reports/hello.txt --output ./downloaded.txt
-
-curl "$S3_ENDPOINT/documents/reports/hello.txt" \
- --output ./downloaded-with-curl.txtCheck that an object exists:
-crowdb-cli s3 object head --root "$S3_ROOT" \
- documents reports/hello.txt
-
-curl -I "$S3_ENDPOINT/documents/reports/hello.txt"Delete an object:
-crowdb-cli s3 object delete --root "$S3_ROOT" \
- documents reports/hello.txt
-
-curl -X DELETE "$S3_ENDPOINT/documents/reports/hello.txt"List the first 100 keys below a prefix:
-crowdb-cli s3 object list --root "$S3_ROOT" documents \
- --prefix reports/ --limit 100
-
-curl --get "$S3_ENDPOINT/documents" \
- --data-urlencode 'list-type=2' \
- --data-urlencode 'prefix=reports/' \
- --data-urlencode 'max-keys=100'When a response is truncated, pass its opaque continuation token unchanged:
-crowdb-cli s3 object list --root "$S3_ROOT" documents \
- --prefix reports/ --limit 100 --continuation "$TOKEN"
-
-curl --get "$S3_ENDPOINT/documents" \
- --data-urlencode 'list-type=2' \
- --data-urlencode 'prefix=reports/' \
- --data-urlencode 'max-keys=100' \
- --data-urlencode "continuation-token=$TOKEN"Ranges are inclusive. --range 3-9 returns seven bytes:
crowdb-cli s3 object get --root "$S3_ROOT" \
- documents reports/hello.txt --range 3-9
-
-curl -H 'Range: bytes=3-9' \
- "$S3_ENDPOINT/documents/reports/hello.txt"Stop every process while preserving the cluster directory and stored objects:
-crowdb-cli s3 cluster stop --root "$S3_ROOT"Restart from the same directory and read the same data:
-crowdb-cli s3 cluster start --root "$S3_ROOT"
-crowdb-cli s3 object get --root "$S3_ROOT" \
- documents reports/hello.txtPermanently stop the cluster, release its port assignments, and remove its
-directory:
-crowdb-cli s3 cluster delete --root "$S3_ROOT"delete is destructive. Use stop when the cluster must be started again.
Cluster lifecycle does not currently have an HTTP management endpoint.
-The remaining sections describe lower-level cluster and server administration.
-They are not required for the local S3 workflow above.
-Management CLI commands omit --system-ip and --system-port for brevity.
They default to the system-group discovery endpoint 127.0.0.1:10000; either
flag may point to any system-group node because leader discovery is automatic.
-The CROWDB_SYSTEM_IP/CROWDB_SYSTEM_PORT environment variables provide the
same overrides. The following console HTTP curl examples assume:
IP=127.0.0.1
-PORT=14000An S3 mini-cluster already starts crowdb-web with its persisted registry.
For a separately managed cluster, before using these commands:
-crowdb-web with crowdb-web --port 14000. Add --test-mode for anin-memory console configuration that is lost on restart.
-CROWDB_KV_SERVER_BIN when crowdb-kv-server is not next to crowdb-web and not available through PATH.
127.0.0.1 for amulti-machine deployment.
-KV server, diskdb, chunkdb, and diskio use TOML startup configuration. Valid
-templates are shipped in each server's conf/ directory. Values resolve in
this order: compiled defaults, then file values, then CLI options that were
-explicitly supplied. A CLI option you omit does not erase its file value.
-KV server and diskio make --config optional; diskdb and chunkdb require a
config path. A malformed, unreadable, or invalid named file stops startup
-instead of silently falling back to defaults. server.rpc_workers controls
the inbound RPC worker count (default 2 for the Rust servers and 4 for diskio),
-must be positive, and takes effect only after restart. File watchers may report
-static changes before restart, but the active listener does not change.
-Console local deployment generates node-specific config files and reuses the
-same paths when restarting services. Keep manually managed files with the
-server's data and deployment records; group 0 currently stores topology, not
-process configuration.
-Create a rack, add nodes, and deploy a server on each node. The
-deploy command starts crowdb-kv-server on the target node (via SSH
if ssh_user is set, or as a local subprocess otherwise). No manual
start needed.
-# Create a rack
-crowdb-cli rack add --id r1 --name "rack-one"
-
-# Register each node (repeat for n2, n3)
-crowdb-cli node add --id n1 --rack r1 --host 127.0.0.1
-
-# Deploy a crowdb-kv-server process on each node (repeat for n2, n3)
-crowdb-cli server deploy --node n1 --rest-port 2001 --rpc-port 20001# Create a rack
-curl -X POST "http://$IP:$PORT/api/racks" -H 'Content-Type: application/json' \
- -d '{"id":"r1"}'
-
-# Register each node (repeat for n2, n3)
-curl -X POST "http://$IP:$PORT/api/nodes" -H 'Content-Type: application/json' \
- -d '{"id":"n1","rack_id":"r1","host":"127.0.0.1","ssh_port":22,"ssh_user":""}'
-
-# Deploy a crowdb-kv-server process on each node (repeat for n2, n3)
-curl -X POST "http://$IP:$PORT/api/nodes/n1/server/deploy" \
- -H 'Content-Type: application/json' \
- -d '{"rest_port":2001,"rpc_port":20001}'Before creating data stores or groups, the cluster must be
-initialized. This creates the system group (store 0, group 0) which
-stores cluster topology metadata as KV entries, providing HA for
-the topology itself.
-# Initialize with all deployed nodes
-crowdb-cli cluster init --nodes n1,n2,n3curl -X POST "http://$IP:$PORT/api/cluster/init" \
- -H 'Content-Type: application/json' \
- -d '{"nodes":["n1","n2","n3"]}'This creates store 0 and group 0 on each selected node, wires remotes
-for multi-node, persists topology in console config, and writes
-hardware hierarchy + KV-cluster topology into group 0 via
-HardwareClient + KVClusterMetaClient (text-path keys, JSON
values). After initialization, data store/group creation is unblocked.
-For a single-node dev cluster, pass one node:
-crowdb-cli cluster init --nodes n1A store is the logical container that owns one or more groups.
-# Create a store on n1
-crowdb-cli store add --store-id 3 --nodes n1
-
-# Create a group with an initial replica on n1
-crowdb-cli paxos add \
- --store-id 3 --group-id 3 --replica-id 1 --nodes n1curl -X POST "http://$IP:$PORT/api/stores" -H 'Content-Type: application/json' \
- -d '{"store_id":3,"nodes":["n1"]}'
-
-curl -X POST "http://$IP:$PORT/api/stores/3/groups" -H 'Content-Type: application/json' \
- -d '{"group_id":3,"replica_id":1,"nodes":["n1"]}'If the cluster has not been initialized, store/group creation returns
-409 Conflict with a message directing you to run cluster init first.
crowdb-cli replica add \
- --store-id 3 --group-id 3 --node n2 --replica-id 2
-
-crowdb-cli replica add \
- --store-id 3 --group-id 3 --node n3 --replica-id 3curl -X POST "http://$IP:$PORT/api/stores/3/groups/3/replicas" \
- -H 'Content-Type: application/json' \
- -d '{"node_id":"n2","replica_id":2}'
-
-curl -X POST "http://$IP:$PORT/api/stores/3/groups/3/replicas" \
- -H 'Content-Type: application/json' \
- -d '{"node_id":"n3","replica_id":3}'The service orchestrates the full add-replica flow: creates the local
-group on the target node, wires remotes bidirectionally, and the new
-replica catches up via snapshot streaming before joining the voting set.
-# Check group health
-crowdb-cli paxos inspect --store-id 3 --group-id 3
-# Look for "leader=" and replica states
-
-# Put / Get
-crowdb-cli kv put --store-id 3 --group-id 3 \
- --key hello --value world
-
-crowdb-cli kv get --store-id 3 --group-id 3 --key hellocurl "http://$IP:$PORT/api/stores/3/groups/3"
-
-curl -X POST "http://$IP:$PORT/api/stores/3/groups/3/kv/put" \
- -H 'Content-Type: application/json' \
- -d '{"key":"hello","value":"world"}'
-
-curl "http://$IP:$PORT/api/stores/3/groups/3/kv/get?key=hello"All KV operations target a specific (store_id, group_id).
# Put
-crowdb-cli kv put --store-id 3 --group-id 3 --key user:1 --value alice
-
-# Get
-crowdb-cli kv get --store-id 3 --group-id 3 --key user:1
-
-# Delete
-crowdb-cli kv delete --store-id 3 --group-id 3 --key user:1
-
-# Prefix scan (list mode — fast, latest values, S3-list semantics)
-crowdb-cli kv scan --store-id 3 --group-id 3 --prefix user: --limit 100curl -X POST "http://$IP:$PORT/api/stores/3/groups/3/kv/put" \
- -H 'Content-Type: application/json' \
- -d '{"key":"user:1","value":"alice"}'
-
-curl "http://$IP:$PORT/api/stores/3/groups/3/kv/get?key=user:1"
-
-curl -X POST "http://$IP:$PORT/api/stores/3/groups/3/kv/delete" \
- -H 'Content-Type: application/json' \
- -d '{"key":"user:1"}'
-
-curl "http://$IP:$PORT/api/stores/3/groups/3/kv/scan?prefix=user:&limit=100"The Web UI KV Operator panel provides the same operations with a
-store/group selector, paginated scan, and inline editing.
-CROWDB provides two range-read modes for different use cases:
-kv scan) — the default scan. Fast, always returns thelatest value per key at each page's read point. S3-list semantics:
-each page is independently consistent, but a key can vanish (deleted
-between pages) or a value can drift (overwritten between pages) within
-a single logical scan. No server-side state beyond the per-page read
-barrier. Use for interactive listing, key discovery, and the KV
-Operator UI.
-snapshot create + snapshot scan) —point-in-time-consistent. Pins a frozen view of the keyspace at a
-specific slot; every page is served from the same frozen view. No key
-vanishes, no value drifts, no phantom keys appear. Use for backup,
-analytics, and any consumer that needs a consistent point-in-time
-view. See §3.2 below.
-A snapshot scan pins a point-in-time view of the keyspace. Creating a
-snapshot flushes the in-memory write buffer (L0) into the durable tree
-(L1), then pins L1 at the current applied slot. The snapshot is a frozen,
-immutable view. Iterating it is pure array traversal with no concurrency
-concerns. Each snapshot has a server-side handle with a lease (default 5
-minutes); the handle is reaped if the client disconnects, preventing
-unbounded pin retention.
-Create a snapshot:
-crowdb-cli snapshot create --store-id 3 --group-id 3
-# Returns: snapshot_handle=42, at_slot=12345List active snapshots:
-crowdb-cli snapshot list --store-id 3 --group-id 3
-# Returns: handle, at_slot, lease_remaining for each active snapshotScan a snapshot (paginated, same prefix/start_after/limit as list scan):
-# First page
-crowdb-cli snapshot scan --store-id 3 --group-id 3 \
- --handle 42 --prefix user: --limit 100
-
-# Next page (start_after = last key from previous page)
-crowdb-cli snapshot scan --store-id 3 --group-id 3 \
- --handle 42 --prefix user: --limit 100 \
- --start-after user:50Release a snapshot (free the pinned pages):
-crowdb-cli snapshot release --store-id 3 --group-id 3 --handle 42# Create
-curl -X POST "http://$IP:$PORT/api/stores/3/groups/3/snapshots"
-
-# List
-curl "http://$IP:$PORT/api/stores/3/groups/3/snapshots"
-
-# Scan
-curl "http://$IP:$PORT/api/stores/3/groups/3/snapshots/42/scan?prefix=user:&limit=100"
-
-# Release
-curl -X DELETE "http://$IP:$PORT/api/stores/3/groups/3/snapshots/42"GC and snapshots: the engine's garbage collector reclaims tombstones
-and stale versions with slot <= gc_watermark. Active snapshots protect
their pinned pages via refcount — GC never frees a page a live snapshot
-still references. Once a snapshot is released (or its lease expires), the
-next GC sweep can reclaim those pages. The GC watermark can be advanced
-explicitly via the management API to control retention:
-# Advance GC watermark (data with slot <= watermark becomes reclaimable)
-curl -X POST "http://$IP:$PORT/api/stores/3/groups/3/gc-watermark" \
- -H 'Content-Type: application/json' \
- -d '{"slot":12000}'# High-level summary (servers + store/group counts)
-crowdb-cli cluster status
-
-# Full topology (logical stores/groups/replicas + physical nodes/servers)
-crowdb-cli cluster topology
-
-# Inspect a specific store, group, or node
-crowdb-cli cluster inspect s3 # store 3
-crowdb-cli cluster inspect s3/g3 # group 3 in store 3
-crowdb-cli cluster inspect n1 # node n1# All nodes
-curl "http://$IP:$PORT/api/nodes"
-
-# All deployed servers
-curl "http://$IP:$PORT/api/servers"
-
-# A specific group
-curl "http://$IP:$PORT/api/stores/3/groups/3"
-# healthy: all replicas up, leader known
-# degraded: some replicas down, quorum + leader available
-# unavailable: quorum lostcrowdb-cli replica add --store-id 3 --group-id 3 --node n4 --replica-id 4curl -X POST "http://$IP:$PORT/api/stores/3/groups/3/replicas" \
- -H 'Content-Type: application/json' \
- -d '{"node_id":"n4","replica_id":4}'The new replica streams a snapshot from the leader, catches up, then
-joins the voting set automatically.
-crowdb-cli replica remove --store-id 3 --group-id 3 --replica-id 3curl -X DELETE "http://$IP:$PORT/api/stores/3/groups/3/replicas/3"If the target is the leader, the service asks it to step down first,
-waits for a new leader, then removes the replica.
-and RPC port.
- store/group configuration from conf/node-config.json on startup.
No --stores/--groups CLI args needed for normal restart:
crowdb-cli server deploy --node n1 --rest-port 2001 --rpc-port 20001 curl -X POST "http://$IP:$PORT/api/nodes/n1/server/deploy" \
- -H 'Content-Type: application/json' \
- -d '{"rest_port":2001,"rpc_port":20001}' If node-config.json is lost, fall back to explicit bootstrap args
by starting crowdb-kv-server manually with --stores/--groups/
--replica:
crowdb-kv-server \
- --management-addr 0.0.0.0 --management-port 2001 \
- --ports 20001 --election-profile default \
- --stores 3 --groups 3 --replica 1If the WAL and config directory were also lost, add the replacement as
-a new replica with a new replica ID instead of reusing the old one.
-Upgrade one node at a time. Wait for each node to rejoin and catch up
-before moving to the next.
-For each node:
- crowdb-cli server stop --node n1 curl -X POST "http://$IP:$PORT/api/nodes/n1/server/stop" configuration from conf/node-config.json on startup:
crowdb-cli server restart --node n1 curl -X POST "http://$IP:$PORT/api/nodes/n1/server/restart" If node-config.json is missing, start crowdb-kv-server manually
with explicit args:
- crowdb-kv-server \
- --management-addr 0.0.0.0 --management-port 2001 \
- --ports 20001 --election-profile default \
- --stores 3 --groups 3 --replica 1 --stores/--groups tells the server to reopen the WAL and rejoin
as a full member. --replica must match the assigned replica ID.
crowdb-cli cluster status
- crowdb-cli paxos inspect --store-id 3 --group-id 3 crowdb-cli kv get --store-id 3 --group-id 3 --key helloWhat to watch: after stopping a node, the remaining nodes elect a
-new leader. Wait for the group view to show a leader before proceeding.
-A brief latency spike during leader transition is normal.
-If two of three nodes fail, the remaining node cannot elect itself
-leader. Writes and linearizable reads block.
- auto-loads from conf/node-config.json; if the config is lost, fall
back to --stores/--groups/--replica args. This is always the
safest path.
-become leader by manually truncating the log. Only safe when the
-other nodes are permanently lost.
-Do not add a new node to a quorum-less group without first recovering
-leadership.
-CROWDB durability comes from the per-store WAL (--wal-root), the
per-node config cache (--config-root), and the durable KV engine
(--data-root). For disaster recovery, back up:
{wal-root}/store{store_id}/ for each store{config-root}/node-config.json — per-node store/group config cache{data-root}/store{store_id}/group{group_id}/ if using crowdb-treedurable KV engine
-Restore by placing these on the replacement node and starting the
-server. With node-config.json present, no --stores/--groups
bootstrap args are needed. If the config is lost, use explicit
---stores/--groups/--replica args to recover from WAL.
Local S3 commands use --root to identify the cluster root and discover its access endpoint:
crowdb-cli s3 cluster start --root <path> — create or restartcrowdb-cli s3 cluster status --root <path> — inspect process livenesscrowdb-cli s3 cluster stop --root <path> — stop and preserve datacrowdb-cli s3 cluster delete --root <path> — stop and delete permanentlycrowdb-cli s3 bucket put --root <path> <bucket>crowdb-cli s3 bucket delete --root <path> <bucket>crowdb-cli s3 bucket list --root <path>crowdb-cli s3 bucket get --root <path> <bucket>crowdb-cli s3 object put --root <path> <bucket> <key> [--file <path> | --text <content> | --random-size <bytes>]crowdb-cli s3 object get --root <path> <bucket> <key> [--output <file>] [--range <start-end>]crowdb-cli s3 object delete --root <path> <bucket> <key>crowdb-cli s3 object head --root <path> <bucket> <key>crowdb-cli s3 object list --root <path> <bucket> [--prefix <prefix>] [--limit <n>] [--continuation <token>]Lower-level management commands accept --system-ip <addr> (default
127.0.0.1) and --system-port <port> (default 10000).
crowdb-cli cluster status — servers + store/group summarycrowdb-cli cluster topology — full logical + physical hierarchycrowdb-cli cluster inspect <id> — s<sid>, s<sid>/g<gid>, s<sid>/g<gid>/r<rid>, or <node-id>
crowdb-cli cluster init --nodes n1,n2,... — initialize cluster (system group)crowdb-cli rack add --id <id> [--name <name>]crowdb-cli rack remove --id <id>crowdb-cli rack listcrowdb-cli node add --id <id> --rack <rack> [--host <host>] [--ssh-user <user>]crowdb-cli node remove --id <id>crowdb-cli node listcrowdb-cli node ping <node>crowdb-cli server deploy --node <id> --rest-port <p> --rpc-port <p>crowdb-cli server restart --node <id>crowdb-cli server stop --node <id>crowdb-cli server listcrowdb-cli store add --store-id <id> [--nodes n1,n2,...]crowdb-cli store remove --store-id <id>crowdb-cli store listcrowdb-cli store inspect --store-id <id>crowdb-cli paxos add --store-id <s> --group-id <g> --replica-id <r> --nodes n1,n2,...crowdb-cli paxos remove --store-id <s> --group-id <g>crowdb-cli paxos list --store-id <s>crowdb-cli paxos inspect --store-id <s> --group-id <g>crowdb-cli replica add --store-id <s> --group-id <g> --node <n> [--replica-id <r>]crowdb-cli replica remove --store-id <s> --group-id <g> --replica-id <r>crowdb-cli kv put --store-id <s> --group-id <g> --key <k> --value <v>crowdb-cli kv get --store-id <s> --group-id <g> --key <k>crowdb-cli kv delete --store-id <s> --group-id <g> --key <k>crowdb-cli kv scan --store-id <s> --group-id <g> --prefix <p> [--limit <n>] — list scan (fast, latest values, S3-list semantics)crowdb-cli snapshot create --store-id <s> --group-id <g> — pin a point-in-time snapshotcrowdb-cli snapshot list --store-id <s> --group-id <g> — list active snapshotscrowdb-cli snapshot scan --store-id <s> --group-id <g> --handle <h> --prefix <p> [--limit <n>] [--start-after <k>] — scan a pinned snapshotcrowdb-cli snapshot release --store-id <s> --group-id <g> --handle <h> — release a snapshotThese endpoints use S3_ENDPOINT, whose local default is
http://127.0.0.1:16000. Cluster lifecycle and benchmark operations do not
currently have HTTP endpoints.
-| Operation | Endpoint |
|---|---|
| List buckets | GET / |
| Create bucket | PUT /{bucket} |
| Inspect bucket | HEAD /{bucket} |
| Delete empty bucket | DELETE /{bucket} |
| Put or replace object | PUT /{bucket}/{key} |
| Get object | GET /{bucket}/{key} |
| Get inclusive range | GET /{bucket}/{key} with Range: bytes={start}-{end} |
| Inspect object | HEAD /{bucket}/{key} |
| Delete object | DELETE /{bucket}/{key} |
| List objects | GET /{bucket}?list-type=2&prefix=...&max-keys=... |
| Continue object list | GET /{bucket}?list-type=2&continuation-token=... |
| Operation | Endpoint |
|---|---|
| Initialize cluster | POST /api/cluster/init |
| Operation | Endpoint |
|---|---|
| List racks | GET /api/racks |
| Create rack | POST /api/racks |
| Delete rack | DELETE /api/racks/{rack_id} |
| List nodes | GET /api/nodes |
| Add node | POST /api/nodes |
| Get node | GET /api/nodes/{id} |
| Remove node | DELETE /api/nodes/{id} |
| Ping node | POST /api/nodes/{id}/ping |
| Get server info | GET /api/nodes/{id}/server |
| Deploy server | POST /api/nodes/{id}/server/deploy |
| Restart server | POST /api/nodes/{id}/server/restart |
| Stop server | POST /api/nodes/{id}/server/stop |
| Operation | Endpoint |
|---|---|
| List stores | GET /api/stores |
| Create store | POST /api/stores |
| Get store | GET /api/stores/{sid} |
| Remove store | DELETE /api/stores/{sid} |
| List groups | GET /api/stores/{sid}/groups |
| Create group | POST /api/stores/{sid}/groups |
| Get group view | GET /api/stores/{sid}/groups/{gid} |
| Remove group | DELETE /api/stores/{sid}/groups/{gid} |
| List replicas | GET /api/stores/{sid}/groups/{gid}/replicas |
| Add replica | POST /api/stores/{sid}/groups/{gid}/replicas |
| Get replica | GET /api/stores/{sid}/groups/{gid}/replicas/{rid} |
| Remove replica | DELETE /api/stores/{sid}/groups/{gid}/replicas/{rid} |
| Resolve leader endpoint | GET /api/stores/{sid}/groups/{gid}/endpoint |
| Operation | Endpoint |
|---|---|
| Get | GET /api/stores/{sid}/groups/{gid}/kv/get?key=... |
| Put | POST /api/stores/{sid}/groups/{gid}/kv/put |
| Delete | POST /api/stores/{sid}/groups/{gid}/kv/delete |
| Scan (list mode) | GET /api/stores/{sid}/groups/{gid}/kv/scan?prefix=...&limit=N |
| Create snapshot | POST /api/stores/{sid}/groups/{gid}/snapshots |
| List snapshots | GET /api/stores/{sid}/groups/{gid}/snapshots |
| Snapshot scan | GET /api/stores/{sid}/groups/{gid}/snapshots/{handle}/scan?prefix=...&limit=N&start_after=... |
| Release snapshot | DELETE /api/stores/{sid}/groups/{gid}/snapshots/{handle} |
| Set GC watermark | POST /api/stores/{sid}/groups/{gid}/gc-watermark |
| Operation | Endpoint |
|---|---|
| System init (bootstrap group 0) | POST /system/init |
| Add store | POST /stores |
| Remove store | DELETE /stores/{sid} |
| Add group | POST /stores/{sid}/groups |
| Remove group | DELETE /stores/{sid}/groups/{gid} |
| Add remote replicas | POST /stores/{sid}/groups/{gid}/remotes |
| Step down leader | POST /stores/{sid}/groups/{gid}/step-down |
| Export topology | GET /topology |
| Health check | GET /health |
| Metrics | GET /metrics |
These endpoints are on the crowdb-kv-server management API (internal,
only called by crowdb-kv-client's KVClusterAdmin). The console's
POST /api/cluster/init orchestrates
/system/init across nodes and auto-finalizes.
The independent crowdb-iceberg binary exposes authenticated catalog configuration
only. Namespace, table and FileIO endpoints are not enabled. It uses an existing
-healthy Group 0, Chunk-KV and chunk-storage deployment; S3 credentials and buckets
-do not select or authorize an Iceberg catalog.
-pixi run -- cargo build -p crowdb-access-server --bin crowdb-iceberg
-export CROWDB_MANAGEMENT_SEEDS=127.0.0.1:10000
-export CROWDB_ICEBERG_LISTEN=127.0.0.1:8181Supply three distinct, randomly generated 32–256-character ASCII tokens through
-your secret-management environment: CROWDB_ICEBERG_READ_TOKEN,
CROWDB_ICEBERG_MANAGE_TOKEN and CROWDB_ICEBERG_CLEAR_TOKEN. Configure every
instance consistently. Management credentials can rename/initialize; only the
-clear credential can replace the catalog. All three can read configuration.
-The listener is plain HTTP: keep it on a trusted loopback/private hop behind a
-TLS-terminating proxy. Do not transmit bearer credentials over public plain HTTP.
-Set CROWDB_ICEBERG_TOKEN to the appropriate management token for CLI commands.
Each mutation takes a fresh UUIDv7 request identity. Preserve both that identity
-and the exact arguments when retrying an interrupted command.
-export CROWDB_ICEBERG_TOKEN="$CROWDB_ICEBERG_MANAGE_TOKEN"
-pixi run -- target/debug/crowdb-iceberg initialize "$INIT_UUIDV7" primary
-pixi run -- target/debug/crowdb-iceberg status
-pixi run -- target/debug/crowdb-iceberg rename "$RENAME_UUIDV7" renamed "$ACTIVE_EPOCH"
-pixi run -- target/debug/crowdb-iceberg serveStatus reports CatalogId, activation epoch, name and phase. Rename preserves the
-CatalogId. The server validates dependencies and reconciles the root before
-opening its listener. Ctrl-C stops admission and drains accepted connections.
-pixi run -- curl -H "Authorization: Bearer $CROWDB_ICEBERG_READ_TOKEN" \
- http://127.0.0.1:8181/v1/configAbsent or empty warehouse selects the active catalog. A nonempty warehouse
returns 404 NoSuchWarehouseException. Unsupported endpoints return 406; all
table-format capabilities are false and HTTP idempotency is not advertised.
-Clear makes the old domain inaccessible and selects a new empty catalog. It is
-not physical erasure. Obtain the exact epoch and CatalogId from status, then
explicitly confirm both:
-export CROWDB_ICEBERG_TOKEN="$CROWDB_ICEBERG_CLEAR_TOKEN"
-pixi run -- target/debug/crowdb-iceberg clear "$CLEAR_UUIDV7" empty \
- "$ACTIVE_EPOCH" "$ACTIVE_CATALOG_ID"Admission returns 503 during maintenance. Default persisted limits require an
-11-second grace after the durable fence is observed. Restart cannot shorten it.
-Another healthy instance resumes interrupted operations. An uncertain command
-must be retried with its original identity and input, not a newly generated key.
-Requests have a 24-hour retry window; expired identities are rejected. Bounded
-ledger-slot collisions can reject new operations without evicting live receipts.
-Run backend restart, two-instance and official-client checks with
-pixi run -e iceberg-e2e test-pyiceberg-e2e. This uses a separate disposable runtime
registry and leaves persistent local cluster reservations intact.
- ---rpc-port
`**
-- **`crowdb-cli server restart --node --group-id --group-id `**
-- **`crowdb-cli paxos inspect --store-id --group-id --group-id --group-id --group-id