docker-compose.dstack-postgres.yaml

# dstack deployment - Topology 2: Postgres + S3
# tinycloud-node in the CVM, Postgres external.
# Database credentials encrypted by dstack KMS.
# Column encryption enabled for sensitive data.
#
# All ${...} variables should be encrypted using dstack's KMS public key
# before deployment. The operator never sees plaintext values.

services:
  tinycloud:
    # N0: unchanged production 1.20.6 binary; this is not a new app release.
    image: ghcr.io/tinycloudlabs/tinycloud-node@sha256:4bdae4fc232d886bb56bfbce4ec62ed4b8fb4e1369a01f6afb73d06dd68c188f
    restart: unless-stopped
    labels:
      tc.logship: tinycloud-node
    # Host Docker daemon sends GELF; the app never joins Vector's network.
    # Nonblocking UDP can lose logs. No raw Docker cache or /logs fallback.
    logging:
      driver: gelf
      options:
        gelf-address: udp://127.0.0.1:12201
        gelf-compression-type: none
        labels: tc.logship
        mode: non-blocking
        max-buffer-size: 4m
        cache-disabled: "true"
    ports:
      - "8000:8000"
    volumes:
      - /var/run/dstack.sock:/var/run/dstack.sock
    environment:
      TINYCLOUD_KEYS_TYPE: Dstack
      TINYCLOUD_STORAGE_DATABASE: "${DATABASE_URL}"
      TINYCLOUD_STORAGE_BLOCKS_TYPE: S3
      TINYCLOUD_STORAGE_BLOCKS_BUCKET: "${S3_BUCKET}"
      TINYCLOUD_STORAGE_BLOCKS_ENDPOINT: "${S3_ENDPOINT}"
      AWS_ACCESS_KEY_ID: "${AWS_KEY}"
      AWS_SECRET_ACCESS_KEY: "${AWS_SECRET}"
      AWS_DEFAULT_REGION: "${AWS_REGION}"
      TINYCLOUD_LOG_LEVEL: normal
      TINYCLOUD_LOG__FORMAT: Json
      # Enable Prometheus telemetry (metrics, span histograms, pool gauges).
      # The metrics handler is intentionally NOT gateway-exposed: port 8001 is
      # not published here, so the unauthenticated /metrics endpoint stays
      # internal to the CVM.
      TINYCLOUD_TELEMETRY__ENABLED: "true"
      # The managed Postgres instance allows max_connections = 25 (measured
      # 2026-07-29, with some of those reserved for superuser roles). The pool
      # default of 100 cannot be satisfied: the node opened and held slots
      # until the instance ran out, and further requests failed with
      # "Failed to acquire connection from pool: Connection pool timed out"
      # after the 30s sqlx default. Keep this below the instance limit with
      # headroom for admin/psql access, and raise it only alongside the
      # instance's own max_connections.
      # TC-950: 10, so two containers that overlap during a deploy (2 x 10)
      # fit in the instance limit with room for admin access.
      TINYCLOUD_DATABASE__MAX_CONNECTIONS: "10"
      # TC-1274: production boots with the epoch_head backfill off while the
      # database is CPU-bound. Writes stay correct without it. Re-enable later
      # by setting "true" (or removing this line); the pace keys are
      # TINYCLOUD_EPOCH_HEAD_BACKFILL__SPACES_PER_SECOND / __PROGRESS_EVERY.
      # Keep the double underscore: a single underscore is silently ignored.
      TINYCLOUD_EPOCH_HEAD_BACKFILL__ENABLED: "false"
      # Required for every manual TC-780 deploy. Keep true through offline
      # verification; only the final, explicitly reviewed redeploy sets false.
      TINYCLOUD_DATABASE__WRITE_FENCE: "${SQL_IDENTITY_FENCE:?set true during cutover, false after verification}"
      TINYCLOUD_CORS: "true"
      # --- addressed sharing (TC-397) ------------------------------------
      #
      # The public trust document shared with Share and OpenCredentials,
      # delivered inline because nothing else can reach this container: the
      # image is `FROM scratch` (no shell, so share-api's decode-to-tmpfs
      # entrypoint cannot work here) and a dstack deployment uploads only
      # this compose file (so there is no host path to bind-mount from).
      #
      # Seal the base64 of the reviewed trust-bundle.production.json — the
      # same bytes share-api serves — into the CVM's encrypted environment as
      # SHARE_TRUST_BUNDLE_BASE64. Same variable name and same wire form
      # share-api already uses, so one sealed value feeds both services and
      # cannot drift between them.
      #
      # The `:?` form is deliberate: compose refuses to interpolate a partial
      # deployment, failing loudly at deploy time rather than letting the node
      # die at boot with "share email trust bundle is required".
      TINYCLOUD_SHARE_EMAIL__ENABLED: "true"
      TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64: "${SHARE_TRUST_BUNDLE_BASE64:?seal the reviewed share-email trust bundle, base64-encoded}"
      # The sealed DATABASE_URL already carries sslmode=verify-full as its sole
      # query parameter. ROOT_CERT_PATH stays unset so verify-full resolves
      # against the compiled-in public root set (TC-363).
      TINYCLOUD_SHARE_EMAIL__POSTGRES_TLS__SSLMODE: verify-full
      TINYCLOUD_QUOTA_URL: "${TINYCLOUD_QUOTA_URL}"
      TINYCLOUD_ADMIN_SECRET: "${TINYCLOUD_ADMIN_SECRET}"
      NODE_STATS_READ_TOKEN: "${NODE_STATS_READ_TOKEN:-}"
      ROCKET_ADDRESS: "0.0.0.0"
      ROCKET_MAX_BLOCKING: "1024"

  dstack-ingress:
    image: ghcr.io/tinycloudlabs/tinycloud-node-ingress@sha256:63c1c44e2ded5b73a5f174807b91ca677826974df0f9ab3045f6ec87865a5744
    ports:
      - "443:443"
    depends_on:
      - tinycloud
    environment:
      - DOMAIN=tee.node.tinycloud.xyz
      - TARGET_ENDPOINT=http://tinycloud:8000
      - CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN}
      # CNAME target the ingress publishes for DOMAIN. Must not start with `_`:
      # Android's resolver (bionic res_hnok) rejects a CNAME target whose first
      # label is `_`, so `_.<gateway>` is unresolvable on Android (TC-513).
      # `gateway.<gateway>` is the target Phala Cloud advertises (gateway.cname)
      # and upstream dstack-ingress documents; routing still uses SNI + the
      # _dstack-app-address TXT record, so TLS stays end-to-end into the CVM.
      - GATEWAY_DOMAIN=gateway.${DSTACK_GATEWAY_DOMAIN}
      - CERTBOT_EMAIL=${CERTBOT_EMAIL}
      - SET_CAA=true
      - DNS_PROVIDER=cloudflare
      - CLIENT_MAX_BODY_SIZE=95m
    volumes:
      - /var/run/dstack.sock:/var/run/dstack.sock
      - cert-data:/etc/letsencrypt
    restart: unless-stopped
    labels:
      tc.logship: tinycloud-ingress
    logging:
      driver: gelf
      options:
        gelf-address: udp://127.0.0.1:12201
        gelf-compression-type: none
        labels: tc.logship
        mode: non-blocking
        max-buffer-size: 4m
        cache-disabled: "true"

  # The dstack gateway's TLS-terminating routes (<app>-<port>.<domain>:443)
  # only forward to compose-published ports, so guest sshd on :22 is
  # unreachable via `phala ssh` even with authorized_keys installed.
  # Publish a TCP forwarder on 10022 that relays to guest sshd via the
  # docker host gateway. Port 10022 exposes ONLY sshd, which is key-gated.
  ssh-gateway:
    image: alpine/socat:1.8.1.3@sha256:188fe0a22182f81c16def9d1137930121eaa3023f427164e32c0d2f118f79dd6
    restart: unless-stopped
    ports:
      - "10022:10022"
    extra_hosts:
      - "host.docker.internal:host-gateway"
    command: TCP-LISTEN:10022,fork,reuseaddr TCP:host.docker.internal:22

  vector:
    restart: unless-stopped
    image: timberio/vector:0.59.0-alpine@sha256:0e36bc2a290039e9b6a5e92f04bd2f133223506f72d09682c1df16c015172b7a
    init: true
    user: 10001:10001
    read_only: true
    cap_drop:
    - ALL
    security_opt:
    - no-new-privileges:true
    mem_limit: 64m
    memswap_limit: 64m
    cpus: 0.5
    ports:
    - 127.0.0.1:12201:12201/udp
    networks:
    - logship
    tmpfs:
    - /var/lib/vector:rw,noexec,nosuid,size=8m,uid=10001,gid=10001,mode=0700
    environment:
      TINYCLOUD_LOGSHIP_PROFILE: ${TINYCLOUD_LOGSHIP_PROFILE:-errors_slow}
      BETTERSTACK_SOURCE_TOKEN: ${BETTERSTACK_SOURCE_TOKEN:?seal the node source token before enabling log shipping}
      VECTOR_CONFIG_JSON: "{\"data_dir\":\"/var/lib/vector\",\"sources\":{\"gelf_in\":{\"type\":\"socket\",\"mode\":\"udp\",\"address\":\"0.0.0.0:12201\",\"max_length\":262144,\"receive_buffer_bytes\":8388608,\"framing\":{\"method\":\"chunked_gelf\",\"chunked_gelf\":{\"decompression\":\"Auto\"}},\"decoding\":{\"codec\":\"bytes\"}}},\"transforms\":{\"gelf_envelope\":{\"type\":\"remap\",\"inputs\":[\"gelf_in\"],\"source\":\"packet = string(.message) ?? \\\"\\\"\\nif length(packet) > 262144 {\\n  . = {\\\"reason\\\": \\\"oversized\\\"}\\n} else {\\n  outer, err = parse_json(packet)\\n  if err != null || !is_object(outer) || (outer.version != \\\"1.1\\\" && outer.version != \\\"1.0\\\") {\\n    . = {\\\"reason\\\": \\\"malformed_envelope\\\"}\\n  } else if exists(outer.partial_message) || exists(outer._partial_message) {\\n    . = {\\\"reason\\\": \\\"partial\\\"}\\n  } else {\\n    label = get(outer, [\\\"_tc.logship\\\"]) ?? null\\n    . = {\\\"_tc.logship\\\": label, \\\"message\\\": outer.short_message}\\n  }\\n}\",\"drop_on_error\":true,\"drop_on_abort\":true},\"logship_gate\":{\"type\":\"remap\",\"inputs\":[\"gelf_envelope\"],\"source\":\"if exists(.reason) {\\n  . = {\\\"reason\\\": .reason}\\n} else {\\nlabel = string(get(., [\\\"_tc.logship\\\"]) ?? null) ?? \\\"\\\"\\nraw = .message\\nreason = \\\"\\\"\\nif label == \\\"\\\" { reason = \\\"unlabelled\\\" }\\nelse if !includes([\\\"tinycloud-node\\\", \\\"tinycloud-ingress\\\"], label) { reason = \\\"unexpected_label\\\" }\\nelse if exists(.partial_message) { reason = \\\"partial\\\" }\\nelse if !is_string(raw) { reason = \\\"malformed_envelope\\\" }\\nelse if length(string!(raw)) > 65536 { reason = \\\"oversized\\\" }\\nif reason != \\\"\\\" {\\n  . = {\\\"reason\\\": reason}\\n} else {\\n  parsed, err = parse_json(string!(raw))\\n  if err != null || !is_object(parsed) { . = {\\\"reason\\\": \\\"non_json\\\"} }\\n  else if exists(parsed.partial_message) { . = {\\\"reason\\\": \\\"partial\\\"} }\\n  else {\\n    target = string(parsed.target) ?? \\\"\\\"\\n    level = downcase(string(parsed.level) ?? \\\"\\\")\\n    if label == \\\"tinycloud-node\\\" && !includes([\\\"warn\\\", \\\"error\\\"], level) && target != \\\"tinycloud\\\" && !starts_with(target, \\\"tinycloud::\\\") && target != \\\"tinycloud_core\\\" && !starts_with(target, \\\"tinycloud_core::\\\") {\\n      . = {\\\"reason\\\": \\\"unknown_target\\\"}\\n    } else if label == \\\"tinycloud-node\\\" && !includes([\\\"warn\\\", \\\"error\\\"], level) && (target != \\\"tinycloud::request\\\" || level != \\\"info\\\") {\\n      . = {\\\"reason\\\": \\\"filtered_level\\\"}\\n    } else { . = {\\\"label\\\": label, \\\"parsed\\\": parsed} }\\n  }\\n}\\n}\",\"drop_on_error\":true,\"drop_on_abort\":true},\"gate_route\":{\"type\":\"route\",\"inputs\":[\"logship_gate\"],\"route\":{\"candidate\":\"exists(.parsed)\",\"rejected\":\"exists(.reason)\"}},\"allowlist\":{\"type\":\"remap\",\"inputs\":[\"gate_route.candidate\"],\"source\":\"# Only this fresh object can leave the process boundary. No envelope, span,\\n# message, error, identity, URI or unknown key is copied into it.\\nj = .parsed\\nlabel = .label\\nout = {}\\nts = \\\"\\\"\\nif label == \\\"tinycloud-node\\\" {\\n  target = string(j.target) ?? \\\"\\\"\\n  level = downcase(string!(j.level))\\n  f = object(j.fields) ?? {}\\n  msg = string(f.message) ?? \\\"\\\"\\n  ts = string!(j.timestamp)\\n  # Literal Rocket template aliases only; never split an arbitrary URI.\\n  aliases = {\\\"/attestation?<nonce>\\\":\\\"/attestation\\\",\\\"/.well-known/encryption/network/<name>?<query..>\\\":\\\"/.well-known/encryption/network/<name>\\\",\\\"/hooks/events?<ticket>\\\":\\\"/hooks/events\\\",\\\"/hooks/webhooks?<query..>\\\":\\\"/hooks/webhooks\\\",\\\"/public/<space_id>/kv?<prefix>\\\":\\\"/public/<space_id>/kv\\\"}\\n  # TC-1840: the node's one 5xx diagnostic line per database failure ships as an individual event.\\n  # The input must carry exactly the declared keys (the message plus these seven, and db_hash only\\n  # when it is a valid pseudonym) with exactly the declared types and enum values. Anything else\\n  # leaves db_event null and the line falls back to the unclassified count below; no value of it\\n  # is ever copied unless it passed, and the shipped object is built from literals and the\\n  # validated values only.\\n  db_event = null\\n  if target == \\\"tinycloud::request\\\" && level == \\\"warn\\\" && msg == \\\"database failure answered with a 5xx\\\" {\\n    db_status = f.status\\n    db_elapsed = f.elapsed_ms\\n    db_code = string(f.code) ?? \\\"\\\"\\n    db_service = string(f.service) ?? \\\"\\\"\\n    db_kind = string(f.failure_kind) ?? \\\"\\\"\\n    db_route = string(f.route) ?? \\\"\\\"\\n    db_canonical_route = get(aliases, [db_route]) ?? null\\n    if is_string(db_canonical_route) { db_route = db_canonical_route }\\n    db_ok = is_integer(db_status) && is_integer(db_elapsed) && is_boolean(f.write_may_have_applied)\\n    if db_ok {\\n      db_status = int!(db_status)\\n      db_elapsed = int!(db_elapsed)\\n      db_ok = db_status >= 500 && db_status <= 599 && db_elapsed >= 0 && db_elapsed <= 9007199254740991 && includes([\\\"unavailable_retryable\\\",\\\"outcome_unknown\\\",\\\"internal\\\",\\\"sql_busy\\\",\\\"request_timeout\\\",\\\"read_audit_busy\\\"], db_code) && includes([\\\"sql\\\",\\\"duckdb\\\",\\\"kv\\\",\\\"other\\\"], db_service) && includes([\\\"pool_timeout\\\",\\\"pool_closed\\\",\\\"io\\\",\\\"statement_timeout\\\",\\\"serialization\\\",\\\"connection_lost\\\",\\\"artifact_changed\\\",\\\"lock_wait\\\",\\\"hydration\\\",\\\"artifact_transfer\\\",\\\"request_timeout\\\",\\\"read_audit_busy\\\",\\\"other\\\"], db_kind) && includes([\\\"/healthz\\\",\\\"/info\\\",\\\"/version\\\",\\\"/invoke\\\",\\\"/delegate\\\",\\\"/revoke\\\",\\\"/attestation\\\",\\\"/signed/kv\\\",\\\"/signed/kv/<ticket_id>\\\",\\\"/admin/quota\\\",\\\"/admin/quota/<space_id>\\\",\\\"/admin/usage\\\",\\\"/admin/stats\\\",\\\"/hooks/events\\\",\\\"/hooks/tickets\\\",\\\"/hooks/webhooks\\\",\\\"/hooks/webhooks/<subscription_id>\\\",\\\"/delegation/query\\\",\\\"/delegation/status\\\",\\\"/encryption/networks\\\",\\\"/encryption/networks/<network_id>\\\",\\\"/encryption/networks/<network_id>/revoke\\\",\\\"/encryption/networks/<network_id>/decrypt\\\",\\\"/policy/v3/status\\\",\\\"/policy/v3/status/<root_cid>\\\",\\\"/policy/v3/policies\\\",\\\"/policy/v3/challenges\\\",\\\"/policy/v3/delegations\\\",\\\"/policy/v3/enforcer-bindings\\\",\\\"/policy/v3/deliveries/authorize\\\",\\\"/.well-known/tinycloud/node-keys\\\",\\\"/.well-known/encryption/network/<name>\\\",\\\"/public/<space_id>/kv\\\",\\\"/public/<space_id>/kv/<key..>\\\",\\\"/peer/generate/<space>\\\",\\\"/evidences/<file>\\\",\\\"/share/v2/readiness\\\",\\\"/share/v2/policies\\\",\\\"/share/v2/policy/challenges\\\",\\\"/share/v2/policy/session\\\",\\\"/share/v2/invoke\\\",\\\"/share/v2/deliveries/authorize\\\",\\\"/<_s..>\\\",\\\"/public/<_space_id>/kv/<_key..>\\\",\\\"<unmatched>\\\"], db_route)\\n    }\\n    db_hash = string(f.db_hash) ?? \\\"\\\"\\n    has_db_hash = exists(f.db_hash)\\n    if db_ok {\\n      if has_db_hash { db_ok = match(db_hash, r'^[0-9a-f]{64}$') && length(f) == 9 } else { db_ok = length(f) == 8 }\\n    }\\n    if db_ok {\\n      db_event = {\\\"event\\\": \\\"node_warning\\\", \\\"svc\\\": \\\"api\\\", \\\"level\\\": \\\"warn\\\", \\\"event_code\\\": \\\"db_failure_5xx\\\", \\\"status\\\": db_status, \\\"code\\\": db_code, \\\"service\\\": db_service, \\\"route\\\": db_route, \\\"elapsed_ms\\\": db_elapsed, \\\"failure_kind\\\": db_kind, \\\"write_may_have_applied\\\": bool!(f.write_may_have_applied)}\\n      if has_db_hash { db_event.db_hash = db_hash }\\n    }\\n  }\\n  if target == \\\"tinycloud::request\\\" && level == \\\"info\\\" && msg == \\\"request\\\" {\\n    method = string!(f.method)\\n    if includes([\\\"TRACE\\\", \\\"CONNECT\\\"], method) { method = \\\"<other>\\\" }\\n    route = string!(f.route)\\n    canonical_route = get(aliases, [route]) ?? null\\n    if is_string(canonical_route) { route = canonical_route }\\n    if !includes([\\\"GET\\\",\\\"HEAD\\\",\\\"POST\\\",\\\"PUT\\\",\\\"PATCH\\\",\\\"DELETE\\\",\\\"OPTIONS\\\",\\\"<other>\\\"], method) || !includes([\\\"/healthz\\\",\\\"/info\\\",\\\"/version\\\",\\\"/invoke\\\",\\\"/delegate\\\",\\\"/revoke\\\",\\\"/attestation\\\",\\\"/signed/kv\\\",\\\"/signed/kv/<ticket_id>\\\",\\\"/admin/quota\\\",\\\"/admin/quota/<space_id>\\\",\\\"/admin/usage\\\",\\\"/admin/stats\\\",\\\"/hooks/events\\\",\\\"/hooks/tickets\\\",\\\"/hooks/webhooks\\\",\\\"/hooks/webhooks/<subscription_id>\\\",\\\"/delegation/query\\\",\\\"/delegation/status\\\",\\\"/encryption/networks\\\",\\\"/encryption/networks/<network_id>\\\",\\\"/encryption/networks/<network_id>/revoke\\\",\\\"/encryption/networks/<network_id>/decrypt\\\",\\\"/policy/v3/status\\\",\\\"/policy/v3/status/<root_cid>\\\",\\\"/policy/v3/policies\\\",\\\"/policy/v3/challenges\\\",\\\"/policy/v3/delegations\\\",\\\"/policy/v3/enforcer-bindings\\\",\\\"/policy/v3/deliveries/authorize\\\",\\\"/.well-known/tinycloud/node-keys\\\",\\\"/.well-known/encryption/network/<name>\\\",\\\"/public/<space_id>/kv\\\",\\\"/public/<space_id>/kv/<key..>\\\",\\\"/peer/generate/<space>\\\",\\\"/evidences/<file>\\\",\\\"/share/v2/readiness\\\",\\\"/share/v2/policies\\\",\\\"/share/v2/policy/challenges\\\",\\\"/share/v2/policy/session\\\",\\\"/share/v2/invoke\\\",\\\"/share/v2/deliveries/authorize\\\",\\\"/<_s..>\\\",\\\"/public/<_space_id>/kv/<_key..>\\\",\\\"<unmatched>\\\"], route) { abort }\\n    status = f.status\\n    latency = f.latency_us\\n    if !is_integer(status) || !is_integer(latency) { abort }\\n    status = int!(status)\\n    latency = int!(latency)\\n    if status < 100 || status > 599 || latency < 0 || latency > 9007199254740991 { abort }\\n    out = {\\\"event\\\": \\\"node_request\\\", \\\"svc\\\": \\\"api\\\", \\\"level\\\": \\\"info\\\", \\\"method\\\": method, \\\"route\\\": route, \\\"status\\\": status, \\\"latency_us\\\": latency}\\n    # N2 correlation fields: copied only when they have exactly the node's emitted shape; anything else is dropped\\n    # (the event still ships). The pseudonyms are keyed HMAC-SHA256 digests, never raw identifiers.\\n    rid = string(f.request_id) ?? \\\"\\\"\\n    if match(rid, r'^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$') { out.request_id = rid }\\n    replaced = f.request_id_replaced\\n    if is_boolean(replaced) { out.request_id_replaced = replaced }\\n    account = string(f.account_hash) ?? \\\"\\\"\\n    if match(account, r'^[0-9a-f]{64}$') { out.account_hash = account }\\n    space = string(f.space_hash) ?? \\\"\\\"\\n    if match(space, r'^[0-9a-f]{64}$') { out.space_hash = space }\\n  } else if is_object(db_event) {\\n    out = db_event\\n  } else {\\n    if !includes([\\\"warn\\\", \\\"error\\\"], level) { abort }\\n    code = \\\"unclassified\\\"\\n    if target == \\\"tinycloud\\\" || starts_with(target, \\\"tinycloud::\\\") || target == \\\"tinycloud_core\\\" || starts_with(target, \\\"tinycloud_core::\\\") {\\n      codes = {\\\"invocation replay insert failed\\\":\\\"invocation_replay_insert_failed\\\",\\\"invocation replay cleanup failed\\\":\\\"invocation_replay_cleanup_failed\\\",\\\"share-v2 local-tee feature active: deriving a deterministic, non-attested TEE identity from node key material\\\":\\\"share_local_tee_identity\\\",\\\"retention: delivered hook delivery prune failed\\\":\\\"retention_hook_delivery_prune_failed\\\",\\\"retention: dead-letter hook delivery prune failed\\\":\\\"retention_dead_letter_prune_failed\\\",\\\"retention: signed KV ticket prune failed\\\":\\\"retention_signed_ticket_prune_failed\\\",\\\"failed to load tunnel state; tunnel not started\\\":\\\"tunnel_state_load_failed\\\",\\\"invalid loopback API address; tunnel not started\\\":\\\"tunnel_address_invalid\\\",\\\"failed to load link state; LAN listener not started\\\":\\\"link_state_load_failed\\\",\\\"invalid link bind address; LAN listener not started\\\":\\\"link_bind_invalid\\\",\\\"invalid loopback API address; LAN listener not started\\\":\\\"link_address_invalid\\\",\\\"failed to bind link LAN listener; LAN listener not started\\\":\\\"link_bind_failed\\\",\\\"failed to build TLS config; LAN listener not started\\\":\\\"link_tls_config_failed\\\",\\\"link LAN listener exited with error\\\":\\\"link_listener_error\\\",\\\"failed to record link listener bind state\\\":\\\"link_listener_state_write_failed\\\",\\\"failed to record link listener bind failure\\\":\\\"link_bind_state_write_failed\\\",\\\"failed to load config for auto-renew\\\":\\\"link_auto_renew_config_failed\\\",\\\"link auto-renew failed\\\":\\\"link_auto_renew_failed\\\",\\\"failed to hot-reload renewed link cert\\\":\\\"link_cert_reload_failed\\\",\\\"link auto-renew succeeded but no state.json found; skipping cert hot-reload\\\":\\\"link_cert_state_missing\\\",\\\"failed to reload TLS material after link auto-renew\\\":\\\"link_listener_cert_reload_failed\\\",\\\"link disable: service delete failed; local state removed anyway\\\":\\\"link_disable_failed\\\",\\\"link connection error\\\":\\\"link_connection_error\\\",\\\"link listener accept error\\\":\\\"link_listener_accept_failed\\\",\\\"webhook delivery attempt failed\\\":\\\"webhook_delivery_failed\\\",\\\"webhook dispatcher loop failed\\\":\\\"webhook_dispatcher_failed\\\",\\\"addressed delegation authorization failed\\\":\\\"share_authorization_failed\\\",\\\"native share authorization denied\\\":\\\"share_native_authorization_denied\\\",\\\"native share KV put failed\\\":\\\"share_native_kv_put_failed\\\",\\\"share v2 readiness probe failed\\\":\\\"share_v2_readiness_failed\\\",\\\"share v2 is enabled but shareEmail.invitationPublicKey is unset\\\":\\\"share_invitation_key_missing\\\",\\\"configured share invitation key does not match the node's derived signing key\\\":\\\"share_invitation_key_mismatch\\\",\\\"share/v2 policy challenge denied\\\":\\\"share_policy_denied\\\",\\\"failed to read committed hook events\\\":\\\"hook_events_read_failed\\\",\\\"failed to load kv write hook rows\\\":\\\"hook_rows_load_failed\\\",\\\"failed to load kv delete hook rows\\\":\\\"hook_rows_load_failed\\\",\\\"missing committed kv hook row for invocation\\\":\\\"missing_committed_hook_row\\\",\\\"database write committed but webhook enqueue failed\\\":\\\"webhook_enqueue_failed\\\",\\\"tunnel: failed to prepare connection attempt\\\":\\\"tunnel_connection_prepare_failed\\\",\\\"tunnel: no frame within idle timeout; treating connection as dead\\\":\\\"tunnel_idle_timeout\\\",\\\"tunnel: read error\\\":\\\"tunnel_read_failed\\\",\\\"tunnel: dropping malformed frame\\\":\\\"tunnel_malformed_frame\\\",\\\"tunnel: relay sent an error frame\\\":\\\"tunnel_relay_error\\\",\\\"tunnel: failed to write frame\\\":\\\"tunnel_write_failed\\\",\\\"tunnel: failed to record runtime state\\\":\\\"tunnel_state_record_failed\\\",\\\"tunnel: failed to clear runtime state after disable\\\":\\\"tunnel_state_clear_failed\\\",\\\"tunnel authentication rejected\\\":\\\"tunnel_auth_rejected\\\",\\\"network DID resolution enabled: did:web/did:tz fetches use the unhardened ssi client (TC-1080)\\\":\\\"network_did_resolution_enabled\\\",\\\"SQL busy: lock wait\\\":\\\"sql_busy_lock_wait\\\",\\\"SQL busy: hydration\\\":\\\"sql_busy_hydration\\\",\\\"SQL busy: artifact transfer\\\":\\\"sql_busy_artifact_transfer\\\",\\\"Request deadline exceeded\\\":\\\"request_timeout\\\",\\\"Request completed after its deadline\\\":\\\"deadline_overrun\\\",\\\"Database hydration gave up loading its artifact\\\":\\\"sql_hydration_timeout\\\",\\\"Read audit wait expired\\\":\\\"read_audit_busy\\\",\\\"Read audit batch slow\\\":\\\"read_audit_batch_slow\\\",\\\"Read audit batch failed\\\":\\\"read_audit_batch_failed\\\",\\\"Read audit drain stalled\\\":\\\"read_audit_drain_stalled\\\",\\\"idx_current_kv_space_deleted_key_c is missing; the node keeps serving and KV list requests are slower until an operator builds it with CREATE INDEX CONCURRENTLY (docs/releasing.md, section idx_current_kv_space_deleted_key_c)\\\":\\\"current_kv_c_index_missing\\\",\\\"idx_current_kv_space_deleted_key_c is invalid or still being built; the node keeps serving and KV list requests are slower. If no CREATE INDEX CONCURRENTLY is running, run DROP INDEX CONCURRENTLY idx_current_kv_space_deleted_key_c; and then the CREATE INDEX CONCURRENTLY in docs/releasing.md, section idx_current_kv_space_deleted_key_c\\\":\\\"current_kv_c_index_invalid\\\",\\\"idx_current_kv_space_deleted_key_c exists with the wrong definition and is not used for KV lists; the node keeps serving and KV list requests are slower. Run DROP INDEX CONCURRENTLY idx_current_kv_space_deleted_key_c; and then the CREATE INDEX CONCURRENTLY in docs/releasing.md, section idx_current_kv_space_deleted_key_c\\\":\\\"current_kv_c_index_wrong_shape\\\"}\\n      known = get(codes, [msg]) ?? null\\n      if is_string(known) { code = known }\\n      if starts_with(msg, \\\"quota service returned \\\") { code = \\\"quota_fallback\\\" }\\nif starts_with(msg, \\\"Failed to get TDX quote: \\\") { code = \\\"attestation_quote_failed\\\" }\\nif starts_with(msg, \\\"quota service returned an invalid body: \\\") { code = \\\"quota_invalid_body\\\" }\\nif starts_with(msg, \\\"dstack socket available but get_info failed: \\\") { code = \\\"dstack_get_info_failed\\\" }\\n    }\\n    event = if level == \\\"warn\\\" { \\\"node_warning\\\" } else { \\\"node_error\\\" }\\n    out = {\\\"event\\\": event, \\\"svc\\\": \\\"api\\\", \\\"level\\\": level, \\\"event_code\\\": code, \\\"event_count\\\": 1}\\n  }\\n} else {\\n  ts = string!(j.time)\\n  method = string!(j.method)\\n  route = string!(j.route)\\n  if route == \\\"<other>\\\" { route = \\\"<unmatched>\\\" }\\n  if !includes([\\\"GET\\\",\\\"HEAD\\\",\\\"POST\\\",\\\"PUT\\\",\\\"PATCH\\\",\\\"DELETE\\\",\\\"OPTIONS\\\",\\\"<other>\\\"], method) || !includes([\\\"/healthz\\\",\\\"/info\\\",\\\"/version\\\",\\\"/invoke\\\",\\\"/delegate\\\",\\\"/revoke\\\",\\\"/attestation\\\",\\\"/signed/kv\\\",\\\"/signed/kv/<ticket_id>\\\",\\\"/admin/quota\\\",\\\"/admin/quota/<space_id>\\\",\\\"/admin/usage\\\",\\\"/admin/stats\\\",\\\"/hooks/events\\\",\\\"/hooks/tickets\\\",\\\"/hooks/webhooks\\\",\\\"/hooks/webhooks/<subscription_id>\\\",\\\"/delegation/query\\\",\\\"/delegation/status\\\",\\\"/encryption/networks\\\",\\\"/encryption/networks/<network_id>\\\",\\\"/encryption/networks/<network_id>/revoke\\\",\\\"/encryption/networks/<network_id>/decrypt\\\",\\\"/policy/v3/status\\\",\\\"/policy/v3/status/<root_cid>\\\",\\\"/policy/v3/policies\\\",\\\"/policy/v3/challenges\\\",\\\"/policy/v3/delegations\\\",\\\"/policy/v3/enforcer-bindings\\\",\\\"/policy/v3/deliveries/authorize\\\",\\\"/.well-known/tinycloud/node-keys\\\",\\\"/.well-known/encryption/network/<name>\\\",\\\"/public/<space_id>/kv\\\",\\\"/public/<space_id>/kv/<key..>\\\",\\\"/peer/generate/<space>\\\",\\\"/evidences/<file>\\\",\\\"/share/v2/readiness\\\",\\\"/share/v2/policies\\\",\\\"/share/v2/policy/challenges\\\",\\\"/share/v2/policy/session\\\",\\\"/share/v2/invoke\\\",\\\"/share/v2/deliveries/authorize\\\",\\\"/<_s..>\\\",\\\"/public/<_space_id>/kv/<_key..>\\\",\\\"<unmatched>\\\"], route) { abort }\\n  status = j.status\\n  request_time = j.request_time\\n  if !is_integer(status) || !(is_integer(request_time) || is_float(request_time)) { abort }\\n  status = int!(status)\\n  request_time = to_float!(request_time)\\n  if status < 100 || status > 599 || request_time < 0 || request_time > 86400 { abort }\\n  out = {\\\"event\\\": \\\"ingress_request\\\", \\\"svc\\\": \\\"ingress\\\", \\\"level\\\": \\\"info\\\", \\\"method\\\": method, \\\"route\\\": route, \\\"status\\\": status, \\\"request_time_seconds\\\": request_time}\\n}\\nif length(ts) > 64 { abort }\\nparsed_ts = parse_timestamp!(ts, format: \\\"%+\\\")\\nout.ts = format_timestamp!(parsed_ts, format: \\\"%Y-%m-%dT%H:%M:%S%.3fZ\\\")\\n. = out\\n\",\"drop_on_error\":true,\"drop_on_abort\":true,\"reroute_dropped\":true},\"gate_counts\":{\"type\":\"remap\",\"inputs\":[\"gate_route.rejected\",\"gate_route._unmatched\",\"allowlist.dropped\",\"event_route._unmatched\"],\"source\":\"reason = .reason\\nif !includes([\\\"malformed_envelope\\\",\\\"unexpected_label\\\",\\\"unlabelled\\\",\\\"non_json\\\",\\\"oversized\\\",\\\"partial\\\",\\\"invalid_fields\\\",\\\"unknown_target\\\",\\\"filtered_level\\\"], reason) { reason = \\\"invalid_fields\\\" }\\n. = {\\\"reason\\\": reason, \\\"count\\\": 1}\",\"drop_on_error\":true,\"drop_on_abort\":true},\"gate_reduce\":{\"type\":\"reduce\",\"inputs\":[\"gate_counts\"],\"group_by\":[\"reason\"],\"end_every_period_ms\":60000,\"expire_after_ms\":75000,\"flush_period_ms\":1000,\"merge_strategies\":{\"count\":\"sum\"}},\"gate_output\":{\"type\":\"remap\",\"inputs\":[\"gate_reduce\"],\"source\":\". = {\\\"ts\\\": format_timestamp!(now(), format: \\\"%Y-%m-%dT%H:%M:%S%.3fZ\\\"), \\\"level\\\": \\\"warn\\\", \\\"event\\\": \\\"logship_gate_summary_total\\\", \\\"svc\\\": \\\"logship\\\", \\\"reason\\\": .reason, \\\"count\\\": .count}\",\"drop_on_error\":true,\"drop_on_abort\":true},\"request_profile\":{\"type\":\"filter\",\"inputs\":[\"allowlist\"],\"condition\":\"\\\"$${TINYCLOUD_LOGSHIP_PROFILE}\\\" == \\\"all_requests\\\" || includes([\\\"node_warning\\\", \\\"node_error\\\"], .event) || (int(.status) ?? 0) >= 400 || (.event == \\\"node_request\\\" && (int(.latency_us) ?? 0) > 1000000) || (.event == \\\"ingress_request\\\" && (to_float(.request_time_seconds) ?? 0.0) > 1.000000)\"},\"event_route\":{\"type\":\"route\",\"inputs\":[\"request_profile\"],\"route\":{\"request\":\"(.event == \\\"node_request\\\" || .event == \\\"ingress_request\\\") && !(((int(.status) ?? 0) >= 400 && (int(.status) ?? 0) < 500)) && !((int(.status) ?? 0) >= 500)\",\"node_request_5xx\":\".event == \\\"node_request\\\" && (int(.status) ?? 0) >= 500\",\"ingress_request_5xx\":\".event == \\\"ingress_request\\\" && (int(.status) ?? 0) >= 500\",\"node_request_4xx\":\".event == \\\"node_request\\\" && ((int(.status) ?? 0) >= 400 && (int(.status) ?? 0) < 500)\",\"ingress_request_4xx\":\".event == \\\"ingress_request\\\" && ((int(.status) ?? 0) >= 400 && (int(.status) ?? 0) < 500)\",\"db_failure_5xx\":\".event == \\\"node_warning\\\" && .event_code == \\\"db_failure_5xx\\\"\",\"diagnostic\":\"(.event == \\\"node_warning\\\" && .event_code != \\\"db_failure_5xx\\\") || .event == \\\"node_error\\\"\"}},\"cap_db_failure_5xx\":{\"type\":\"lua\",\"version\":\"2\",\"inputs\":[\"event_route.db_failure_5xx\"],\"hooks\":{\"process\":\"process\"},\"timers\":[{\"interval_seconds\":60,\"handler\":\"flush\"}],\"source\":\"local class = \\\"db_failure_5xx\\\"\\nlocal cap = 10\\nlocal passed = 0\\nlocal dropped = 0\\nlocal ticks = 0\\nfunction process(event, emit)\\n  if passed < cap then\\n    passed = passed + 1\\n    emit(event)\\n  else\\n    dropped = dropped + 1\\n  end\\nend\\nfunction flush(emit)\\n  ticks = ticks + 1\\n  if ticks == 1 then return end\\n  if dropped > 0 then\\n    emit({log = {ts = os.date(\\\"!%Y-%m-%dT%H:%M:%S.000Z\\\"), level = \\\"warn\\\", event = \\\"node_warning\\\", svc = \\\"api\\\", event_code = \\\"errors_dropped\\\", class = class, dropped = dropped, window_s = 60}})\\n  end\\n  passed = 0\\n  dropped = 0\\nend\"},\"cap_node_request_5xx\":{\"type\":\"lua\",\"version\":\"2\",\"inputs\":[\"event_route.node_request_5xx\"],\"hooks\":{\"process\":\"process\"},\"timers\":[{\"interval_seconds\":60,\"handler\":\"flush\"}],\"source\":\"local class = \\\"node_request_5xx\\\"\\nlocal cap = 10\\nlocal passed = 0\\nlocal dropped = 0\\nlocal ticks = 0\\nfunction process(event, emit)\\n  if passed < cap then\\n    passed = passed + 1\\n    emit(event)\\n  else\\n    dropped = dropped + 1\\n  end\\nend\\nfunction flush(emit)\\n  ticks = ticks + 1\\n  if ticks == 1 then return end\\n  if dropped > 0 then\\n    emit({log = {ts = os.date(\\\"!%Y-%m-%dT%H:%M:%S.000Z\\\"), level = \\\"warn\\\", event = \\\"node_warning\\\", svc = \\\"api\\\", event_code = \\\"errors_dropped\\\", class = class, dropped = dropped, window_s = 60}})\\n  end\\n  passed = 0\\n  dropped = 0\\nend\"},\"cap_ingress_request_5xx\":{\"type\":\"lua\",\"version\":\"2\",\"inputs\":[\"event_route.ingress_request_5xx\"],\"hooks\":{\"process\":\"process\"},\"timers\":[{\"interval_seconds\":60,\"handler\":\"flush\"}],\"source\":\"local class = \\\"ingress_request_5xx\\\"\\nlocal cap = 10\\nlocal passed = 0\\nlocal dropped = 0\\nlocal ticks = 0\\nfunction process(event, emit)\\n  if passed < cap then\\n    passed = passed + 1\\n    emit(event)\\n  else\\n    dropped = dropped + 1\\n  end\\nend\\nfunction flush(emit)\\n  ticks = ticks + 1\\n  if ticks == 1 then return end\\n  if dropped > 0 then\\n    emit({log = {ts = os.date(\\\"!%Y-%m-%dT%H:%M:%S.000Z\\\"), level = \\\"warn\\\", event = \\\"node_warning\\\", svc = \\\"api\\\", event_code = \\\"errors_dropped\\\", class = class, dropped = dropped, window_s = 60}})\\n  end\\n  passed = 0\\n  dropped = 0\\nend\"},\"cap_node_request_4xx\":{\"type\":\"lua\",\"version\":\"2\",\"inputs\":[\"event_route.node_request_4xx\"],\"hooks\":{\"process\":\"process\"},\"timers\":[{\"interval_seconds\":60,\"handler\":\"flush\"}],\"source\":\"local class = \\\"node_request_4xx\\\"\\nlocal cap = 3\\nlocal passed = 0\\nlocal dropped = 0\\nlocal ticks = 0\\nfunction process(event, emit)\\n  if passed < cap then\\n    passed = passed + 1\\n    emit(event)\\n  else\\n    dropped = dropped + 1\\n  end\\nend\\nfunction flush(emit)\\n  ticks = ticks + 1\\n  if ticks == 1 then return end\\n  if dropped > 0 then\\n    emit({log = {ts = os.date(\\\"!%Y-%m-%dT%H:%M:%S.000Z\\\"), level = \\\"warn\\\", event = \\\"node_warning\\\", svc = \\\"api\\\", event_code = \\\"errors_dropped\\\", class = class, dropped = dropped, window_s = 60}})\\n  end\\n  passed = 0\\n  dropped = 0\\nend\"},\"cap_ingress_request_4xx\":{\"type\":\"lua\",\"version\":\"2\",\"inputs\":[\"event_route.ingress_request_4xx\"],\"hooks\":{\"process\":\"process\"},\"timers\":[{\"interval_seconds\":60,\"handler\":\"flush\"}],\"source\":\"local class = \\\"ingress_request_4xx\\\"\\nlocal cap = 3\\nlocal passed = 0\\nlocal dropped = 0\\nlocal ticks = 0\\nfunction process(event, emit)\\n  if passed < cap then\\n    passed = passed + 1\\n    emit(event)\\n  else\\n    dropped = dropped + 1\\n  end\\nend\\nfunction flush(emit)\\n  ticks = ticks + 1\\n  if ticks == 1 then return end\\n  if dropped > 0 then\\n    emit({log = {ts = os.date(\\\"!%Y-%m-%dT%H:%M:%S.000Z\\\"), level = \\\"warn\\\", event = \\\"node_warning\\\", svc = \\\"api\\\", event_code = \\\"errors_dropped\\\", class = class, dropped = dropped, window_s = 60}})\\n  end\\n  passed = 0\\n  dropped = 0\\nend\"},\"diagnostic_reduce\":{\"type\":\"reduce\",\"inputs\":[\"event_route.diagnostic\"],\"group_by\":[\"event\",\"event_code\",\"level\",\"svc\"],\"end_every_period_ms\":60000,\"expire_after_ms\":75000,\"flush_period_ms\":1000,\"merge_strategies\":{\"event_count\":\"sum\",\"ts\":\"discard\"}},\"diagnostic_output\":{\"type\":\"remap\",\"inputs\":[\"diagnostic_reduce\"],\"source\":\". = {\\\"ts\\\": format_timestamp!(now(), format: \\\"%Y-%m-%dT%H:%M:%S%.3fZ\\\"), \\\"level\\\": .level, \\\"event\\\": .event, \\\"svc\\\": \\\"api\\\", \\\"event_code\\\": .event_code, \\\"event_count\\\": .event_count}\",\"drop_on_error\":true,\"drop_on_abort\":true}},\"sinks\":{\"betterstack\":{\"type\":\"http\",\"inputs\":[\"event_route.request\",\"cap_db_failure_5xx\",\"cap_node_request_5xx\",\"cap_ingress_request_5xx\",\"cap_node_request_4xx\",\"cap_ingress_request_4xx\",\"diagnostic_output\",\"gate_output\"],\"method\":\"post\",\"uri\":\"https://s2795437.us-west-2a.betterstackdata.com/\",\"encoding\":{\"codec\":\"json\"},\"framing\":{\"method\":\"newline_delimited\"},\"compression\":\"gzip\",\"auth\":{\"strategy\":\"bearer\",\"token\":\"SECRET[bs.bs_token]\"},\"batch\":{\"max_bytes\":262144,\"timeout_secs\":1},\"buffer\":{\"type\":\"memory\",\"max_events\":1000,\"when_full\":\"block\"},\"request\":{\"timeout_secs\":10,\"retry_max_duration_secs\":300},\"healthcheck\":{\"enabled\":false}}},\"secret\":{\"bs\":{\"type\":\"directory\",\"path\":\"/var/lib/vector/secrets\",\"remove_trailing_whitespace\":true}}}"
    entrypoint:
    - /bin/sh
    - -ec
    - "umask 077\nmkdir -p /var/lib/vector/secrets\ncase \"$$TINYCLOUD_LOGSHIP_PROFILE\" in\n  errors_slow|all_requests) ;;\n  *) printf '%s\\n' 'logship profile invalid' >&2; exit 78 ;;\nesac\nif [ -z \"$$BETTERSTACK_SOURCE_TOKEN\" ]; then\n  printf '%s\\n' 'logship source token missing' >&2\n  exit 78\nfi\nprintf '%s' \"$$BETTERSTACK_SOURCE_TOKEN\" > /var/lib/vector/secrets/bs_token\nprintf '%s' \"$$VECTOR_CONFIG_JSON\" > /var/lib/vector/runtime.json\nunset BETTERSTACK_SOURCE_TOKEN VECTOR_CONFIG_JSON\nexec vector --config /var/lib/vector/runtime.json\n"
    logging:
      driver: json-file
      options:
        max-size: 10m
        max-file: '3'

volumes:
  cert-data:

networks:
  logship:
    driver: bridge