compose.yaml

# The streaming stack for the Local course, on your laptop:
#
#   kafka            Ursa for Kafka, one broker. The login topic is diskless:
#                    its records live in the object store, not on the broker.
#   oxia             metadata for diskless topics
#   object-store     S3-compatible storage for diskless topics
#   schema-registry  the Avro schema of the login topic
#   risingwave       streaming SQL: the materialized view your agent reads
#   risingwave-mcp   RisingWave's MCP server: your agent's SQL tools
#
#   docker compose -f local/compose.yaml up -d --wait
#
# Every image is pinned and every published port binds to 127.0.0.1.
# The Agent Engine is a separate stack: see local/engine.sh.

name: hello-data-agent

services:
  oxia:
    image: oxia/oxia:0.16.7
    entrypoint: ["/oxia/bin/oxia"]
    command: ["standalone", "--data-dir=/data", "--shards=8"]
    volumes:
      - oxia-data:/data
    healthcheck:
      test: ["CMD-SHELL", "wget --spider -q http://localhost:8080/metrics || exit 1"]
      interval: 5s
      timeout: 5s
      retries: 20

  # No port is published, so these static credentials only work inside the stack.
  object-store:
    image: rustfs/rustfs:1.0.0
    environment:
      RUSTFS_ACCESS_KEY: ursa-local
      RUSTFS_SECRET_KEY: ursa-local-secret
    volumes:
      - object-data:/data
    healthcheck:
      test: ["CMD", "curl", "-fsS", "http://localhost:9000/health"]
      interval: 5s
      timeout: 5s
      retries: 30

  # Creates the bucket, unless it exists. It reuses the object store's image,
  # whose curl signs S3 requests itself, so nothing else has to be pulled.
  object-store-init:
    image: rustfs/rustfs:1.0.0
    entrypoint: ["/bin/sh", "-ec"]
    command:
      - |
        bucket=http://object-store:9000/kafka-ursa
        if curl -fs -o /dev/null --head --aws-sigv4 aws:amz:us-east-1:s3 \
            --user ursa-local:ursa-local-secret "$$bucket"; then
          echo "Bucket kafka-ursa exists."
        else
          curl -fsS -o /dev/null -X PUT --aws-sigv4 aws:amz:us-east-1:s3 \
            --user ursa-local:ursa-local-secret "$$bucket"
          echo "Created bucket kafka-ursa."
        fi
    depends_on:
      object-store:
        condition: service_healthy
    restart: "no"

  kafka:
    image: lakestream/kafka:4.3.1.3
    hostname: kafka
    ports:
      - "127.0.0.1:29092:9092"
    environment:
      KAFKA_NODE_ID: "1"
      KAFKA_PROCESS_ROLES: "broker,controller"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT"
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093"
      # Containers reach the broker at kafka:19092; your terminal at 127.0.0.1:29092
      # (the address, not `localhost`, which clients may try over IPv6 first).
      KAFKA_LISTENERS: "PLAINTEXT://:19092,CONTROLLER://:9093,PLAINTEXT_HOST://:9092"
      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:19092,PLAINTEXT_HOST://127.0.0.1:29092"
      KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
      KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
      CLUSTER_ID: "4L6g3nShT-eMCtK--X86sw"
      # One broker, so every internal topic has one replica.
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "1"
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "1"
      # A topic you did not create is an error, not a silently created classic topic.
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
      # KRaft metadata lives on the data volume, so a restart keeps your topics.
      KAFKA_LOG_DIRS: "/var/lib/kafka/data/kraft-combined-logs"

      # Diskless storage through Ursa: metadata in Oxia, records in the object store.
      KAFKA_URSA_STORAGE_ENABLE: "true"
      KAFKA_URSA_CATALOG_OXIA_SERVICE_URL: "oxia://oxia:6648/default"
      KAFKA_URSA_OXIA_SERVICE_URL: "oxia://oxia:6648/default"
      KAFKA_URSA_STORAGE_BACKEND_TYPE: "S3"
      KAFKA_URSA_STORAGE_PATH: "ursa/wal"
      KAFKA_URSA_STORAGE_S3_ENDPOINT: "http://object-store:9000"
      KAFKA_URSA_STORAGE_S3_ACCESS_KEY: "ursa-local"
      KAFKA_URSA_STORAGE_S3_SECRET_KEY: "ursa-local-secret"
      KAFKA_URSA_STORAGE_S3_BUCKET: "kafka-ursa"
      KAFKA_URSA_STORAGE_S3_REGION: "us-east-1"
      KAFKA_URSA_STORAGE_S3_PATH_STYLE_ACCESS: "true"
      KAFKA_URSA_STORAGE_COMPACTION_BUCKET: "kafka-ursa"
      KAFKA_URSA_STORAGE_COMPACTION_PREFIX: "ursa/compacted"
      KAFKA_URSA_STORAGE_WRITE_BUFFER_FLUSH_INTERVAL_MS: "250"
      KAFKA_URSA_STORAGE_WRITE_BUFFER_FLUSH_SIZE: "268435456"
      KAFKA_URSA_STORAGE_WRITE_BUFFER_SIZE: "16777216"
    volumes:
      - kafka-data:/var/lib/kafka/data
    depends_on:
      oxia:
        condition: service_healthy
      object-store-init:
        condition: service_completed_successfully
    healthcheck:
      test:
        - CMD-SHELL
        - /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:19092 >/dev/null 2>&1
      interval: 5s
      timeout: 30s
      retries: 30
      start_period: 30s

  # Karapace implements the Confluent Schema Registry API. Schemas live in a
  # classic `_schemas` topic on the broker.
  schema-registry:
    image: ghcr.io/aiven-open/karapace:6.2.2
    hostname: schema-registry
    ports:
      - "127.0.0.1:18081:8081"
    # The image has no entrypoint; this module is the schema registry.
    entrypoint: ["python3", "-m", "karapace"]
    environment:
      KARAPACE_KARAPACE_REGISTRY: "true"
      KARAPACE_KARAPACE_REST: "false"
      KARAPACE_BOOTSTRAP_URI: kafka:19092
      KARAPACE_HOST: 0.0.0.0
      KARAPACE_PORT: "8081"
      KARAPACE_ADVERTISED_HOSTNAME: schema-registry
      KARAPACE_CLIENT_ID: schema-registry
      KARAPACE_GROUP_ID: schema-registry
      KARAPACE_TOPIC_NAME: _schemas
      KARAPACE_REPLICATION_FACTOR: "1"
      KARAPACE_COMPATIBILITY: BACKWARD
      KARAPACE_LOG_LEVEL: WARNING
    depends_on:
      kafka:
        condition: service_healthy
    healthcheck:
      # Ready means `_schemas` is read to its end and this node is the primary.
      test:
        - CMD-SHELL
        - >-
          health="$$(curl -sf http://localhost:8081/_health)" &&
          echo "$$health" | grep -Eq '"schema_registry_ready": ?true' &&
          echo "$$health" | grep -Eq '"schema_registry_is_primary": ?true'
      interval: 5s
      timeout: 5s
      retries: 60
      start_period: 20s

  risingwave:
    image: risingwavelabs/risingwave:v3.1.0
    command: ["single_node"]
    ports:
      - "127.0.0.1:4566:4566"   # SQL (Postgres wire protocol)
      - "127.0.0.1:5691:5691"   # dashboard
    volumes:
      - risingwave-data:/root/.risingwave
    healthcheck:
      test: ["CMD-SHELL", "bash -c 'printf \"\" > /dev/tcp/127.0.0.1/4566'"]
      interval: 3s
      timeout: 5s
      retries: 40
      start_period: 10s

  # The server offers more than a hundred tools. Your agent's definition
  # (agent/local/) enables the few it needs.
  risingwave-mcp:
    image: risingwavelabs/risingwave-mcp-server:0.1.0
    hostname: risingwave-mcp
    ports:
      - "127.0.0.1:8000:8000"
    environment:
      RISINGWAVE_CONNECTION_STR: "postgresql://root:root@risingwave:4566/dev"
    depends_on:
      risingwave:
        condition: service_healthy

  # `psql` for local/sql.sh, so there is nothing to install. Not started by `up`.
  psql:
    image: postgres:16-alpine
    profiles: ["tools"]
    entrypoint: ["psql", "-h", "risingwave", "-p", "4566", "-d", "dev", "-U", "root"]

volumes:
  oxia-data:
  object-data:
  kafka-data:
  risingwave-data: