compose.yml

name: bento
# GPU Configuration:
# The gpu_prove_agent service automatically detects all available GPUs using nvidia-smi
# and spawns a prove agent process for each GPU. No manual configuration needed.
#
# To check available GPUs:
#   nvidia-smi --list-gpus

# Anchors:
x-base-environment: &base-environment
  DATABASE_URL: postgresql://${POSTGRES_USER:-worker}:${POSTGRES_PASSWORD:-password}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-taskdb}
  # With Redis auth set REDIS_PASSWORD; URL is redis://:password@host:6379 (empty password OK when no auth)
  REDIS_URL: redis://:${REDIS_PASSWORD:-}@${REDIS_HOST:-redis}:6379
  S3_URL: http://${MINIO_HOST:-minio}:9000
  S3_BUCKET: ${MINIO_BUCKET:-workflow}
  S3_ACCESS_KEY: ${MINIO_ROOT_USER:-admin}
  S3_SECRET_KEY: ${MINIO_ROOT_PASS:-password}
  NO_COLOR: 1
  RUST_LOG: ${RUST_LOG:-info}
  RISC0_HOME: /usr/local/risc0
  RUST_BACKTRACE: 1
  POVW_LOG_ID: ${REWARD_ADDRESS:-${POVW_LOG_ID:-}}
x-cpu-agent-common: &cpu-agent-common
  image: ${CPU_AGENT_IMAGE:-ghcr.io/boundless-xyz/boundless/bento-agent-cpu:latest}
  runtime: runc
  build:
    context: .
    dockerfile: ${CPU_AGENT_DOCKERFILE:-dockerfiles/agent.cpu.dockerfile}
  restart: always
  depends_on:
    postgres:
      condition: service_healthy
    redis:
      condition: service_healthy
    minio:
      condition: service_healthy
  environment:
    <<: *base-environment

x-gpu-agent-common: &gpu-agent-common
  image: ${AGENT_IMAGE-ghcr.io/boundless-xyz/boundless/bento-agent:latest}
  runtime: ${DOCKER_RUNTIME:-nvidia}
  build:
    context: .
    dockerfile: ${GPU_AGENT_DOCKERFILE:-dockerfiles/agent.dockerfile}
    args:
      BLAKE3_GROTH16_ARTIFACTS_URL: ${BLAKE3_GROTH16_ARTIFACTS_URL:-https://staging-signal-artifacts.beboundless.xyz/v3/proving/blake3_groth16_artifacts.tar.xz}
      USE_LOCAL_BLAKE3_GROTH16_SETUP: ${USE_LOCAL_BLAKE3_GROTH16_SETUP:-0}
  restart: always
  depends_on:
    postgres:
      condition: service_healthy
    redis:
      condition: service_healthy
    minio:
      condition: service_healthy
  environment:
    <<: *base-environment

x-exec-agent-common: &exec-agent-common
  <<: *cpu-agent-common
  mem_limit: 4G
  cpus: 3
  environment:
    <<: *base-environment
    RISC0_KECCAK_PO2: ${RISC0_KECCAK_PO2:-17}
    SEGMENT_SIZE: ${SEGMENT_SIZE:-21}
    REDIS_TTL: ${REDIS_TTL:-57600}
  entrypoint:
    [
      "/bin/sh",
      "-c",
      "/app/agent -t exec --segment-po2 $$SEGMENT_SIZE --redis-ttl $$REDIS_TTL",
    ]

x-broker-environment: &broker-environment
  NO_COLOR: 1
  RUST_LOG: ${RUST_LOG:-info,broker=debug,boundless_market=debug}
  # RUST_BACKTRACE: 1
  PROVER_PRIVATE_KEY: ${PROVER_PRIVATE_KEY:-${PRIVATE_KEY}}
  PROVER_RPC_URL: ${PROVER_RPC_URL:-${RPC_URL}}
  PROVER_RPC_URLS: ${PROVER_RPC_URLS}
  # Per-chain RPC URLs for multichain broker (set only the chains you want to operate on)
  PROVER_RPC_URL_8453: ${PROVER_RPC_URL_8453:-}
  PROVER_RPC_URL_167000: ${PROVER_RPC_URL_167000:-}
  PROVER_RPC_URL_11155111: ${PROVER_RPC_URL_11155111:-}
  PROVER_RPC_URL_84532: ${PROVER_RPC_URL_84532:-}
  BOUNDLESS_MARKET_ADDRESS:
  SET_VERIFIER_ADDRESS:
  ORDER_STREAM_URL:
  # TODO these env vars are temporary for handling cancellations. These should be removed when
  # updated.
  POSTGRES_HOST:
  POSTGRES_DB:
  POSTGRES_PORT:
  POSTGRES_USER:
  POSTGRES_PASS:

x-gpu-prove-agent: &gpu-prove-agent
  <<: *gpu-agent-common
  environment:
    <<: *base-environment
    REDIS_TTL: ${REDIS_TTL:-57600}
  # Detect GPUs and spawn one agent per GPU, then wait for all
  entrypoint:
    [
      "/bin/sh",
      "-c",
      "for gpu in $(nvidia-smi --query-gpu=index --format=csv,noheader,nounits); do CUDA_VISIBLE_DEVICES=$$gpu /app/agent -t prove --redis-ttl $$REDIS_TTL & done; wait",
    ]
  # When USE_LOCAL_BLAKE3_GROTH16_SETUP=1 the image has no artifacts; set BLAKE3_GROTH16_SETUP_DIR
  # to the host path containing verify_for_guest_final.zkey and verify_for_guest_graph.bin.
  # When using Ansible, BLAKE3_GROTH16_SETUP_DIR is set in .env and this dir is populated.
  volumes:
    - type: bind
      source: ${BLAKE3_GROTH16_SETUP_DIR}
      target: /.blake3_groth16_artifacts/
      read_only: true

x-broker-common: &broker-common
  image: ${BROKER_IMAGE-ghcr.io/boundless-xyz/boundless/broker:latest}
  restart: always
  depends_on:
    rest_api:
      condition: service_healthy
    gpu_prove_agent:
      condition: service_started
    exec_agent:
      condition: service_started
    aux_agent:
      condition: service_started
    redis:
      condition: service_healthy
    postgres:
      condition: service_healthy
  profiles: [broker]
  build:
    context: .
    dockerfile: ${BROKER_DOCKERFILE:-dockerfiles/broker.dockerfile}
  mem_limit: 2G
  cpus: 2
  stop_grace_period: 3h
  network_mode: host

services:
  redis:
    hostname: ${REDIS_HOST:-redis}
    image: ${REDIS_IMG:-valkey/valkey:9.0}
    restart: always
    environment:
      REDIS_PASSWORD: ${REDIS_PASSWORD:-}
    ports:
      - 6379:6379
    volumes:
      - redis-data:/data
    command:
      [
        "sh",
        "-c",
        'exec valkey-server --maxmemory-policy allkeys-lru --save 900 1 --appendonly yes $$(test -n "$$REDIS_PASSWORD" && echo "--requirepass $$REDIS_PASSWORD")',
      ]
    healthcheck:
      test:
        [
          "CMD",
          "sh",
          "-c",
          'redis-cli $$(test -n "$$REDIS_PASSWORD" && echo "-a $$REDIS_PASSWORD") ping',
        ]
      interval: 5s
      timeout: 3s
      retries: 5
      start_period: 10s

  postgres:
    hostname: ${POSTGRES_HOST:-postgres}
    image: ${POSTGRES_IMG:-postgres:16.3-bullseye}
    restart: always
    environment:
      POSTGRES_DB: ${POSTGRES_DB:-taskdb}
      POSTGRES_USER: ${POSTGRES_USER:-worker}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-password}
      POSTGRES_PORT: ${POSTGRES_PORT:-5432}
    ports:
      - "${POSTGRES_PORT:-5432}:${POSTGRES_PORT:-5432}"
    volumes:
      - postgres-data:/var/lib/postgresql/data
    command:
      ["postgres", "-p", "${POSTGRES_PORT:-5432}", "-c", "max_connections=500"]
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "pg_isready -U ${POSTGRES_USER:-worker} -d ${POSTGRES_DB:-taskdb}",
        ]
      interval: 5s
      timeout: 5s
      retries: 5
      start_period: 10s

  minio:
    hostname: ${MINIO_HOST:-minio}
    image: ${MINIO_IMG:-minio/minio:RELEASE.2024-05-28T17-19-04Z}
    restart: always
    ports:
      - "9000:9000"
      - "9001:9001"
    volumes:
      - minio-data:/data
    command: server /data --console-address ":9001"
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-admin}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASS:-password}
      MINIO_DEFAULT_BUCKETS: ${MINIO_BUCKET:-workflow}
    healthcheck:
      test: ["CMD", "mc", "ready", "local"]
      interval: 5s
      timeout: 5s
      retries: 5

  minio-init:
    image: ${MINIO_IMG:-minio/minio:RELEASE.2024-05-28T17-19-04Z}
    depends_on:
      minio:
        condition: service_healthy
    restart: "no"
    entrypoint:
      - /bin/bash
      - -c
      - |
        if [ "$${INPUT_TTL_DAYS:-0}" -le 0 ] 2>/dev/null; then
          echo "INPUT_TTL_DAYS is 0 or unset, skipping lifecycle rule"
          exit 0
        fi
        mc alias set local http://$${MINIO_HOST}:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD} &&
        if [[ "$$(mc ilm rule list local/$${MINIO_BUCKET} --json 2>/dev/null)" == *'"Prefix":"inputs/"'* ]]; then
          echo "Lifecycle rule for inputs/ already exists, skipping"
        else
          mc ilm rule add local/$${MINIO_BUCKET} --prefix "inputs/" --expire-days "$${INPUT_TTL_DAYS}" &&
          echo "Lifecycle rule set: inputs/ objects expire after $${INPUT_TTL_DAYS} day(s)"
        fi
    environment:
      MINIO_HOST: ${MINIO_HOST:-minio}
      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-admin}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASS:-password}
      MINIO_BUCKET: ${MINIO_BUCKET:-workflow}
      INPUT_TTL_DAYS: ${INPUT_TTL_DAYS:-0}

  prometheus:
    image: ${PROMETHEUS_IMG:-prom/prometheus:v2.53.0}
    restart: unless-stopped
    ports:
      - "9091:9090"
    volumes:
      - ./dockerfiles/prometheus:/etc/prometheus:ro
      - prometheus-data:/prometheus
    command:
      - "--config.file=/etc/prometheus/prometheus.yml"
      - "--storage.tsdb.retention.time=30d"

  grafana:
    image: ${GRAFANA_IMG:-grafana/grafana:11.0.0}
    restart: unless-stopped
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_LOG_LEVEL=WARN
      - POSTGRES_HOST=${POSTGRES_HOST:-postgres}
      - POSTGRES_DB=${POSTGRES_DB:-taskdb}
      - POSTGRES_PORT=${POSTGRES_PORT:-5432}
      - POSTGRES_USER=${POSTGRES_USER:-worker}
      - POSTGRES_PASS=${POSTGRES_PASSWORD:-password}
      - GF_INSTALL_PLUGINS=frser-sqlite-datasource
    volumes:
      - ./dockerfiles/grafana:/etc/grafana/provisioning/
      - grafana-data:/var/lib/grafana
      - broker-data:/db
    depends_on:
      - postgres
      - redis
      - minio
      - prometheus

  exec_agent:
    <<: *exec-agent-common
    deploy:
      replicas: ${BENTO_EXECUTOR_COUNT:-4}

  aux_agent:
    <<: *cpu-agent-common
    mem_limit: 256M
    deploy:
      replicas: ${BENTO_AUX_WORKER_COUNT:-2}
    cpus: 1
    environment:
      <<: *base-environment
      # Default to 4 max client connections if not set (work, poll for requeue, completed cleanup, stuck tasks cleanup)
      DB_MAX_CONNECTIONS: ${DB_MAX_CONNECTIONS:-4}
      CLEANUP_POLL_INTERVAL:
      STUCK_TASKS_POLL_INTERVAL:
      BENTO_DISABLE_COMPLETED_CLEANUP:
      BENTO_DISABLE_STUCK_TASK_CLEANUP:
      REDIS_TTL: ${REDIS_TTL:-57600}
    entrypoint:
      [
        "/bin/sh",
        "-c",
        "/app/agent -t aux --monitor-requeue --redis-ttl $$REDIS_TTL",
      ]

  # GPU Prover Agent - automatically detects and uses all available GPUs
  # Uses nvidia-smi to detect GPUs and spawns one agent process per GPU
  gpu_prove_agent:
    <<: *gpu-prove-agent
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              # No device_ids restriction - all GPUs are accessible
              # The entrypoint script will detect and use all available GPUs
              capabilities: [gpu]

  rest_api:
    image: ${REST_API_IMAGE-ghcr.io/boundless-xyz/boundless/bento-rest-api:latest}
    build:
      context: .
      dockerfile: ${REST_API_DOCKERFILE:-dockerfiles/rest_api.dockerfile}
    restart: always
    depends_on:
      postgres:
        condition: service_healthy
      minio:
        condition: service_healthy

    mem_limit: 4G
    cpus: 1

    environment:
      <<: *base-environment

    ports:
      - "8081:8081"

    entrypoint: /app/rest_api --bind-addr 0.0.0.0:8081 --snark-timeout ${SNARK_TIMEOUT:-180}
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8081/health"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s

  broker:
    <<: *broker-common
    volumes:
      - type: bind
        source: ./broker.toml
        target: /app/broker.toml
      - type: bind
        source: ./chain-overrides
        target: /app/chain-overrides
        read_only: true
      - broker-data:/db/
    # Uncomment when using locally built set-builder and assessor guest programs
    # - type: bind
    #   source: ./target/riscv-guest/guest-set-builder/set-builder/riscv32im-risc0-zkvm-elf/release/set-builder.bin
    #   target: /target/riscv-guest/guest-set-builder/set-builder/riscv32im-risc0-zkvm-elf/release/set-builder.bin
    # - type: bind
    #   source: ./target/riscv-guest/guest-assessor/assessor-guest/riscv32im-risc0-zkvm-elf/release/assessor-guest.bin
    #   target: /target/riscv-guest/guest-assessor/assessor-guest/riscv32im-risc0-zkvm-elf/release/assessor-guest.bin
    environment:
      <<: *broker-environment
    entrypoint: ["/bin/sh", "-c", "exec /app/broker --db-url 'sqlite:///db/broker.db' --config-file /app/broker.toml --bento-api-url http://localhost:8081 ${BROKER_EXTRA_ARGS:-}"]

  # # Example second broker with different configuration
  # broker2:
  #   <<: *broker-common
  #   volumes:
  #     - type: bind
  #       source: ./broker2.toml
  #       target: /app/broker.toml
  #     - broker2-data:/db/
  #   environment:
  #     <<: *broker-environment
  #     # Note: use a different variable if you want to use different private keys in each broker
  #     PRIVATE_KEY: ${PROVER_PRIVATE_KEY:-${PRIVATE_KEY}}
  #     RPC_URL: ${RPC_URL_2}
  #   entrypoint: /app/broker --db-url 'sqlite:///db/broker2.db' --config-file /app/broker.toml --bento-api-url http://localhost:8081

  miner:
    image: ${BENTO_CLI_IMAGE:-ghcr.io/boundless-xyz/boundless/bento-cli:latest}
    build:
      context: .
      dockerfile: ${BENTO_CLI_DOCKERFILE:-dockerfiles/bento_cli.dockerfile}
    restart: always
    depends_on:
      rest_api:
        condition: service_healthy
    profiles: [miner]
    mem_limit: 2G
    cpus: 2
    network_mode: host
    environment:
      <<: *base-environment
      RUST_LOG: ${RUST_LOG:-info}
    entrypoint: /bin/bash -c "while sleep 1; do /app/bento_cli --iter-count 500000; done"

  caddy:
    image: caddy:2-alpine
    restart: always
    depends_on:
      rest_api:
        condition: service_healthy
    profiles: [caddy]
    network_mode: host
    environment:
      CADDY_DOMAIN: ${CADDY_DOMAIN:-:80}
      CADDY_API_HOST: ${CADDY_API_HOST:-127.0.0.1}
      CADDY_API_PORT: ${CADDY_API_PORT:-8081}
      CADDY_ACME_EMAIL: ${CADDY_ACME_EMAIL:-}
      CADDY_API_KEY: ${CADDY_API_KEY:-}
    volumes:
      - caddy-data:/data
      - caddy-config:/config
      - ./caddy/${CADDYFILE:-Caddyfile}:/etc/caddy/Caddyfile:ro
volumes:
  redis-data:
  postgres-data:
  minio-data:
  prometheus-data:
  grafana-data:
  broker-data:
  caddy-data:
  caddy-config:
  # broker2-data: