docker-compose.postgres.yml

# z4j - Postgres-backed production stack.
#
# Use this compose file when you need:
#   - more than one concurrent admin (SQLite is single-writer)
#   - throughput or retention that exceeds your measured SQLite capacity
#   - audit-grade Postgres (range-partitioned events, LISTEN/NOTIFY,
#     tsvector full-text search)
#   - horizontal brain replicas behind a load balancer
#   - compliance regimes (SOC 2, HIPAA, ISO 27001) that mandate Postgres
#
# For simpler deployments (homelab, small team, evaluation), use the
# default `docker-compose.yml` instead - same image, SQLite mode.
#
# Bring this stack up with:
#
#   docker compose -f docker-compose.postgres.yml up -d --build
#
# Services:
#
#   z4j-postgres      PostgreSQL 18 Trixie     persistent volume only,
#                     no exposed port
#   z4j               backend + dashboard,     single image, port 8080
#                     built from this repo's   (put behind a TLS reverse
#                     Dockerfile by default    proxy in production)
#   scheduler-certs   one-shot mTLS bootstrap  mints the CA and both
#                     for the scheduler        certificates into the
#                     channel, then exits      z4j_scheduler_pki volume
#   scheduler         z4j-scheduler serve,     two replicas, Postgres
#                     same image               leader election, mTLS to
#                                              the brain on :7701
#
# Quickstart:
#
#   1. Create a `.env` file at the repo root with at least:
#
#         POSTGRES_PASSWORD=<a long random string>
#         Z4J_SECRET=<openssl rand -hex 48>
#         Z4J_SESSION_SECRET=<openssl rand -hex 48>
#         Z4J_AUDIT_CHAIN_SECRET=<openssl rand -hex 48>
#         Z4J_PUBLIC_URL=https://z4j.example.com
#         Z4J_ALLOWED_HOSTS=["z4j.example.com"]
#
#   2. Build + start:
#
#         docker compose -f docker-compose.postgres.yml up -d --build
#
#   3. Tail the brain logs to find the first-boot setup token:
#
#         docker compose -f docker-compose.postgres.yml logs -f z4j
#
#   4. Open the printed URL, create the admin user, and paste the
#      bearer token from the dashboard "Agents" page into your
#      Celery worker's z4j configuration.
#
# For contributor development with hot-reload + dashboard HMR,
# use docker-compose.dev.yml instead.
#
# See https://docs.z4j.com/guides/self-hosting/ for the full self-host quickstart.

name: z4j

services:
  # ====================================================================
  # PostgreSQL 18 - brain database
  # ====================================================================
  z4j-postgres:
    image: postgres:18.6@sha256:06cad38a5d9f5d24b4d83d86def30795d5e4b757fedbf5281172b576dedcd941
    container_name: z4j-postgres
    environment:
      POSTGRES_USER: ${POSTGRES_USER:-z4j}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
      POSTGRES_DB: ${POSTGRES_DB:-z4j}
      # Tune Postgres settings via the standard env-var hooks.
      # Override in your .env if you have a beefier host.
      POSTGRES_INITDB_ARGS: "--data-checksums"
    command:
      - postgres
      - -c
      - max_connections=200
      - -c
      - shared_buffers=256MB
      - -c
      - effective_cache_size=1GB
      - -c
      - work_mem=8MB
      - -c
      - maintenance_work_mem=64MB
      - -c
      - random_page_cost=1.1
      - -c
      - wal_compression=on
    volumes:
      # Postgres 18+ stores data under a major-version subdir
      # (/var/lib/postgresql/18/...) so the recommended mount is
      # /var/lib/postgresql, NOT the legacy /var/lib/postgresql/data.
      # This makes future pg_upgrade --link work without mount-point
      # boundary issues. See docker-library/postgres#1259.
      - z4j_pg_data:/var/lib/postgresql
    networks:
      - z4j_net
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-z4j} -d ${POSTGRES_DB:-z4j}"]
      interval: 5s
      timeout: 5s
      retries: 10
      start_period: 30s
    restart: unless-stopped
    # Operative resource limits for the packaged example. Validate them
    # against your workload and override them when needed. Without these
    # limits a runaway query or batch ingest can OOM the host and
    # cascade-kill the brain alongside Postgres.
    deploy:
      resources:
        limits:
          memory: 1g
          cpus: "1.0"
        reservations:
          memory: 256m
          cpus: "0.25"
    # IMPORTANT: do NOT expose the postgres port publicly.
    # The brain connects over the internal docker network only.

  # ====================================================================
  # z4j brain - backend + dashboard, single image
  # ====================================================================
  z4j:
    build:
      context: .
      # Use the split-safe release build. backend/Dockerfile depends on
      # monorepo siblings that are absent from the packaged source tree.
      dockerfile: Dockerfile
      target: runtime
      args:
        Z4J_VERSION: ${Z4J_VERSION:-1.12.1}
    image: ${Z4J_BRAIN_IMAGE:-z4jdev/z4j:latest}
    container_name: z4j
    depends_on:
      z4j-postgres:
        condition: service_healthy
    environment:
      # ----- required -----
      # z4j translates libpq-style sslmode/sslrootcert/sslcert/sslkey
      # URL parameters into asyncpg's explicit SSL configuration before
      # SQLAlchemy connects. For brain-to-postgres traffic over this private
      # docker bridge we leave TLS off and use Z4J_REQUIRE_DB_SSL=false below.
      # Pass credentials as data, not by interpolating them into a URL.
      # z4j builds the SQLAlchemy URL with delimiter-safe quoting at its
      # single configuration-capture boundary.
      Z4J_DATABASE_HOST: z4j-postgres
      Z4J_DATABASE_PORT: "5432"
      Z4J_DATABASE_USER: ${POSTGRES_USER:-z4j}
      Z4J_DATABASE_PASSWORD: ${POSTGRES_PASSWORD}
      Z4J_DATABASE_NAME: ${POSTGRES_DB:-z4j}
      Z4J_SECRET: ${Z4J_SECRET:?Z4J_SECRET is required (48+ bytes of random data)}
      Z4J_SESSION_SECRET: ${Z4J_SESSION_SECRET:?Z4J_SESSION_SECRET is required (48+ bytes of random data)}
      Z4J_AUDIT_CHAIN_SECRET: ${Z4J_AUDIT_CHAIN_SECRET:?Z4J_AUDIT_CHAIN_SECRET is required (48+ bytes of independent random data)}
      Z4J_PUBLIC_URL: ${Z4J_PUBLIC_URL:?Z4J_PUBLIC_URL is required (e.g. https://z4j.example.com)}
      Z4J_ALLOWED_HOSTS: ${Z4J_ALLOWED_HOSTS:?Z4J_ALLOWED_HOSTS is required (e.g. ["z4j.example.com"])}

      # ----- safety / hardening -----
      #
      # !!!  REMOTE / MANAGED DB WARNING - READ THIS  !!!
      #
      # The line below disables Postgres TLS verification because
      # this compose talks brain↔z4j-postgres over a PRIVATE docker
      # bridge with no outside access. If you point
      # z4j at a remote / managed database (RDS, Neon, Supabase,
      # Cloud SQL, ...) you MUST use an override file that supplies:
      #   1. Z4J_DATABASE_URL with verified TLS, for example:
      #      postgresql+asyncpg://user:pass@db.example/z4j?sslmode=verify-full&sslrootcert=/run/secrets/postgres-ca.pem
      #      (mount the CA file at that container path; an explicit URL
      #      takes precedence over the structured fields above)
      #   2. Z4J_REQUIRE_DB_SSL=true
      #      (or just delete the line below - true is the default).
      # sslmode=require encrypts but does not verify the server identity
      # unless sslrootcert is also supplied. Prefer verify-full for a remote
      # database. The brain refuses to start in production with sslmode=disable
      # unless this flag is explicitly set to false (R3 release-
      # blocker - quiet credentials-on-the-wire was the failure
      # mode otherwise). Set ``Z4J_REQUIRE_DB_SSL=false`` only when
      # you're 100% sure your DB lives on a trusted private network.
      Z4J_REQUIRE_DB_SSL: ${Z4J_REQUIRE_DB_SSL:-false}

      # ----- operational -----
      Z4J_LOG_LEVEL: ${Z4J_LOG_LEVEL:-INFO}
      Z4J_LOG_JSON: ${Z4J_LOG_JSON:-true}
      Z4J_ENVIRONMENT: ${Z4J_ENVIRONMENT:-production}

      # --- optional: skip interactive first-boot by seeding an admin ---
      # When both are set and the users table is empty, the brain
      # provisions the admin user without printing a setup URL. This is
      # the zero-log-exposure path the setup banner itself recommends;
      # without this passthrough the vars in your .env never reach the
      # container and the token banner prints anyway.
      Z4J_BOOTSTRAP_ADMIN_EMAIL: ${Z4J_BOOTSTRAP_ADMIN_EMAIL:-}
      Z4J_BOOTSTRAP_ADMIN_PASSWORD: ${Z4J_BOOTSTRAP_ADMIN_PASSWORD:-}
      Z4J_BOOTSTRAP_ADMIN_DISPLAY_NAME: ${Z4J_BOOTSTRAP_ADMIN_DISPLAY_NAME:-}

      # ----- retention -----
      Z4J_EVENT_RETENTION_DAYS: ${Z4J_EVENT_RETENTION_DAYS:-30}
      Z4J_AUDIT_RETENTION_DAYS: ${Z4J_AUDIT_RETENTION_DAYS:-90}

      # ----- registry (multi-worker safe by default) -----
      Z4J_REGISTRY_BACKEND: postgres_notify

      # ----- scheduler channel (mTLS gRPC on :7701, bridge network only) -----
      # The brain serves the z4j-scheduler gRPC service with the server
      # certificate that scheduler-certs minted into the z4j_scheduler_pki
      # volume, and accepts exactly one client identity: the CN the same
      # one-shot minted for the scheduler replicas below. The port is never
      # published to the host.
      Z4J_SCHEDULER_GRPC_ENABLED: "true"
      Z4J_SCHEDULER_GRPC_BIND_HOST: "0.0.0.0"
      Z4J_SCHEDULER_GRPC_BIND_PORT: "7701"
      Z4J_SCHEDULER_GRPC_TLS_CERT: /pki/brain.crt
      Z4J_SCHEDULER_GRPC_TLS_KEY: /pki/brain.key
      Z4J_SCHEDULER_GRPC_TLS_CA: /pki/ca.crt
      Z4J_SCHEDULER_GRPC_ALLOWED_CNS: '["${Z4J_PKI_SCHEDULER_CN:-z4j-scheduler}"]'
      Z4J_SCHEDULER_GRPC_REQUIRE_ALLOWLIST: "true"
      # The dashboard's Schedulers page polls each listed /info endpoint.
      # Compose names the replicas <project>-scheduler-<n>, and this file's
      # project name is z4j. Extend the list when you scale past two or
      # run the stack under another -p project name.
      Z4J_SCHEDULER_INFO_URLS: ${Z4J_SCHEDULER_INFO_URLS:-["http://z4j-scheduler-1:7800","http://z4j-scheduler-2:7800"]}
    ports:
      - "${Z4J_BRAIN_PORT:-8080}:7700"
      # IMPORTANT: put this behind a TLS-terminating reverse proxy
      # (Caddy, nginx, Traefik) in production. Do NOT expose port
      # 8080 directly to the public internet.
    volumes:
      # The 1.7 -> 1.8 activation ceremony creates and consumes its
      # owner-private manifest in separate ``docker compose run --rm``
      # containers. A named volume is required; the image's anonymous
      # /data volume would be discarded with the first container.
      - z4j_brain_state:/data
      # mTLS material for the scheduler channel, minted by scheduler-certs.
      - z4j_scheduler_pki:/pki:ro
    networks:
      - z4j_net
    healthcheck:
      test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:7700/api/v1/health',timeout=3).status==200 else 1)"]
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 30s
    restart: unless-stopped
    # Operative resource limits for the packaged example. Validate them
    # against your workload and override them when needed. Without a
    # limit a misbehaving consumer or accidentally-massive payload
    # can OOM the host and take Postgres with it.
    deploy:
      resources:
        limits:
          memory: 1g
          cpus: "2.0"
        reservations:
          memory: 256m
          cpus: "0.5"
    # tini is the entrypoint inside the image; we get clean PID 1
    # signal handling for free.

  # ====================================================================
  # scheduler-certs - one-shot mTLS bootstrap for the scheduler channel
  # ====================================================================
  # Runs docker/scheduler-certs.sh (POSIX sh + openssl, both in the brain
  # image) as root once per `docker compose up`: mints a private CA, the
  # brain's server certificate (SANs z4j, brain, localhost) and the
  # scheduler's client certificate (CN z4j-scheduler) into the
  # z4j_scheduler_pki volume, hands them to the uid the brain and the
  # scheduler run as, and exits. Idempotent: a valid bundle is left alone
  # and a certificate is minted again only when it is missing, mismatched
  # or within 30 days of expiry (restart brain + scheduler to pick up a
  # renewal). The CA key stays root-only. Bring your own PKI by replacing
  # the files in the volume, keeping their names.
  scheduler-certs:
    image: ${Z4J_BRAIN_IMAGE:-z4jdev/z4j:latest}
    container_name: z4j-scheduler-certs
    user: "0:0"
    network_mode: none
    entrypoint: ["/usr/bin/tini", "--", "sh", "/usr/local/libexec/z4j/scheduler-certs.sh"]
    environment:
      Z4J_PKI_DIR: /pki
      Z4J_PKI_BRAIN_SANS: z4j,brain,localhost
      Z4J_PKI_SCHEDULER_CN: ${Z4J_PKI_SCHEDULER_CN:-z4j-scheduler}
      Z4J_PKI_OWNER: "10001:10001"
    volumes:
      - ./docker/scheduler-certs.sh:/usr/local/libexec/z4j/scheduler-certs.sh:ro
      - z4j_scheduler_pki:/pki
    restart: "no"

  # ====================================================================
  # z4j-scheduler - two replicas, Postgres leader election
  # ====================================================================
  # The brain image with its entrypoint switched to `z4j-scheduler serve`.
  # Both replicas dial the brain's gRPC channel at z4j:7701 with the same
  # minted client certificate and race for an advisory lock in the
  # stack's own Postgres; the holder ticks, the other stays warm and
  # takes over within a few heartbeats. No container_name, so Compose
  # names them z4j-scheduler-1 and z4j-scheduler-2. Add replicas with:
  #
  #   docker compose -f docker-compose.postgres.yml up -d --scale scheduler=3
  #
  # /health, /ready and /info answer on :7800 inside z4j_net only.
  # /metrics ships off: the scheduler binds every container interface and
  # refuses unauthenticated metrics outside dev, so set BOTH
  # Z4J_SCHEDULER_METRICS_ENABLED=true and Z4J_SCHEDULER_METRICS_AUTH_TOKEN
  # in .env to scrape it (an empty token locks the endpoint rather than
  # opening it).
  scheduler:
    image: ${Z4J_BRAIN_IMAGE:-z4jdev/z4j:latest}
    entrypoint: ["/usr/bin/tini", "--", "z4j-scheduler"]
    command: ["serve"]
    depends_on:
      scheduler-certs:
        condition: service_completed_successfully
      z4j-postgres:
        condition: service_healthy
      z4j:
        condition: service_healthy
    environment:
      Z4J_SCHEDULER_BRAIN_GRPC_URL: z4j:7701
      Z4J_SCHEDULER_BRAIN_REST_URL: http://z4j:7700
      Z4J_SCHEDULER_TLS_CERT: /pki/scheduler.crt
      Z4J_SCHEDULER_TLS_KEY: /pki/scheduler.key
      Z4J_SCHEDULER_TLS_CA: /pki/ca.crt
      Z4J_SCHEDULER_LEADER_BACKEND: postgres
      # Advisory-lock election in the stack's own database, from the same
      # POSTGRES_* values the brain uses. This is a URL, so a password that
      # contains URL delimiters (: @ / % ?) must be percent-encoded here;
      # set Z4J_SCHEDULER_LEADER_PG_DSN in .env to supply the DSN yourself.
      Z4J_SCHEDULER_LEADER_PG_DSN: ${Z4J_SCHEDULER_LEADER_PG_DSN:-postgres://${POSTGRES_USER:-z4j}:${POSTGRES_PASSWORD}@z4j-postgres:5432/${POSTGRES_DB:-z4j}}
      Z4J_SCHEDULER_BIND_HOST: "0.0.0.0"
      Z4J_SCHEDULER_BIND_PORT: "7800"
      Z4J_SCHEDULER_ENVIRONMENT: ${Z4J_ENVIRONMENT:-production}
      Z4J_SCHEDULER_METRICS_ENABLED: ${Z4J_SCHEDULER_METRICS_ENABLED:-false}
      Z4J_SCHEDULER_METRICS_AUTH_TOKEN: ${Z4J_SCHEDULER_METRICS_AUTH_TOKEN:-}
      Z4J_SCHEDULER_LOG_LEVEL: ${Z4J_LOG_LEVEL:-INFO}
      Z4J_SCHEDULER_LOG_JSON: ${Z4J_LOG_JSON:-true}
    volumes:
      - z4j_scheduler_pki:/pki:ro
    networks:
      - z4j_net
    healthcheck:
      test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:7800/ready',timeout=3).status==200 else 1)"]
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 30s
    restart: unless-stopped
    deploy:
      replicas: 2
      resources:
        limits:
          memory: 256m
          cpus: "0.5"
        reservations:
          memory: 64m
          cpus: "0.05"

networks:
  # Private network for brain ↔ postgres traffic. Both services
  # join this network explicitly so the only host-exposed surface
  # is the brain's port 8080. Postgres has no port mapping and is
  # unreachable from the host or any other compose project.
  z4j_net:
    name: z4j_net
    driver: bridge

volumes:
  z4j_pg_data:
    name: z4j_pg_data
  z4j_brain_state:
    name: z4j_brain_state
  # CA, brain server cert and scheduler client cert (see scheduler-certs).
  z4j_scheduler_pki:
    name: z4j_scheduler_pki