docker-compose.yml

# Testing environment for the Yggdrasil workflow.
#
#   make compose-up        # statusdb + yggdrasil
#   make compose-up-full   # the above + biomate + genomics-status
#                          # (--profile full)
#   make compose-up-dind   # yggdrasil image with Nextflow + Docker CLI,
#                          # able to run nf-core pipelines via a docker:dind
#                          # sidecar (profile "dind"); test with
#                          # make compose-demux-test
#   make compose-rebuild   # rebuild all images without cache (recreate containers)
#   make compose-logs      # follow yggdrasil logs
#   make compose-reset     # tear everything down, including volumes
#
# Yggdrasil starts only after its dependencies are healthy (compose
# depends_on) and its entrypoint verifies the seeded databases exist,
# creates its own databases, and (by default) injects a test scenario
# that the daemon then picks up, generates a plan for, and executes.

name: dataflow-testing

services:
  statusdb:
    image: dataflow-statusdb:latest
    build:
      context: ./StatusDB_NGI
      dockerfile: Dockerfile
    environment:
      # Stack-specific variable names (STACK_*) so we never pick up
      # unrelated COUCHDB_* variables from the ambient shell environment.
      COUCHDB_USER: ${STACK_COUCH_USER:-admin}
      COUCHDB_PASSWORD: ${STACK_COUCH_PASSWORD:-secret}
    ports:
      - "5984:5984"
    volumes:
      - couchdb-data:/opt/couchdb/data
      # Test-flowcell seeds live in deploy/seed (not in the submodule —
      # they are only needed by this testing environment). The image's
      # init-seed-data.sh loads every subdirectory of
      # /opt/couchdb/seed_data as a same-named database at first boot,
      # so these mounts are picked up automatically. Re-seeding only
      # happens after make compose-reset (the INIT_MARKER lives on the
      # couchdb-data volume).
      # Deliberately not :ro — the image entrypoint chowns everything
      # under /opt/couchdb at boot (and aborts on a read-only mount), so
      # first boot hands ownership of the mounted files to the image's
      # couchdb user (uid 5984); `sudo chown -R $(id -u):$(id -g)` on
      # the two directories restores editability.
      - ./deploy/seed/demux_sample_info:/opt/couchdb/seed_data/demux_sample_info
      - ./deploy/seed/flowcell_status:/opt/couchdb/seed_data/flowcell_status
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:5984/_up"]
      interval: 5s
      timeout: 5s
      retries: 30
      start_period: 15s
    networks:
      - yggdrasil-net

  biomate:
    image: dataflow-biomate:latest
    build:
      context: ./BioMate
      dockerfile: Dockerfile
    # Started only with --profile full (make compose-up-full /
    # compose-up-dind): nothing at runtime depends on the BioMate web
    # interface (the dmx_realm demux steps are simulated).
    profiles: ["full"]
    command: ["biomate", "web-interface", "--host", "0.0.0.0", "--port", "8765"]
    user: "${HOST_UID:-1000}:${HOST_GID:-1000}"
    ports:
      - "8765:8765"
    # No curl in this image; use python for the healthcheck.
    healthcheck:
      test:
        [
          "CMD",
          "python",
          "-c",
          "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8765/', timeout=5)",
        ]
      interval: 10s
      timeout: 5s
      retries: 18
      start_period: 20s
    networks:
      - yggdrasil-net

  genomics-status:
    image: dataflow-genomics-status:latest
    # Source + configs are baked into the image (see
    # deploy/Dockerfile.genomics-status, which builds on top of the image
    # produced by the submodule's Dockerfile). No bind mounts: the submodule
    # working tree must never be written to at runtime.
    build:
      context: .
      dockerfile: deploy/Dockerfile.genomics-status
    profiles: ["full"]
    ports:
      - "9761:9761"
    healthcheck:
      test:
        [
          "CMD",
          "/opt/conda/bin/python",
          "-c",
          "import urllib.request; urllib.request.urlopen('http://127.0.0.1:9761/', timeout=5)",
        ]
      interval: 10s
      timeout: 5s
      retries: 18
      start_period: 30s
    depends_on:
      statusdb:
        condition: service_healthy
    networks:
      - yggdrasil-net

  yggdrasil:
    # Overridable so the same service can run the base image or the
    # dind-flavoured one (Nextflow + Docker CLI; see
    # deploy/Dockerfile.yggdrasil.dind, Makefile target compose-up-dind).
    image: ${YGG_IMAGE:-dataflow-yggdrasil:latest}
    build:
      context: .
      dockerfile: ${YGG_DOCKERFILE:-deploy/Dockerfile.yggdrasil}
    environment:
      YGG_COUCH_USER: ${STACK_COUCH_USER:-admin}
      YGG_COUCH_PASS: ${STACK_COUCH_PASSWORD:-secret}
      YGG_COUCH_URL: http://statusdb:5984
      YGG_WORK_ROOT: /work
      YGG_EVENT_SPOOL: /events
      OPS_DB: yggdrasil_ops
      YGG_INJECT_TEST_SCENARIO: ${YGG_INJECT_TEST_SCENARIO:-true}
      YGG_SCENARIO_ID: ${YGG_SCENARIO_ID:-test_scenario:compose-demo}
      # Put Nextflow's home (pipeline asset clones + caches) on the shared
      # ygg-work volume: the dind sidecar mounts the same volume at /work,
      # and nested pipeline containers bind-mount repo-internal files
      # (e.g. MULTIQC's assets/multiqc_config.yml) through the dind daemon.
      # Invisible paths make Nextflow stage dangling symlinks there.
      NXF_HOME: /work/nf-nxf
      PYTHONUNBUFFERED: "1"
    volumes:
      # Bind-mount the demux_realm submodule over the copy baked into the
      # image (editable-installed at /app/demux_realm) so local edits are
      # picked up on container (re)start without rebuilding the image.
      - ./demux_realm:/app/demux_realm
      - ygg-work:/work
      - ygg-events:/events
      - ygg-logs:/logs/yggdrasil
      # Tiny Illumina BCL run for nf-core/demultiplex test runs
      # (make compose-demux-test); unused by the base image.
      - ./BioMate/temp:/data/test_run:ro
    depends_on:
      # statusdb only. biomate is profile-gated ("full") and compose
      # rejects depends_on entries that point at profile-excluded
      # services; nothing at runtime needs biomate anyway (the
      # dmx_realm demux steps are simulated, not real BioMate runs).
      statusdb:
        condition: service_healthy
    networks:
      - yggdrasil-net

  # Docker daemon-in-docker sidecar: lets the dind-flavoured yggdrasil
  # image run nf-core pipeline containers with -profile docker.
  # The daemon itself stays arm64 (native runc + network hooks work);
  # amd64-only pipeline tool images (e.g. samshee, bcl-convert) run as
  # leaf containers translated by Rosetta — requires Docker Desktop's
  # "Use Rosetta for x86_64 emulation" to be enabled.
  # Only started when the "dind" profile is active (compose-up-dind).
  dind:
    image: docker:27-dind
    profiles: ["dind"]
    privileged: true
    # Expose the daemon on 0.0.0.0:2375 (plain TCP; TLS is disabled via
    # DOCKER_TLS_CERTDIR="") so the yggdrasil container can reach it at
    # $DOCKER_HOST=tcp://dind:2375. The default command only binds the
    # unix socket.
    command: ["dockerd", "-H", "unix:///var/run/docker.sock", "-H", "tcp://0.0.0.0:2375"]
    environment:
      # The dind image defaults to TLS on :2376 with auto-generated certs
      # (enabled by DOCKER_TLS_CERTDIR); this stack is local-only, so use
      # plain TCP on :2375 (matches DOCKER_HOST baked into the
      # dind-flavoured yggdrasil image).
      DOCKER_TLS_CERTDIR: ""
    volumes:
      - dind-data:/var/lib/docker
      # Share the ygg-work volume: Nextflow work dirs must be visible
      # from BOTH containers so the nested daemon can bind-mount them
      # into pipeline containers (run nextflow with
      # --work-dir /work/... — see compose-demux-test).
      - ygg-work:/work
    healthcheck:
      test: ["CMD-SHELL", "docker info >/dev/null 2>&1"]
      interval: 5s
      timeout: 10s
      retries: 30
      start_period: 10s
    networks:
      - yggdrasil-net

networks:
  yggdrasil-net:

volumes:
  couchdb-data:
  ygg-work:
  ygg-events:
  ygg-logs:
  dind-data: