From 785eb2aa43559fdc18751cc9f1a644a909b5fdd7 Mon Sep 17 00:00:00 2001 From: Michael D Date: Fri, 4 Sep 2026 07:27:47 +0800 Subject: [PATCH 01/13] Add non-interactive setup, fix reset leaving an orphan sidecar Supports Condor delegating to this setup instead of hand-writing .env, which is how the two schemas drifted apart. - HBAPI_NONINTERACTIVE=1 takes the three typed credentials from the environment (HBAPI_USERNAME / _PASSWORD / _CONFIG_PASSWORD) and aborts naming any that are missing rather than writing a partial .env. The interactive path is unchanged. Everything this script GENERATES -- both broker passwords above all -- stays generated in both modes, so a caller cannot supply a weak one. - TAILSCALE_MODE (none|host|sidecar) is accepted as a forward-looking alias; TAILSCALE_ENABLED remains the .env key and keeps working. Caller-supplied TAILSCALE_AUTH_KEY / _HOSTNAME are captured before the defaults reset them, and validated. - HBAPI_SKIP_DEPS=1 skips the apt/Docker install pass for a caller that has already established both. Without it, delegation re-ran an apt-get build-dep pass and a Docker probe on every Condor install. The Hummingbot image pull is deliberately NOT skipped -- no caller covers that today. - make reset now tears down with both compose files and --remove-orphans. The Tailscale sidecar and its state volume are declared only in the overlay, so a base-file-only `down -v` left the container running -- still holding the host's tailscale0 device -- while .env was deleted, so the next `make setup` had no record it existed. Verified: base-only down leaves the overlay container up; two-file down --remove-orphans removes both it and its volume. - clear failing on unset TERM no longer aborts setup under `set -e`. Co-Authored-By: Claude Opus 5 --- Makefile | 11 ++++-- setup.sh | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 105 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index f1161a74..3f81ba7a 100644 --- a/Makefile +++ b/Makefile @@ -202,9 +202,14 @@ build: # - removes all .yml files under bots/credentials/master_account/ reset: @echo "[INFO] Checking for running hummingbot-api services..." - @if docker ps --format '{{.Names}}' 2>/dev/null | grep -qx 'hummingbot-api'; then \ - echo "[INFO] Docker containers running — stopping and wiping volumes..."; \ - docker compose down -v; \ + @# Both compose files, always. The Tailscale sidecar and its state volume are + @# declared ONLY in the overlay, so a base-file-only `down -v` leaves the + @# container running as an orphan -- still holding the host's tailscale0 + @# device -- while .env is deleted, so the next `make setup` has no record + @# that it exists. --remove-orphans covers a sidecar left by an older layout. + @if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qxE 'hummingbot-api|$(TAILSCALE_CONTAINER)'; then \ + echo "[INFO] Docker containers found — stopping and wiping volumes..."; \ + docker compose -f docker-compose.yml -f docker-compose.tailscale.yml down -v --remove-orphans; \ else \ echo "[INFO] No Docker containers running."; \ fi diff --git a/setup.sh b/setup.sh index 62d20277..6fd6f958 100755 --- a/setup.sh +++ b/setup.sh @@ -393,12 +393,29 @@ pull_hummingbot_image() { # -------------------------- echo "[INFO] OS=${OS} ARCH=${ARCH}" +# HBAPI_SKIP_DEPS=1 skips the apt/Docker installation pass only -- for a caller +# that has already established both (Condor's installer checks `docker info` +# before it ever gets here). Without it, delegation means every Condor install +# re-runs an apt-get build-dep pass and a Docker install probe that cannot +# change anything, adding minutes to a run that already satisfied them. +# +# Deliberately NOT skipped: the Hummingbot image pull below. Dependency checks +# are the caller's to vouch for; a missing bot image is a functional gap no +# caller currently covers. +if [ "${HBAPI_SKIP_DEPS:-0}" = "1" ]; then + echo "[INFO] Skipping dependency install (HBAPI_SKIP_DEPS=1) — caller vouches for docker + build deps." + DOCKER_ALREADY_PRESENT=true + COMPOSE_ALREADY_PRESENT=true +else + if is_linux; then install_linux_build_deps fi ensure_docker_and_compose +fi + # Show summary of what was done echo "" if [ "$DOCKER_ALREADY_PRESENT" = true ] && [ "$COMPOSE_ALREADY_PRESENT" = true ]; then @@ -437,15 +454,87 @@ if [ -f ".env" ]; then exit 0 fi +# -------------------------- +# Non-interactive mode +# -------------------------- +# Lets another installer (Condor's setup-environment.sh, CI, a provisioning +# script) drive this setup without a human. It exists because the alternative +# -- a caller hand-writing .env itself -- is how the schema drifted before: +# a second author shipped BROKER_PASSWORD=password and omitted +# BROKER_DASHBOARD_PASSWORD entirely, leaving the broker with credentials that +# did not match its own bootstrap file. +# +# Only the values a human would TYPE are accepted here. Everything this script +# GENERATES -- both broker passwords above all -- stays generated in both +# modes, so a caller cannot supply a weak one. That is the whole point: .env +# has exactly one author regardless of who started the run. +# +# The prompts below read from /dev/tty, so they cannot be fed by a pipe; this +# is the supported way to answer them programmatically. +HBAPI_NONINTERACTIVE="${HBAPI_NONINTERACTIVE:-0}" + +# Captured before the defaults below reset them -- these three are the only +# Tailscale settings a caller may supply, and the reset would otherwise +# silently discard them. +_env_ts_enabled="${TAILSCALE_ENABLED:-}" +_env_ts_key="${TAILSCALE_AUTH_KEY:-}" +_env_ts_host="${TAILSCALE_HOSTNAME:-}" + # Clear screen before prompting user (only if running interactively) -if [[ -t 0 ]] && [[ -c /dev/tty ]]; then - if has_cmd clear; then - clear - else - printf "\033c" - fi +if [ "$HBAPI_NONINTERACTIVE" != "1" ] && [[ -t 0 ]] && [[ -c /dev/tty ]]; then + # `clear` exits non-zero when TERM is unset (a tty without a termcap entry -- + # CI runners, `script` wrappers, some IDE terminals). Under `set -e` that + # aborted the whole setup before a single prompt, so fall back to the escape + # sequence rather than trusting the exit status. + clear 2>/dev/null || printf "\033c" fi +TAILSCALE_ENABLED=false +TAILSCALE_AUTH_KEY="" +TAILSCALE_HOSTNAME="hummingbot-api" + +if [ "$HBAPI_NONINTERACTIVE" = "1" ]; then + echo "Hummingbot API Setup (non-interactive)" + echo "" + # Fail loudly rather than writing a half-configured .env: these three guard + # an API that can place orders and read balances, and there is no safe + # default for any of them. + : "${HBAPI_USERNAME:?HBAPI_NONINTERACTIVE=1 requires HBAPI_USERNAME}" + : "${HBAPI_PASSWORD:?HBAPI_NONINTERACTIVE=1 requires HBAPI_PASSWORD}" + : "${HBAPI_CONFIG_PASSWORD:?HBAPI_NONINTERACTIVE=1 requires HBAPI_CONFIG_PASSWORD}" + USERNAME="$HBAPI_USERNAME" + PASSWORD="$HBAPI_PASSWORD" + CONFIG_PASSWORD="$HBAPI_CONFIG_PASSWORD" + + # TAILSCALE_ENABLED is the caller-facing switch and keeps its historical + # name. TAILSCALE_MODE is accepted as a forward-looking alias so callers can + # already say what they mean (none|host|sidecar); only "none" differs from + # enabled=true today, the host/sidecar split lands with the ownership work. + case "${TAILSCALE_MODE:-}" in + none) TAILSCALE_ENABLED=false ;; + host|sidecar) TAILSCALE_ENABLED=true ;; + "") case "$_env_ts_enabled" in + [Tt]rue|[Yy]es|1) TAILSCALE_ENABLED=true ;; + *) TAILSCALE_ENABLED=false ;; + esac ;; + *) echo "[ERROR] TAILSCALE_MODE must be none, host or sidecar (got: $TAILSCALE_MODE)" >&2; exit 1 ;; + esac + + if [ "$TAILSCALE_ENABLED" = true ]; then + TAILSCALE_AUTH_KEY="$_env_ts_key" + TAILSCALE_HOSTNAME="${_env_ts_host:-hummingbot-api}" + if [ -z "$TAILSCALE_AUTH_KEY" ]; then + echo "[ERROR] Tailscale requested but TAILSCALE_AUTH_KEY is empty" >&2 + exit 1 + fi + if [[ ! "$TAILSCALE_AUTH_KEY" =~ ^tskey-auth- ]]; then + echo "[ERROR] TAILSCALE_AUTH_KEY must start with 'tskey-auth-'" >&2 + exit 1 + fi + fi + echo "[INFO] Credentials taken from the environment; broker passwords generated locally." +else + echo "Hummingbot API Setup" echo "" echo "Set API credentials (use a strong username, password, and config password):" @@ -458,9 +547,6 @@ CONFIG_PASSWORD="$(prompt_required_secret_tty "Config password: ")" # -------------------------- # Tailscale Configuration # -------------------------- -TAILSCALE_ENABLED=false -TAILSCALE_AUTH_KEY="" -TAILSCALE_HOSTNAME="hummingbot-api" if prompt_yes_no "Use Tailscale for secure private networking? [y/N]: " "n"; then echo "" @@ -488,6 +574,8 @@ if prompt_yes_no "Use Tailscale for secure private networking? [y/N]: " "n"; the TAILSCALE_ENABLED=true fi +fi # end interactive / non-interactive split + # Broker credentials are never typed by the user — the API and the bots are the only clients — # so generate strong ones instead of shipping well-known defaults. Rotating BROKER_PASSWORD # later requires `make emqx-auth-reset`, since EMQX only imports the bootstrap file for users From f4397908fe40d997cbd5b8da7de18b93358e1299 Mon Sep 17 00:00:00 2001 From: Michael D Date: Fri, 4 Sep 2026 07:45:00 +0800 Subject: [PATCH 02/13] Resolve tailnet ownership by predicate, enforced at deploy Two kernel-mode tailscaled in one network namespace cannot both own tailscale0. The loser dies with tstun.New("tailscale0"): device or resource busy inside a container that stays "running", so restart:unless-stopped never fires and docker ps shows it healthy. - tailnet-state.sh: one predicate, sourced by setup.sh and the Makefile. Detects the RESOURCE, not the sibling product -- the most common conflict is a VPS already running Tailscale for the admin's own SSH access, which no "is Condor here?" check would ever see. - TAILSCALE_MODE (none|host|sidecar) records how. TAILSCALE_ENABLED keeps its historical meaning and remains the switch; enabled with no mode resolves from the host. - Enforcement lives in deploy, not only setup. A setup-time decision is bypassed by every normal day-2 command: reset deletes .env while a sidecar may still run, and deploy is routinely used alone against an .env written weeks earlier. Mode sidecar on a host that already has a daemon starts the sidecar with TS_USERSPACE=true instead. Netstack claims no TUN device and no host routes, so the two coexist -- and correctness stops depending on detection being right. Verified on a live tailnet: a userspace node registered alongside a kernel-mode one in the same netns and served a loopback-bound port to the other node, with a single tailscale0 between them. - Host mode needs no auth key. A key registers a NEW node; reusing the one this machine already has does not. Demanding one failed installs that were entirely correct. - Fix the serve syntax: the two-positional form is rejected by current clients (tested against 1.102.3), and host mode depends on it. - A backtick in an .env heredoc comment executed a make target mid-setup and seeded the broker bootstrap with the default password while .env got the generated one. Caught by the emqx-bootstrap assertion. Co-Authored-By: Claude Opus 5 --- Makefile | 45 ++++++++++--- docker-compose.tailscale.yml | 8 ++- setup.sh | 126 +++++++++++++++++++++++++++-------- tailnet-state.sh | 93 ++++++++++++++++++++++++++ 4 files changed, 234 insertions(+), 38 deletions(-) create mode 100644 tailnet-state.sh diff --git a/Makefile b/Makefile index 3f81ba7a..391aae78 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ run: emqx-auth sudo tailscale up --authkey="$${TAILSCALE_AUTH_KEY}" --hostname="$${TAILSCALE_HOSTNAME:-hummingbot-api}" --accept-dns=true; \ fi; \ tailscale serve status 2>/dev/null | grep -q ":8000" || \ - sudo tailscale serve --bg http:8000 http://localhost:8000; \ + sudo tailscale serve --bg --tcp=8000 tcp://127.0.0.1:8000; \ echo "[INFO] Binding uvicorn to 127.0.0.1 (tailscale serve exposes port 8000 on tailnet)"; \ conda run --no-capture-output -n hummingbot-api uvicorn main:app --reload --host 127.0.0.1 --port 8000; \ else \ @@ -33,15 +33,44 @@ run: emqx-auth fi # Deploy with Docker -# When TAILSCALE_ENABLED=true: adds the Tailscale sidecar compose override +# +# Tailnet ownership is resolved HERE, not only at setup time. A setup-time +# decision is bypassed by every normal day-2 command: `make reset` deletes +# .env while a sidecar may still be running, and this target is routinely run +# on its own against an .env written weeks earlier. Whatever setup.sh recorded +# is a preference; what the host actually looks like right now is the fact. +# +# none plain compose, no overlay +# host this machine is already on the tailnet -- expose 8000 via +# `tailscale serve` on the existing node instead of joining twice +# sidecar the API gets its own tailnet node, in userspace mode when a +# native daemon is already present deploy: $(SETUP_SENTINEL) emqx-auth @set -a; [ -f .env ] && . ./.env; set +a; \ - if [ "$${TAILSCALE_ENABLED:-false}" = "true" ]; then \ - echo "[INFO] Deploying with Tailscale sidecar..."; \ - docker compose -f docker-compose.yml -f docker-compose.tailscale.yml up -d; \ - else \ - docker compose up -d; \ - fi + . ./tailnet-state.sh; \ + mode="$$(tailnet_mode)" || exit 1; \ + state="$$(tailnet_state)"; \ + case "$$mode" in \ + none) \ + docker compose up -d ;; \ + host) \ + echo "[INFO] Tailscale: reusing this host's existing node (mode=host)."; \ + docker compose up -d; \ + if command -v tailscale >/dev/null 2>&1; then \ + tailscale serve status 2>/dev/null | grep -q ":8000" || \ + sudo tailscale serve --bg --tcp=8000 tcp://127.0.0.1:8000 || \ + echo "[WARN] Could not configure serve for :8000 — run it yourself: sudo tailscale serve --bg --tcp=8000 tcp://127.0.0.1:8000"; \ + else \ + echo "[WARN] mode=host but no tailscale CLI on PATH — port 8000 is loopback-only."; \ + fi ;; \ + sidecar) \ + if [ "$$state" = native ]; then \ + echo "[INFO] A tailscaled already owns this host's tailnet device."; \ + echo "[INFO] Starting the sidecar in userspace mode so both can coexist."; \ + export TS_USERSPACE=true; \ + fi; \ + docker compose -f docker-compose.yml -f docker-compose.tailscale.yml up -d ;; \ + esac # Verify dependencies, .env, containers, port exposure and API access. # Read-only; exits non-zero when a check actually fails. diff --git a/docker-compose.tailscale.yml b/docker-compose.tailscale.yml index 22e3e8ed..921d99ea 100644 --- a/docker-compose.tailscale.yml +++ b/docker-compose.tailscale.yml @@ -26,7 +26,13 @@ services: environment: - TS_AUTHKEY=${TAILSCALE_AUTH_KEY} - TS_STATE_DIR=/var/lib/tailscale - - TS_USERSPACE=false + # Kernel mode by default -- but `make deploy` sets TS_USERSPACE=true when + # this host already runs its own tailscaled, because two kernel-mode + # daemons in one network namespace cannot both own tailscale0 and the + # loser dies silently inside a container that still reports "running". + # Netstack coexists with a kernel-mode daemon unconditionally and can + # still serve a loopback-bound port, which is all this sidecar does. + - TS_USERSPACE=${TS_USERSPACE:-false} - TS_HOSTNAME=${TAILSCALE_HOSTNAME:-hummingbot-api} - TS_SERVE_CONFIG=/config/tailscale-serve.json volumes: diff --git a/setup.sh b/setup.sh index 6fd6f958..16cc4c6a 100755 --- a/setup.sh +++ b/setup.sh @@ -523,13 +523,20 @@ if [ "$HBAPI_NONINTERACTIVE" = "1" ]; then if [ "$TAILSCALE_ENABLED" = true ]; then TAILSCALE_AUTH_KEY="$_env_ts_key" TAILSCALE_HOSTNAME="${_env_ts_host:-hummingbot-api}" - if [ -z "$TAILSCALE_AUTH_KEY" ]; then - echo "[ERROR] Tailscale requested but TAILSCALE_AUTH_KEY is empty" >&2 - exit 1 - fi - if [[ ! "$TAILSCALE_AUTH_KEY" =~ ^tskey-auth- ]]; then - echo "[ERROR] TAILSCALE_AUTH_KEY must start with 'tskey-auth-'" >&2 - exit 1 + # An auth key registers a NEW node, so only the sidecar needs one. In host + # mode we serve on a node this machine already has, and demanding a + # credential for that would fail installs that are entirely correct -- + # which is exactly the co-located Condor case. + if [ "${TAILSCALE_MODE:-}" != "host" ]; then + if [ -z "$TAILSCALE_AUTH_KEY" ]; then + echo "[ERROR] Tailscale requested but TAILSCALE_AUTH_KEY is empty" >&2 + echo " (not needed when TAILSCALE_MODE=host — this host would reuse its own node)" >&2 + exit 1 + fi + if [[ ! "$TAILSCALE_AUTH_KEY" =~ ^tskey-auth- ]]; then + echo "[ERROR] TAILSCALE_AUTH_KEY must start with 'tskey-auth-'" >&2 + exit 1 + fi fi fi echo "[INFO] Credentials taken from the environment; broker passwords generated locally." @@ -549,33 +556,83 @@ CONFIG_PASSWORD="$(prompt_required_secret_tty "Config password: ")" # -------------------------- if prompt_yes_no "Use Tailscale for secure private networking? [y/N]: " "n"; then - echo "" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo " How to get a Tailscale auth key:" - echo " 1. Create a free account at https://tailscale.com" - echo " 2. Go to: https://tailscale.com/admin/settings/keys" - echo " 3. Click 'Generate auth key'" - echo " 4. Check 'Reusable' for multiple server deployments" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "" - while true; do - TAILSCALE_AUTH_KEY="$(prompt_tty "Tailscale auth key (tskey-auth-...): " "")" - if [[ -z "$TAILSCALE_AUTH_KEY" ]]; then - echo "[WARN] Auth key cannot be empty" - continue - fi - if [[ ! "$TAILSCALE_AUTH_KEY" =~ ^tskey-auth- ]]; then - echo "[WARN] Auth key must start with 'tskey-auth-'" - continue - fi - break - done - # Hostname defaults to "hummingbot-api" — override via TAILSCALE_HOSTNAME in .env if needed TAILSCALE_ENABLED=true + # Ask what the answer depends on before asking for an auth key. A machine + # that already runs tailscaled needs no key -- it is on the tailnet, and + # joining a second time from the same host would only contend for the same + # device. Prompting anyway would demand a credential to do something we are + # about to decide not to do. + # shellcheck source=tailnet-state.sh + . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/tailnet-state.sh" + if [ "$(tailnet_state)" != none ]; then + TAILSCALE_MODE=host + echo "" + echo "[OK] This machine is already on a tailnet — no auth key needed." + echo " Port 8000 will be served on the node it already has." + echo " Want the API to have its own tailnet identity instead?" + echo " Set TAILSCALE_MODE=sidecar and TAILSCALE_AUTH_KEY in .env, then 'make deploy'." + else + TAILSCALE_MODE=sidecar + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " How to get a Tailscale auth key:" + echo " 1. Create a free account at https://tailscale.com" + echo " 2. Go to: https://tailscale.com/admin/settings/keys" + echo " 3. Click 'Generate auth key'" + echo " 4. Check 'Reusable' for multiple server deployments" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "" + while true; do + TAILSCALE_AUTH_KEY="$(prompt_tty "Tailscale auth key (tskey-auth-...): " "")" + if [[ -z "$TAILSCALE_AUTH_KEY" ]]; then + echo "[WARN] Auth key cannot be empty" + continue + fi + if [[ ! "$TAILSCALE_AUTH_KEY" =~ ^tskey-auth- ]]; then + echo "[WARN] Auth key must start with 'tskey-auth-'" + continue + fi + break + done + # Hostname defaults to "hummingbot-api" — override via TAILSCALE_HOSTNAME in .env if needed + fi fi fi # end interactive / non-interactive split +# -------------------------- +# Tailnet ownership +# -------------------------- +# Recorded in .env as a preference; `make deploy` re-resolves it against the +# live host on every run, so this is a starting point, not a guarantee. +_setup_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tailnet-state.sh +. "$_setup_here/tailnet-state.sh" + +if [ "$TAILSCALE_ENABLED" != true ]; then + TAILSCALE_MODE=none +elif [ -n "${TAILSCALE_MODE:-}" ]; then + : # caller was explicit (validated earlier); respect it +else + case "$(tailnet_state)" in + native) + TAILSCALE_MODE=host + echo "" + echo "[INFO] This machine is already on a tailnet." + echo "[INFO] Port 8000 will be served on the node it already has, rather than" + echo "[INFO] joining a second time — two daemons cannot share one tailnet device." + echo "[INFO] Want a separate identity anyway? Set TAILSCALE_MODE=sidecar in .env." + ;; + sidecar) + TAILSCALE_MODE=sidecar + echo "[INFO] Reusing the existing hummingbot-tailscale sidecar." + ;; + *) + TAILSCALE_MODE=sidecar + ;; + esac +fi + # Broker credentials are never typed by the user — the API and the bots are the only clients — # so generate strong ones instead of shipping well-known defaults. Rotating BROKER_PASSWORD # later requires `make emqx-auth-reset`, since EMQX only imports the bootstrap file for users @@ -633,7 +690,18 @@ GATEWAY_PASSPHRASE=$CONFIG_PASSWORD BOTS_PATH=$(pwd) # Tailscale +# TAILSCALE_ENABLED is the switch and keeps its historical meaning. +# TAILSCALE_MODE says HOW: host = share this machine's existing tailnet node, +# sidecar = give the API a node of its own. Deploy re-resolves this against +# the live host every time, so an .env that was right in June cannot put a +# second kernel-mode daemon on a box that grew one in July. +# +# NOTE: this heredoc is unquoted so plain variables expand. Command +# substitution in these comments would EXECUTE while .env is written -- a +# backtick here once ran a make target mid-setup and seeded the broker with +# the wrong password. Keep substitution out of the comments. TAILSCALE_ENABLED=$TAILSCALE_ENABLED +TAILSCALE_MODE=$TAILSCALE_MODE TAILSCALE_AUTH_KEY=$TAILSCALE_AUTH_KEY TAILSCALE_HOSTNAME=$TAILSCALE_HOSTNAME EOF diff --git a/tailnet-state.sh b/tailnet-state.sh new file mode 100644 index 00000000..bbf89068 --- /dev/null +++ b/tailnet-state.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Who, if anyone, already owns this host's tailnet interface? +# +# Sourced by setup.sh (to pick a default mode) and by the Makefile (to enforce +# one at deploy time). Prints exactly one word: +# +# sidecar our own hummingbot-tailscale container is running +# native a tailscaled is running on the host itself +# none nothing here owns a tailnet interface +# +# The predicate is deliberately about the RESOURCE, not about which product +# installed it. "Is Condor here?" is the wrong question: the most common real +# conflict is a VPS that already runs Tailscale for the admin's own SSH +# access, and no product check sees that. tailscale0 is the thing actually +# contended, it is path-independent, and on a host where the other component +# lives elsewhere there is simply nothing local to find. +# +# Why it matters: two kernel-mode tailscaled in one network namespace is +# fatal and silent. The second dies with +# tstun.New("tailscale0"): device or resource busy +# while its container stays "running", so restart:unless-stopped never fires +# and `docker ps` shows it healthy. + +# Order matters: a kernel-mode sidecar ALSO creates tailscale0, so checking +# for our own container first is what distinguishes "we did this" from +# "something else did". +tailnet_state() { + if command -v docker >/dev/null 2>&1 && + docker ps --format '{{.Names}}' 2>/dev/null | grep -qx 'hummingbot-tailscale'; then + echo sidecar; return + fi + + # A userspace daemon has no interface at all, so the process check is not + # redundant with the link check -- each catches a case the other misses. + if pgrep -x tailscaled >/dev/null 2>&1; then + echo native; return + fi + if ip link show tailscale0 >/dev/null 2>&1; then + echo native; return + fi + # macOS names the device utunN, not tailscale0; fall back to asking the + # CLI, which is the only portable signal there. + if command -v tailscale >/dev/null 2>&1 && tailscale status >/dev/null 2>&1; then + echo native; return + fi + + echo none +} + +# Resolve the effective mode from .env plus what is actually on the host. +# +# TAILSCALE_MODE is authoritative when set. TAILSCALE_ENABLED remains the +# historical switch and keeps working: enabled-but-unspecified means "put the +# API on the tailnet, you pick how", and how depends on whether this host +# already has a daemon. +# +# Echoes: none | host | sidecar +tailnet_mode() { + local declared="${TAILSCALE_MODE:-}" + local enabled="${TAILSCALE_ENABLED:-false}" + local state; state="$(tailnet_state)" + + case "$declared" in + none|host|sidecar) echo "$declared"; return ;; + "") : ;; + *) echo "[ERROR] TAILSCALE_MODE must be none, host or sidecar (got: $declared)" >&2; return 1 ;; + esac + + case "$enabled" in + [Tt]rue|[Yy]es|1) : ;; + *) echo none; return ;; + esac + + # Enabled, mode unspecified. A host that already runs its own tailscaled + # gets `host`: joining the tailnet a second time from the same machine + # would contend for the same device for no gain. + if [ "$state" = native ]; then echo host; else echo sidecar; fi +} + +# Should the sidecar run in userspace (netstack) mode? +# +# This is the safety net that makes correctness independent of detection being +# right. Netstack creates no TUN device and touches no host routes or +# iptables, so it coexists with a kernel-mode daemon unconditionally -- +# verified on a live tailnet: a userspace node registered alongside a +# kernel-mode one in the same netns and served a loopback-bound port to the +# other node, with a single tailscale0 between them. +# +# The cost is real but small: a userspace node cannot route for the host, only +# accept inbound. Serving port 8000 is exactly that, so nothing is lost here. +tailnet_needs_userspace() { + [ "$(tailnet_state)" = native ] +} From 0a8009fd54a1a0f464faa1dbaae726f9ab8f4030 Mon Sep 17 00:00:00 2001 From: Michael D Date: Fri, 4 Sep 2026 09:28:37 +0800 Subject: [PATCH 03/13] doctor: report against the resolved mode, and check broker accounts Adding TAILSCALE_MODE left doctor.sh stale: host and sidecar produced identical output, and host mode was told to "check tailscale-serve.json is mounted" -- a file it does not have and does not need. - Resolve the mode the same way `make deploy` does, and report against it. tailnet_mode reads the environment, so the values env_get already parsed are handed to it explicitly; doctor deliberately never sources .env, since a password containing shell metacharacters is enough to break that. - Host mode gets its own checks and its own remedy: the host's own tailscaled, and `tailscale serve --bg --tcp=8000 tcp://127.0.0.1:8000`. - A sidecar container that is up while its daemon is dead is now named as what it is -- another tailscaled already owns tailscale0 -- instead of blaming an auth key that was never the problem. That is the signature of the kernel-mode collision: the loser exits with "device or resource busy", containerboot stays up, restart never fires. - TAILSCALE_ENABLED=true with TAILSCALE_MODE=none now warns rather than falling through to the sidecar branch. - New: broker accounts. The bootstrap file missing, replaced by a root-owned directory, or holding a password that no longer matches .env all leave a broker that looks healthy with authentication enabled and zero accounts -- every MQTT connection refused while the API's HTTP health check still passes. Each case now reports the specific fix, including that a mismatch needs emqx-auth-reset. Co-Authored-By: Claude Opus 5 --- doctor.sh | 129 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 105 insertions(+), 24 deletions(-) diff --git a/doctor.sh b/doctor.sh index 7f267278..fcd61276 100755 --- a/doctor.sh +++ b/doctor.sh @@ -17,6 +17,12 @@ set -uo pipefail cd "$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" || exit 1 +# Kept in step with the Makefile's EMQX_AUTH_FILE. +EMQX_BOOTSTRAP=".emqx/auth-bootstrap.csv" +# Defaults so `set -u` holds when there is no .env to read them from. +BROKER_PASSWORD="" +TS_ENABLED="" + # ── Palette ────────────────────────────────────────── if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then @@ -190,6 +196,7 @@ else TS_ENABLED="$(env_get TAILSCALE_ENABLED)" TS_HOSTNAME="$(env_get TAILSCALE_HOSTNAME)" BIND_HOST="$(env_get API_BIND)" + BROKER_PASSWORD="$(env_get BROKER_PASSWORD)" GATEWAY_PASSPHRASE="$(env_get GATEWAY_PASSPHRASE)" # Credentials that used to be shipped as defaults. This API can place @@ -267,6 +274,31 @@ else row fail "$label" "container $cname is not running — the API needs it; \`make deploy\`, or \`docker compose up emqx postgres -d\` for a source run" fi done + + # The broker's accounts come from this file, and EMQX imports it only at + # first start. Three ways it goes wrong, all of which leave a broker that + # is "healthy" with authentication enabled and NO accounts -- every MQTT + # connection refused while the API's HTTP health check still passes: + # + # missing -- deployed with plain `docker compose up` instead of + # `make deploy`, so the emqx-auth prerequisite never ran + # directory -- same, but Docker then created the absent bind-mount + # source as a root-owned directory, which also blocks the + # `make emqx-auth` that would have fixed it + # stale -- .env was rewritten afterwards, so the password the API + # uses no longer matches the one the broker was seeded with + if [ -d "$EMQX_BOOTSTRAP" ]; then + row fail "Broker accounts" "$EMQX_BOOTSTRAP is a DIRECTORY, not a file — Docker created it because the file was missing at deploy time. EMQX imported no accounts and will refuse every connection. Fix: sudo rm -rf $EMQX_BOOTSTRAP && make deploy" + elif [ ! -f "$EMQX_BOOTSTRAP" ]; then + row fail "Broker accounts" "$EMQX_BOOTSTRAP is missing — the broker was started without \`make emqx-auth\`, so it has no accounts and will refuse every connection. Fix: make deploy" + else + csv_pass="$(awk -F, 'NR==2{print $2}' "$EMQX_BOOTSTRAP" 2>/dev/null)" + if [ -n "$BROKER_PASSWORD" ] && [ "$csv_pass" != "$BROKER_PASSWORD" ]; then + row fail "Broker accounts" "$EMQX_BOOTSTRAP does not match BROKER_PASSWORD in .env — the API authenticates with one password against a broker seeded with another. Re-seeding needs the volume dropped: make emqx-auth-reset" + else + row ok "Broker accounts" "bootstrap file present and matches .env" + fi + fi fi echo "" @@ -330,33 +362,82 @@ section "Tailscale" if [ "${TS_ENABLED:-}" != "true" ]; then row ok "Tailscale" "not enabled (TAILSCALE_ENABLED is not true)" else - TS_EXEC="" - if container_running hummingbot-tailscale; then - TS_EXEC="docker exec hummingbot-tailscale tailscale" - row ok "Sidecar" "hummingbot-tailscale container is running" - elif has_cmd tailscale; then - TS_EXEC="tailscale" - row ok "Client" "installed locally" - else - row fail "Tailscale" "TAILSCALE_ENABLED=true but neither the sidecar container nor a local tailscale client is present — \`make deploy\` (Docker) or \`make run\` (source)" - fi - - if [ -n "$TS_EXEC" ]; then - if ts_status="$($TS_EXEC status 2>&1)"; then - row ok "Tailnet" "$(printf '%s' "$ts_status" | grep -v '^[[:space:]]*$' | head -1)" + # Report against the mode actually in force, resolved the same way + # `make deploy` resolves it -- .env records a preference, the live host + # decides. Reporting "check tailscale-serve.json is mounted" at someone + # running host mode, which has no sidecar and no serve JSON, is worse than + # saying nothing. + # shellcheck source=tailnet-state.sh + . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/tailnet-state.sh" + # tailnet_mode reads the ENVIRONMENT; doctor deliberately never sources + # .env (a password containing shell metacharacters is enough to break + # that), so hand it the values env_get already parsed. Without this it + # sees nothing set and reports every install as the default mode. + TS_MODE="$( + export TAILSCALE_MODE="$(env_get TAILSCALE_MODE)" TAILSCALE_ENABLED="$TS_ENABLED" + tailnet_mode + )" || TS_MODE=sidecar + TS_STATE="$(tailnet_state)" + + if [ "$TS_MODE" = none ]; then + row warn "Mode" "TAILSCALE_ENABLED=true but TAILSCALE_MODE=none — nothing will put the API on the tailnet. Set TAILSCALE_MODE to host or sidecar, or TAILSCALE_ENABLED=false" + elif [ "$TS_MODE" = host ]; then + row ok "Mode" "host — served on this machine's existing tailnet node" + if has_cmd tailscale; then + if ts_status="$(tailscale status 2>&1)"; then + row ok "Tailnet" "$(printf '%s' "$ts_status" | grep -v '^[[:space:]]*$' | head -1)" + else + row fail "Tailnet" "this host's tailscaled is not connected — \`sudo tailscale up\`" + fi + if [ -n "$BIND_HOST" ] && is_tailscale_ip "$BIND_HOST"; then + row ok "Serve (port 8000)" "skipped — API_BIND=$BIND_HOST binds the API directly" + elif ts_serve="$(tailscale serve status 2>&1)" && printf '%s' "$ts_serve" | grep -q "8000"; then + row ok "Serve (port 8000)" "forwarded to 127.0.0.1:8000 on this node" + else + row fail "Serve (port 8000)" "port 8000 is not forwarded, so nothing on the tailnet can reach the API. Run: sudo tailscale serve --bg --tcp=8000 tcp://127.0.0.1:8000" + fi else - row fail "Tailnet" "not connected — check TAILSCALE_AUTH_KEY (keys expire) and re-run \`make deploy\`" + row fail "Tailscale" "mode=host but no tailscale client on PATH — install it, or set TAILSCALE_MODE=sidecar to give the API its own node" fi - # Joining the tailnet is not the same as being reachable on it: with - # API_BIND left at its 127.0.0.1 default, TS_SERVE_CONFIG (tailscale - # serve) is what forwards the tailnet's :8000 to the loopback bind -- - # skip this check if API_BIND is set directly to a tailscale IP instead. - if [ -n "$BIND_HOST" ] && is_tailscale_ip "$BIND_HOST"; then - row ok "Serve (port 8000)" "skipped — API_BIND=$BIND_HOST binds the API directly, without relying on tailscale serve" - elif ts_serve="$($TS_EXEC serve status 2>&1)" && printf '%s' "$ts_serve" | grep -q "8000"; then - row ok "Serve (port 8000)" "proxied to the tailnet as http://${TS_HOSTNAME:-hummingbot-api}:8000" + else + row ok "Mode" "sidecar — the API has its own tailnet node" + TS_EXEC="" + if container_running hummingbot-tailscale; then + TS_EXEC="docker exec hummingbot-tailscale tailscale" + row ok "Sidecar" "hummingbot-tailscale container is running" + elif has_cmd tailscale; then + TS_EXEC="tailscale" + row ok "Client" "installed locally" else - row fail "Serve (port 8000)" "the node is on the tailnet but port 8000 is not proxied — with API_BIND=127.0.0.1 nothing can reach the API at all. Check tailscale-serve.json is mounted, then \`make deploy\`" + row fail "Tailscale" "TAILSCALE_ENABLED=true but neither the sidecar container nor a local tailscale client is present — \`make deploy\` (Docker) or \`make run\` (source)" + fi + + if [ -n "$TS_EXEC" ]; then + if ts_status="$($TS_EXEC status 2>&1)"; then + row ok "Tailnet" "$(printf '%s' "$ts_status" | grep -v '^[[:space:]]*$' | head -1)" + elif container_running hummingbot-tailscale && [ "$TS_STATE" = sidecar ] && + { ip link show tailscale0 >/dev/null 2>&1 || pgrep -x tailscaled >/dev/null 2>&1; }; then + # A container that is "running" while its daemon is dead is the + # signature of two kernel-mode tailscaled contending for one + # tailscale0: the loser exits with "device or resource busy", + # containerboot stays up, restart:unless-stopped never fires. + # Blaming the auth key here sent people to reissue a key that + # was never the problem. + row fail "Tailnet" "the sidecar container is up but its daemon is not — another tailscaled already owns this host's tailscale0. Set TAILSCALE_MODE=host, or re-run \`make deploy\`, which starts the sidecar in userspace mode when a native daemon is present" + else + row fail "Tailnet" "not connected — check TAILSCALE_AUTH_KEY (keys expire) and re-run \`make deploy\`" + fi + # Joining the tailnet is not the same as being reachable on it: with + # API_BIND left at its 127.0.0.1 default, TS_SERVE_CONFIG (tailscale + # serve) is what forwards the tailnet's :8000 to the loopback bind -- + # skip this check if API_BIND is set directly to a tailscale IP instead. + if [ -n "$BIND_HOST" ] && is_tailscale_ip "$BIND_HOST"; then + row ok "Serve (port 8000)" "skipped — API_BIND=$BIND_HOST binds the API directly, without relying on tailscale serve" + elif ts_serve="$($TS_EXEC serve status 2>&1)" && printf '%s' "$ts_serve" | grep -q "8000"; then + row ok "Serve (port 8000)" "proxied to the tailnet as http://${TS_HOSTNAME:-hummingbot-api}:8000" + else + row fail "Serve (port 8000)" "the node is on the tailnet but port 8000 is not proxied — with API_BIND=127.0.0.1 nothing can reach the API at all. Check tailscale-serve.json is mounted, then \`make deploy\`" + fi fi fi fi From a3988e4bf0c97c5387ce702f0ace2225b01cdf33 Mon Sep 17 00:00:00 2001 From: Michael D Date: Fri, 4 Sep 2026 11:01:59 +0800 Subject: [PATCH 04/13] Fix review findings: sidecar masked native ownership, reset skipped teardown Both from the PR review, both real. 1. tailnet_state checked the sidecar first and returned early, so a host running BOTH a native tailscaled and our sidecar reported "sidecar" -- which is exactly the co-located Condor install after its first deploy. tailnet_needs_userspace then said no, and a redeploy started the sidecar in kernel mode to contend for a tailscale0 that was already taken: the precise failure this change exists to prevent. The signals are not symmetric, which is why order alone could not fix it. A kernel-mode sidecar runs with network_mode: host, so it creates tailscale0 in the host's namespace and its process is visible from the host -- `ip link` and `pgrep` cannot tell it from a native daemon. The host's tailscaled socket can: a sidecar keeps its own inside the container. So `tailscale status` on the host is checked first and unconditionally, and the other two only when no sidecar is running to be mistaken for. tailnet_needs_userspace now asks tailnet_has_native directly rather than comparing against a composite state that can hide it. Same for the deploy target and doctor's wedged-sidecar check. 2. make reset gated its teardown on finding hummingbot-api or the sidecar by name. After `make run` (source) only the broker and postgres containers exist, so the teardown was skipped while .env and the credential state were still deleted -- leaving the broker's volume, and the accounts seeded from the old BROKER_PASSWORD, to be reused by a setup that generates a new one. That is the .env/broker desync `make emqx-auth-reset` exists to repair. Now unconditional whenever Docker is available; `down` with nothing running is a no-op. Verified: with both a native daemon and a sidecar present, state is native, needs_userspace is true, and a sidecar redeploy is forced to TS_USERSPACE=true. Reset now runs the teardown after a source-only run. Co-Authored-By: Claude Opus 5 --- Makefile | 18 +++++++++++---- doctor.sh | 4 +--- tailnet-state.sh | 60 ++++++++++++++++++++++++++++++++---------------- 3 files changed, 54 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index 391aae78..f149faa4 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,6 @@ deploy: $(SETUP_SENTINEL) emqx-auth @set -a; [ -f .env ] && . ./.env; set +a; \ . ./tailnet-state.sh; \ mode="$$(tailnet_mode)" || exit 1; \ - state="$$(tailnet_state)"; \ case "$$mode" in \ none) \ docker compose up -d ;; \ @@ -64,7 +63,7 @@ deploy: $(SETUP_SENTINEL) emqx-auth echo "[WARN] mode=host but no tailscale CLI on PATH — port 8000 is loopback-only."; \ fi ;; \ sidecar) \ - if [ "$$state" = native ]; then \ + if tailnet_needs_userspace; then \ echo "[INFO] A tailscaled already owns this host's tailnet device."; \ echo "[INFO] Starting the sidecar in userspace mode so both can coexist."; \ export TS_USERSPACE=true; \ @@ -236,11 +235,20 @@ reset: @# container running as an orphan -- still holding the host's tailscale0 @# device -- while .env is deleted, so the next `make setup` has no record @# that it exists. --remove-orphans covers a sidecar left by an older layout. - @if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qxE 'hummingbot-api|$(TAILSCALE_CONTAINER)'; then \ - echo "[INFO] Docker containers found — stopping and wiping volumes..."; \ + @# Unconditional when Docker is available, rather than gated on finding a + @# named container. `make run` (source) starts only emqx and postgres, so a + @# name check for hummingbot-api/the sidecar skipped the teardown entirely + @# while reset still deleted .env and the credential state -- leaving the + @# broker's volume, and the accounts seeded from the OLD BROKER_PASSWORD, + @# to be reused by a setup that generates a new one. That is the same + @# .env/broker desync `make emqx-auth-reset` exists to repair. + @# `down` on a project with nothing running is a no-op, so there is no + @# case worth guarding against. + @if docker info >/dev/null 2>&1; then \ + echo "[INFO] Stopping containers and wiping volumes..."; \ docker compose -f docker-compose.yml -f docker-compose.tailscale.yml down -v --remove-orphans; \ else \ - echo "[INFO] No Docker containers running."; \ + echo "[INFO] Docker is not available — skipping container teardown."; \ fi @if pgrep -f "uvicorn main[:]app" >/dev/null 2>&1; then \ echo "[INFO] Source uvicorn process found — stopping..."; \ diff --git a/doctor.sh b/doctor.sh index fcd61276..1350df5c 100755 --- a/doctor.sh +++ b/doctor.sh @@ -377,7 +377,6 @@ else export TAILSCALE_MODE="$(env_get TAILSCALE_MODE)" TAILSCALE_ENABLED="$TS_ENABLED" tailnet_mode )" || TS_MODE=sidecar - TS_STATE="$(tailnet_state)" if [ "$TS_MODE" = none ]; then row warn "Mode" "TAILSCALE_ENABLED=true but TAILSCALE_MODE=none — nothing will put the API on the tailnet. Set TAILSCALE_MODE to host or sidecar, or TAILSCALE_ENABLED=false" @@ -415,8 +414,7 @@ else if [ -n "$TS_EXEC" ]; then if ts_status="$($TS_EXEC status 2>&1)"; then row ok "Tailnet" "$(printf '%s' "$ts_status" | grep -v '^[[:space:]]*$' | head -1)" - elif container_running hummingbot-tailscale && [ "$TS_STATE" = sidecar ] && - { ip link show tailscale0 >/dev/null 2>&1 || pgrep -x tailscaled >/dev/null 2>&1; }; then + elif container_running hummingbot-tailscale && tailnet_has_native; then # A container that is "running" while its daemon is dead is the # signature of two kernel-mode tailscaled contending for one # tailscale0: the loser exits with "device or resource busy", diff --git a/tailnet-state.sh b/tailnet-state.sh index bbf89068..e9310c82 100644 --- a/tailnet-state.sh +++ b/tailnet-state.sh @@ -21,29 +21,46 @@ # while its container stays "running", so restart:unless-stopped never fires # and `docker ps` shows it healthy. -# Order matters: a kernel-mode sidecar ALSO creates tailscale0, so checking -# for our own container first is what distinguishes "we did this" from -# "something else did". -tailnet_state() { - if command -v docker >/dev/null 2>&1 && - docker ps --format '{{.Names}}' 2>/dev/null | grep -qx 'hummingbot-tailscale'; then - echo sidecar; return - fi +# Is our own sidecar container running? +tailnet_sidecar_running() { + command -v docker >/dev/null 2>&1 && + docker ps --format '{{.Names}}' 2>/dev/null | grep -qx 'hummingbot-tailscale' +} - # A userspace daemon has no interface at all, so the process check is not - # redundant with the link check -- each catches a case the other misses. - if pgrep -x tailscaled >/dev/null 2>&1; then - echo native; return - fi - if ip link show tailscale0 >/dev/null 2>&1; then - echo native; return - fi - # macOS names the device utunN, not tailscale0; fall back to asking the - # CLI, which is the only portable signal there. +# Is a tailscaled running on the HOST ITSELF -- something other than our +# sidecar, and therefore something the sidecar would contend with? +# +# This is the question that actually decides userspace mode, and it cannot be +# answered by "is a sidecar running": both can be true at once, which is the +# normal state on a co-located Condor install after the first deploy. +tailnet_has_native() { + # The host's own daemon answers on the host's socket. A sidecar keeps its + # socket inside its container, so this signal cannot mistake one for the + # other -- which is why it is checked first and unconditionally. if command -v tailscale >/dev/null 2>&1 && tailscale status >/dev/null 2>&1; then - echo native; return + return 0 + fi + # The other two signals CAN be confused by a sidecar: it runs with + # network_mode: host, so a kernel-mode one creates tailscale0 in the host's + # own namespace, and its process is visible from the host on most setups. + # Trust them only when no sidecar is running to be mistaken for. + if ! tailnet_sidecar_running; then + # A userspace daemon has no interface at all, so the process check is + # not redundant with the link check -- each catches what the other + # misses. macOS names the device utunN rather than tailscale0, and is + # covered by the CLI check above. + pgrep -x tailscaled >/dev/null 2>&1 && return 0 + ip link show tailscale0 >/dev/null 2>&1 && return 0 fi + return 1 +} +# Native is reported FIRST when both are present. Ownership of the device is +# the fact that matters, and the sidecar is the thing that has to yield to it; +# reporting "sidecar" there would hide the contention it is meant to reveal. +tailnet_state() { + tailnet_has_native && { echo native; return; } + tailnet_sidecar_running && { echo sidecar; return; } echo none } @@ -89,5 +106,8 @@ tailnet_mode() { # The cost is real but small: a userspace node cannot route for the host, only # accept inbound. Serving port 8000 is exactly that, so nothing is lost here. tailnet_needs_userspace() { - [ "$(tailnet_state)" = native ] + # Deliberately NOT `tailnet_state = native`: when a native daemon and our + # sidecar are both up, that is precisely when the downgrade is required, + # and a composite state that reported "sidecar" there would skip it. + tailnet_has_native } From 6e6e32485a3b5a3cd77ec0c957dc842dbbfdc6b3 Mon Sep 17 00:00:00 2001 From: Michael D Date: Sat, 5 Sep 2026 06:34:09 +0800 Subject: [PATCH 05/13] Accept optional backtesting tuning from the calling installer Condor's main branch writes BACKTESTING_MAX_CONCURRENT=4 and BACKTESTING_TIMEOUT_SECONDS=1800 into this repo's .env: its mm_optimizer_cycle submits a base config plus its variations at once, and a default of one concurrent worker turns that sweep into a queue. Delegating .env authorship here would have dropped those values. They now come through the non-interactive contract as HBAPI_BACKTESTING_MAX_CONCURRENT / _TIMEOUT_SECONDS, so the caller's tuning survives without a second writer touching the file. Emitted only when supplied: a standalone install's .env is unchanged, and this repo's own defaults keep applying to anyone who did not ask. Co-Authored-By: Claude Opus 5 --- setup.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/setup.sh b/setup.sh index 16cc4c6a..9be7c17a 100755 --- a/setup.sh +++ b/setup.sh @@ -479,6 +479,10 @@ HBAPI_NONINTERACTIVE="${HBAPI_NONINTERACTIVE:-0}" _env_ts_enabled="${TAILSCALE_ENABLED:-}" _env_ts_key="${TAILSCALE_AUTH_KEY:-}" _env_ts_host="${TAILSCALE_HOSTNAME:-}" +# Optional caller tuning. Emitted only when supplied, so a standalone install's +# .env is unchanged and this repo's own defaults keep applying. +_env_bt_concurrent="${HBAPI_BACKTESTING_MAX_CONCURRENT:-}" +_env_bt_timeout="${HBAPI_BACKTESTING_TIMEOUT_SECONDS:-}" # Clear screen before prompting user (only if running interactively) if [ "$HBAPI_NONINTERACTIVE" != "1" ] && [[ -t 0 ]] && [[ -c /dev/tty ]]; then @@ -706,6 +710,18 @@ TAILSCALE_AUTH_KEY=$TAILSCALE_AUTH_KEY TAILSCALE_HOSTNAME=$TAILSCALE_HOSTNAME EOF +# Appended rather than templated: these are the caller's tuning, not this +# repo's defaults, and an install that did not ask for them should not carry +# them at all. +if [ -n "$_env_bt_concurrent" ] || [ -n "$_env_bt_timeout" ]; then + { + echo "" + echo "# Backtesting (set by the installer that drove this setup)" + [ -n "$_env_bt_concurrent" ] && echo "BACKTESTING_MAX_CONCURRENT=$_env_bt_concurrent" + [ -n "$_env_bt_timeout" ] && echo "BACKTESTING_TIMEOUT_SECONDS=$_env_bt_timeout" + } >> .env +fi + # Holds the API password, the config/Gateway passphrase and (when set) a # Tailscale auth key. The usual 644 umask makes all of that readable by every # other account on the box. From 540ef57077c58e04f46c122a5d9447b1e564ef7a Mon Sep 17 00:00:00 2001 From: Michael D Date: Sat, 5 Sep 2026 06:47:19 +0800 Subject: [PATCH 06/13] Tell a host tailscaled from the sidecar's by cgroup, not by suppressing pgrep The previous guard skipped the pgrep and ip checks whenever the sidecar was running, so a native daemon went unseen when the host had no tailscale CLI to answer the socket probe -- leaving TS_USERSPACE unset and the sidecar to contend for tailscale0. Simply trusting pgrep instead would break the opposite case. On a native-Linux host the PID namespace is the parent of every container's, so the sidecar's own tailscaled DOES appear in host process lists; under Docker Desktop / WSL2 it does not. Verified both here. Blanket-trusting pgrep would therefore report a native daemon that is not there on exactly the deployment this targets -- a VPS. The cgroup distinguishes them: a containerised tailscaled carries docker/containerd/libpod/kubepods in /proc//cgroup, a host one does not. Reading it needs no docker permissions either, which covers an installing user who cannot run `docker ps` -- previously that made sidecar_running false and the sidecar's daemon read as native. tailscale0 alone stays the weakest signal and is now trusted only when neither a sidecar container nor a containerised tailscaled could have created it. Unreadable /proc resolves toward "native", which is the safe direction: it can only cause an unnecessary userspace downgrade, never a missed one. Co-Authored-By: Claude Opus 5 --- tailnet-state.sh | 56 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/tailnet-state.sh b/tailnet-state.sh index e9310c82..82204d7d 100644 --- a/tailnet-state.sh +++ b/tailnet-state.sh @@ -33,23 +33,53 @@ tailnet_sidecar_running() { # This is the question that actually decides userspace mode, and it cannot be # answered by "is a sidecar running": both can be true at once, which is the # normal state on a co-located Condor install after the first deploy. +# +# A bare `pgrep -x tailscaled` cannot tell a host daemon from the sidecar's +# own: on a native-Linux host the PID namespace is the parent of every +# container's, so container processes DO appear in host process lists (under +# Docker Desktop / WSL2 they do not). The cgroup is the portable +# discriminator, and reading it needs no docker permissions -- which matters +# when the installing user cannot run `docker ps` at all. +_tailnet_tailscaled_pids() { pgrep -x tailscaled 2>/dev/null; } + +_tailnet_in_container() { + [ -r "/proc/$1/cgroup" ] && + grep -qE '(docker|containerd|libpod|kubepods)' "/proc/$1/cgroup" 2>/dev/null +} + +# A tailscaled that is NOT inside a container. No /proc (macOS) means we +# cannot say it is containerised, and there a container's processes are not +# visible here anyway -- so treating it as a host daemon is right. +_tailnet_host_tailscaled() { + local pid + for pid in $(_tailnet_tailscaled_pids); do + _tailnet_in_container "$pid" && continue + return 0 + done + return 1 +} + +_tailnet_container_tailscaled() { + local pid + for pid in $(_tailnet_tailscaled_pids); do + _tailnet_in_container "$pid" && return 0 + done + return 1 +} + tailnet_has_native() { - # The host's own daemon answers on the host's socket. A sidecar keeps its - # socket inside its container, so this signal cannot mistake one for the - # other -- which is why it is checked first and unconditionally. + # 1. The host's own tailscaled socket. Definitive: a sidecar keeps its + # socket inside its container and cannot answer here. if command -v tailscale >/dev/null 2>&1 && tailscale status >/dev/null 2>&1; then return 0 fi - # The other two signals CAN be confused by a sidecar: it runs with - # network_mode: host, so a kernel-mode one creates tailscale0 in the host's - # own namespace, and its process is visible from the host on most setups. - # Trust them only when no sidecar is running to be mistaken for. - if ! tailnet_sidecar_running; then - # A userspace daemon has no interface at all, so the process check is - # not redundant with the link check -- each catches what the other - # misses. macOS names the device utunN rather than tailscale0, and is - # covered by the CLI check above. - pgrep -x tailscaled >/dev/null 2>&1 && return 0 + # 2. A tailscaled process outside any container. Also definitive, and it + # still works when the host has no tailscale CLI installed. + _tailnet_host_tailscaled && return 0 + # 3. tailscale0 on its own is weaker evidence: a kernel-mode sidecar runs + # with network_mode: host and creates that device in the host's own + # namespace. Trust it only when nothing containerised could have. + if ! tailnet_sidecar_running && ! _tailnet_container_tailscaled; then ip link show tailscale0 >/dev/null 2>&1 && return 0 fi return 1 From b2287d8067028a8c1a2f993ebae851a6312dcc0c Mon Sep 17 00:00:00 2001 From: Michael D Date: Sat, 5 Sep 2026 07:17:12 +0800 Subject: [PATCH 07/13] Treat a foreign host-networked Tailscale container as contention The tailscale0 fallback was suppressed whenever any containerised tailscaled existed, on the assumption it could only be our own sidecar. Another host-networked Tailscale container -- not ours -- owns the device just as a host daemon does, and with the host CLI probe unavailable it went unseen: make deploy then started this sidecar with TS_USERSPACE unset, to contend for tailscale0 and wedge. Ownership is now attributed by container id from the cgroup path rather than assumed. Our sidecar's tailscaled is a child of containerboot, so matching the container's own main PID would not have worked. Only OUR sidecar is excused. Anything else -- another container, or an owner that cannot be identified because docker inspect is unavailable -- resolves toward native. The asymmetry is what settles the unknown case: answering native when it was only our own sidecar costs an unnecessary userspace downgrade, and userspace serves port 8000 perfectly well; answering not-native when something else owns the device costs a wedged sidecar and an unreachable API. Co-Authored-By: Claude Opus 5 --- tailnet-state.sh | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/tailnet-state.sh b/tailnet-state.sh index 82204d7d..c064b394 100644 --- a/tailnet-state.sh +++ b/tailnet-state.sh @@ -59,10 +59,20 @@ _tailnet_host_tailscaled() { return 1 } -_tailnet_container_tailscaled() { - local pid +# A containerised tailscaled that is NOT our own sidecar -- someone else's +# host-networked Tailscale container, which contends for tailscale0 exactly +# like a host daemon would. Identified by container id in the cgroup path, +# since the sidecar's tailscaled is a child of containerboot and so does not +# match the container's own main PID. +_tailnet_foreign_container_tailscaled() { + local ours pid + ours="$(docker inspect -f '{{.Id}}' hummingbot-tailscale 2>/dev/null || true)" for pid in $(_tailnet_tailscaled_pids); do - _tailnet_in_container "$pid" && return 0 + _tailnet_in_container "$pid" || continue + if [ -n "$ours" ] && grep -q "$ours" "/proc/$pid/cgroup" 2>/dev/null; then + continue + fi + return 0 done return 1 } @@ -76,11 +86,22 @@ tailnet_has_native() { # 2. A tailscaled process outside any container. Also definitive, and it # still works when the host has no tailscale CLI installed. _tailnet_host_tailscaled && return 0 - # 3. tailscale0 on its own is weaker evidence: a kernel-mode sidecar runs - # with network_mode: host and creates that device in the host's own - # namespace. Trust it only when nothing containerised could have. - if ! tailnet_sidecar_running && ! _tailnet_container_tailscaled; then - ip link show tailscale0 >/dev/null 2>&1 && return 0 + # 3. tailscale0 exists. A kernel-mode sidecar runs with network_mode: + # host and creates that device in the host's own namespace, so OUR + # sidecar is the one owner that is not a conflict. Anything else -- + # another host-networked Tailscale container, or an owner that cannot + # be identified -- contends just as a host daemon would. + # + # The asymmetry decides the unknown case: answering "native" when it is + # only our own sidecar costs an unnecessary userspace downgrade, and + # userspace serves port 8000 perfectly well. Answering "not native" + # when something else owns the device costs a wedged sidecar and an + # unreachable API. So anything unproven resolves toward native. + if ip link show tailscale0 >/dev/null 2>&1; then + if tailnet_sidecar_running && ! _tailnet_foreign_container_tailscaled; then + return 1 + fi + return 0 fi return 1 } From 6cc64ee77a3dd2950f18401a202107b81578a072 Mon Sep 17 00:00:00 2001 From: Michael D Date: Sat, 5 Sep 2026 07:31:07 +0800 Subject: [PATCH 08/13] Make reset all-or-nothing rather than half-completing without Docker The previous guard skipped container teardown when Docker was unavailable or inaccessible, then carried on deleting .env and the credential state. The next setup generates a new BROKER_PASSWORD while the retained EMQX volume still holds the account seeded from the old one, so the broker comes up healthy and rejects the API's correct credentials. Recovering needs `make emqx-auth-reset`, which needs the Docker that was missing in the first place. Not resetting is better than resetting halfway, so reset now fails with the reason and how to proceed. ALLOW_PARTIAL_RESET=1 still removes local files for anyone who genuinely wants that, and says plainly that the volumes are untouched and emqx-auth-reset will be needed. Co-Authored-By: Claude Opus 5 --- Makefile | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f149faa4..e8048602 100644 --- a/Makefile +++ b/Makefile @@ -244,11 +244,28 @@ reset: @# .env/broker desync `make emqx-auth-reset` exists to repair. @# `down` on a project with nothing running is a no-op, so there is no @# case worth guarding against. - @if docker info >/dev/null 2>&1; then \ + @# Reset is all-or-nothing. If the volumes cannot be dropped, deleting .env + @# and the credential state anyway is worse than not resetting at all: the + @# next setup generates a new BROKER_PASSWORD while the retained EMQX volume + @# still holds the account seeded from the old one, so the broker comes up + @# healthy and rejects the API's correct credentials. Recovering needs + @# `make emqx-auth-reset` -- which also needs the Docker that was missing. + @if ! docker info >/dev/null 2>&1; then \ + if [ "$(ALLOW_PARTIAL_RESET)" = "1" ]; then \ + echo "[WARN] Docker unavailable; removing local files only (ALLOW_PARTIAL_RESET=1)."; \ + echo "[WARN] Container volumes are UNTOUCHED. Run 'make emqx-auth-reset' once Docker is back,"; \ + echo "[WARN] or the broker will keep rejecting the credentials the next setup generates."; \ + else \ + echo "[ERROR] Docker is not available, so container volumes cannot be wiped." >&2; \ + echo "[ERROR] Refusing to reset: removing .env while the broker keeps its old" >&2; \ + echo "[ERROR] accounts leaves an install that authenticates against nothing." >&2; \ + echo "[ERROR] Start Docker (or fix its permissions) and re-run 'make reset'." >&2; \ + echo "[ERROR] To remove local files anyway: make reset ALLOW_PARTIAL_RESET=1" >&2; \ + exit 1; \ + fi; \ + else \ echo "[INFO] Stopping containers and wiping volumes..."; \ docker compose -f docker-compose.yml -f docker-compose.tailscale.yml down -v --remove-orphans; \ - else \ - echo "[INFO] Docker is not available — skipping container teardown."; \ fi @if pgrep -f "uvicorn main[:]app" >/dev/null 2>&1; then \ echo "[INFO] Source uvicorn process found — stopping..."; \ From e1aa6da5ce31b4a23736a35210f2b7675948c2fd Mon Sep 17 00:00:00 2001 From: Michael D Date: Sat, 5 Sep 2026 07:48:35 +0800 Subject: [PATCH 09/13] Close the fail-open probes, and validate .env values at the one writer Found by auditing every external command the tailnet decisions depend on for what its ABSENCE implies, rather than fixing the reported instance. Two of eight failed open; the rest already biased safe. - `ip link show tailscale0` missing read as "device free", which is the permissive answer and sends a caller into the collision this is meant to detect. /sys/class/net is present on every Linux kernel and needs no iproute2, so tailnet_device_present now asks the kernel first and keeps ip/ifconfig as fallbacks. No /sys and no tools means this is not Linux, where a network_mode: host container cannot hold the host's device anyway -- so "absent" is the answer, not a guess. - `pgrep` missing read as "no daemon running", skipping the userspace downgrade. /proc//comm answers the same question with no procps. Separately, from reviewing rather than from a report: the Makefile's run/deploy/emqx-auth targets read .env with `. ./.env`, so a value there is executed, not parsed. A password containing $(...) or `;` runs as the deploying user -- and Condor now hands typed credentials to this script, so the path is reachable from an installer prompt. Spaces were already rejected because ".env is read by three different parsers that disagree on quoting". This extends that to the characters where the disagreement is arbitrary code, in both the interactive and non-interactive paths. setup.sh being the only writer of .env is what makes one check sufficient. Rejected: whitespace and ` $ " ' \ ; & | < > ( ) ~ Allowed: A-Z a-z 0-9 ! @ # % ^ * - _ = + [ ] { } : , . ? / Also: doctor reports a rejected TAILSCALE_MODE as the failure it is, rather than defaulting to sidecar and showing a mode `make deploy` will refuse to run. Co-Authored-By: Claude Opus 5 --- doctor.sh | 15 +++++++++++---- setup.sh | 34 ++++++++++++++++++++++++++++++++-- tailnet-state.sh | 36 ++++++++++++++++++++++++++++++++++-- 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/doctor.sh b/doctor.sh index 1350df5c..686b38e5 100755 --- a/doctor.sh +++ b/doctor.sh @@ -373,12 +373,19 @@ else # .env (a password containing shell metacharacters is enough to break # that), so hand it the values env_get already parsed. Without this it # sees nothing set and reports every install as the default mode. - TS_MODE="$( + # A rejected TAILSCALE_MODE must be reported as the error it is. Falling + # back to a default here would show a mode the deploy is going to refuse + # to run, sending someone to debug the wrong thing. + if ! TS_MODE="$( export TAILSCALE_MODE="$(env_get TAILSCALE_MODE)" TAILSCALE_ENABLED="$TS_ENABLED" - tailnet_mode - )" || TS_MODE=sidecar + tailnet_mode 2>/dev/null + )"; then + TS_MODE="" + fi - if [ "$TS_MODE" = none ]; then + if [ -z "$TS_MODE" ]; then + row fail "Mode" "TAILSCALE_MODE=$(env_get TAILSCALE_MODE) is not one of none, host or sidecar — \`make deploy\` will refuse to run until this is corrected in .env" + elif [ "$TS_MODE" = none ]; then row warn "Mode" "TAILSCALE_ENABLED=true but TAILSCALE_MODE=none — nothing will put the API on the tailnet. Set TAILSCALE_MODE to host or sidecar, or TAILSCALE_ENABLED=false" elif [ "$TS_MODE" = host ]; then row ok "Mode" "host — served on this machine's existing tailnet node" diff --git a/setup.sh b/setup.sh index 9be7c17a..bed4f3cd 100755 --- a/setup.sh +++ b/setup.sh @@ -94,8 +94,9 @@ prompt_required_secret_tty() { echo "[WARN] This value cannot be empty" >&2 continue fi - if [[ "$value" =~ [[:space:]] ]]; then - echo "[WARN] Spaces are not supported here -- .env is read by three different parsers that disagree on quoting" >&2 + if env_value_unsafe "$value"; then + echo "[WARN] Not supported here: spaces and \` \$ \" ' \\ ; & | < > ( ) ~" >&2 + echo "[WARN] .env is read by three different parsers, and the Makefile sources it -- these would execute rather than parse." >&2 continue fi confirm="$(prompt_secret_tty "Confirm: ")" @@ -108,6 +109,27 @@ prompt_required_secret_tty() { done } +# Characters that are unsafe in a .env value. +# +# The Makefile's run/deploy/emqx-auth targets read .env with `. ./.env`, so a +# value is not merely parsed there -- it is EXECUTED. A password containing +# $(...) or a `;` runs as the deploying user. Spaces were already rejected +# below because ".env is read by three different parsers that disagree on +# quoting"; this is the same argument, for the characters where the +# disagreement is arbitrary code rather than a truncated value. +# +# Rejected: whitespace and ` $ " ' \ ; & | < > ( ) ~ +# Still allowed: A-Z a-z 0-9 ! @ # % ^ * - _ = + [ ] { } : , . ? / +# +# Validated HERE because setup.sh is the only writer of .env -- one author +# means one place to enforce this, for typed and caller-supplied values alike. +env_value_unsafe() { + case "$1" in + *[[:space:]]*|*'`'*|*'$'*|*'"'*|*"'"*|*'\'*|*';'*|*'&'*|*'|'*|*'<'*|*'>'*|*'('*|*')'*|*'~'*) return 0 ;; + *) return 1 ;; + esac +} + resolve_script_dir() { local src="${BASH_SOURCE[0]}" while [ -h "$src" ]; do @@ -509,6 +531,14 @@ if [ "$HBAPI_NONINTERACTIVE" = "1" ]; then USERNAME="$HBAPI_USERNAME" PASSWORD="$HBAPI_PASSWORD" CONFIG_PASSWORD="$HBAPI_CONFIG_PASSWORD" + for _v in USERNAME PASSWORD CONFIG_PASSWORD; do + eval "_val=\$$_v" + if env_value_unsafe "$_val"; then + echo "[ERROR] $_v contains a character that is unsafe in .env: spaces or \` \$ \" ' \\ ; & | < > ( ) ~" >&2 + echo "[ERROR] The Makefile sources .env, so these would execute rather than parse." >&2 + exit 1 + fi + done # TAILSCALE_ENABLED is the caller-facing switch and keeps its historical # name. TAILSCALE_MODE is accepted as a forward-looking alias so callers can diff --git a/tailnet-state.sh b/tailnet-state.sh index c064b394..6509e099 100644 --- a/tailnet-state.sh +++ b/tailnet-state.sh @@ -40,7 +40,39 @@ tailnet_sidecar_running() { # Docker Desktop / WSL2 they do not). The cgroup is the portable # discriminator, and reading it needs no docker permissions -- which matters # when the installing user cannot run `docker ps` at all. -_tailnet_tailscaled_pids() { pgrep -x tailscaled 2>/dev/null; } +# procps is not guaranteed on a minimal host, and its absence must not read +# as "no daemon running" -- that is the permissive answer, and it would skip +# the userspace downgrade. /proc is always there on Linux and needs no tools. +_tailnet_tailscaled_pids() { + if command -v pgrep >/dev/null 2>&1; then + pgrep -x tailscaled 2>/dev/null + return + fi + local d + for d in /proc/[0-9]*; do + [ -r "$d/comm" ] || continue + if [ "$(cat "$d/comm" 2>/dev/null)" = tailscaled ]; then + printf '%s\n' "${d#/proc/}" + fi + done +} + +# Does the tailnet device exist on this host? +# +# /sys/class/net is present on every Linux kernel and reading it needs no +# iproute2, so a host without `ip` still gets a real answer rather than the +# permissive one -- previously a missing `ip` read as "device free" and sent +# a caller straight into the collision this is meant to detect. +# +# No /sys and no tools means this is not Linux, and a network_mode: host +# container cannot hold the host's device there in the first place, so +# "absent" is the correct answer rather than a guess. +tailnet_device_present() { + [ -e /sys/class/net/tailscale0 ] && return 0 + command -v ip >/dev/null 2>&1 && ip link show tailscale0 >/dev/null 2>&1 && return 0 + command -v ifconfig >/dev/null 2>&1 && ifconfig tailscale0 >/dev/null 2>&1 && return 0 + return 1 +} _tailnet_in_container() { [ -r "/proc/$1/cgroup" ] && @@ -97,7 +129,7 @@ tailnet_has_native() { # userspace serves port 8000 perfectly well. Answering "not native" # when something else owns the device costs a wedged sidecar and an # unreachable API. So anything unproven resolves toward native. - if ip link show tailscale0 >/dev/null 2>&1; then + if tailnet_device_present; then if tailnet_sidecar_running && ! _tailnet_foreign_container_tailscaled; then return 1 fi From f1d95621a96bc66b6777068af74ef1b184ec50e5 Mon Sep 17 00:00:00 2001 From: Michael D Date: Sat, 5 Sep 2026 07:55:38 +0800 Subject: [PATCH 10/13] reset: verify the teardown took, rather than that it ran "Docker is available" was still only a proxy for "the volumes are gone". Compose derives the project name from COMPOSE_PROJECT_NAME, else the directory name, so a stack deployed under a different name -- or before this directory was renamed -- is untouched by `down -v`, which then succeeds as a no-op. Deleting .env after that leaves the broker holding accounts seeded from a password the next setup will never generate, and it surfaces as bots that cannot authenticate against a healthy broker. reset now checks for a surviving emqx-data volume and refuses if it finds one, naming it so it can be removed. Any such volume is treated as ours to be careful about: aborting costs one environment variable, proceeding costs a silent authentication failure that needs emqx-auth-reset to unpick. Co-Authored-By: Claude Opus 5 --- Makefile | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Makefile b/Makefile index e8048602..9775e15d 100644 --- a/Makefile +++ b/Makefile @@ -267,6 +267,29 @@ reset: echo "[INFO] Stopping containers and wiping volumes..."; \ docker compose -f docker-compose.yml -f docker-compose.tailscale.yml down -v --remove-orphans; \ fi + @# Verify the teardown actually took, rather than trusting that it ran. + @# Compose derives the project name from COMPOSE_PROJECT_NAME, else the + @# directory name -- so a stack deployed under a different name, or before + @# this directory was renamed, is untouched by the `down` above, which then + @# succeeds as a no-op. Deleting .env after that leaves the broker holding + @# accounts seeded from a password the next setup will never generate, and + @# it surfaces as bots that cannot authenticate against a healthy broker. + @# + @# Any surviving emqx-data volume is treated as ours to be safe about: + @# aborting costs one environment variable, proceeding costs a silent + @# authentication failure that needs `make emqx-auth-reset` to unpick. + @leftover="$$(docker volume ls -q --filter 'label=com.docker.compose.volume=emqx-data' 2>/dev/null)"; \ + if [ -n "$$leftover" ] && [ "$(ALLOW_PARTIAL_RESET)" != "1" ]; then \ + echo "[ERROR] A broker data volume survived the teardown:" >&2; \ + for v in $$leftover; do echo " $$v" >&2; done; \ + echo "[ERROR] It belongs to a compose project under a different name — this stack was" >&2; \ + echo "[ERROR] deployed with another COMPOSE_PROJECT_NAME, or this directory has been" >&2; \ + echo "[ERROR] renamed since. Removing .env now would leave that broker holding accounts" >&2; \ + echo "[ERROR] the next setup cannot reproduce." >&2; \ + echo "[ERROR] Remove it first: docker volume rm " >&2; \ + echo "[ERROR] Or reset local files anyway: make reset ALLOW_PARTIAL_RESET=1" >&2; \ + exit 1; \ + fi @if pgrep -f "uvicorn main[:]app" >/dev/null 2>&1; then \ echo "[INFO] Source uvicorn process found — stopping..."; \ pkill -f "uvicorn main[:]app" || true; \ From 28eba595892a9749f72430734acdb4ee52d78251 Mon Sep 17 00:00:00 2001 From: Michael D Date: Sat, 5 Sep 2026 08:07:36 +0800 Subject: [PATCH 11/13] reset: an unanswerable volume check is not a clean one `docker volume ls` failing produced an empty result, indistinguishable from "no leftovers", so reset deleted .env on a teardown nobody could verify. Its failure is now treated exactly like a positive result. Same fail-open class as the ip/pgrep probes, in code written after that audit -- the check itself needed the rule applied to it. Verified: a failing volume query now preserves .env and exits non-zero. A failing `docker compose down` already did, since make aborts the target on a non-zero recipe line, so that half of the report needed no change. Co-Authored-By: Claude Opus 5 --- Makefile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9775e15d..1d8e45e1 100644 --- a/Makefile +++ b/Makefile @@ -278,7 +278,14 @@ reset: @# Any surviving emqx-data volume is treated as ours to be safe about: @# aborting costs one environment variable, proceeding costs a silent @# authentication failure that needs `make emqx-auth-reset` to unpick. - @leftover="$$(docker volume ls -q --filter 'label=com.docker.compose.volume=emqx-data' 2>/dev/null)"; \ + @# The query's own failure must not read as "no leftovers": that is the + @# permissive answer, and it would delete .env on a teardown nobody could + @# verify. An unanswerable check is treated exactly like a positive one. + @if leftover="$$(docker volume ls -q --filter 'label=com.docker.compose.volume=emqx-data' 2>/dev/null)"; then \ + :; \ + else \ + leftover=''; \ + fi; \ if [ -n "$$leftover" ] && [ "$(ALLOW_PARTIAL_RESET)" != "1" ]; then \ echo "[ERROR] A broker data volume survived the teardown:" >&2; \ for v in $$leftover; do echo " $$v" >&2; done; \ From dccc73136ea46bbffba89608504350e6b14877b0 Mon Sep 17 00:00:00 2001 From: david-hummingbot Date: Mon, 7 Sep 2026 08:34:50 +0800 Subject: [PATCH 12/13] Report the name the tailnet assigned, not the one .env asked for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tailscale suffixes a hostname that is already taken. On a tailnet that already has a `hummingbot-api` -- a colleague's box, an older install, a machine someone rebuilt -- this node registers as `hummingbot-api-1`, and TAILSCALE_HOSTNAME then records a name belonging to somebody else. doctor printed that requested name as the URL to reach this API, one line under the row showing the real one: ✓ Tailnet 100.67.153.47 hummingbot-api-3 ✓ Serve (port 8000) proxied to the tailnet as http://hummingbot-api:8000 Following it lands on the other machine, which -- being the same software -- answers, so the mistake arrives as "401 Incorrect username or password" and sends people to check a password that was never wrong. `tailnet_node_name` asks the daemon what it actually got. `--self=true --peers=false` prints one line whose second field is the assigned DNS label, so no JSON parser is needed -- which matters in the sidecar image, which has neither python nor jq. It takes the command that reaches the right daemon, so the same helper serves the host and the sidecar. Host mode now names its node too, rather than only confirming that a forward exists. Also: a loopback API_BIND is correct and is *also* the reason a Condor on another machine cannot reach this API, and the symptom -- a connection that times out over there while everything here reports healthy -- does not point at a bind address. Say so on the row someone is already reading. --- doctor.sh | 27 ++++++++++++++++++++++++--- tailnet-state.sh | 28 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/doctor.sh b/doctor.sh index 686b38e5..1ad23d70 100755 --- a/doctor.sh +++ b/doctor.sh @@ -236,7 +236,17 @@ else # is enabled, so unset is always the safe case here -- only an explicit, # wider value is worth a second look. if [ -z "$BIND_HOST" ] || [ "$BIND_HOST" = "127.0.0.1" ]; then - row ok "API_BIND" "127.0.0.1 (default) — port 8000 is loopback-only" + # Loopback is right, and it is also the reason a Condor on ANOTHER + # machine cannot reach this API. Nothing in either installer says so at + # the point that choice is made, and the symptom -- a connection that + # times out from the other box while everything here reports healthy -- + # does not point at a bind address. Say it here, where someone is + # already looking for the answer. + if [ "$TS_ENABLED" = "true" ]; then + row ok "API_BIND" "127.0.0.1 (default) — port 8000 is loopback-only, and reaches the tailnet via tailscale serve" + else + row ok "API_BIND" "127.0.0.1 (default) — port 8000 is loopback-only. Only this machine can reach the API; a Condor running elsewhere needs Tailscale, an SSH tunnel, or an explicit API_BIND here" + fi elif [ "$TS_ENABLED" = "true" ] && is_tailscale_ip "$BIND_HOST"; then row ok "API_BIND" "$BIND_HOST looks like a tailscale IP — the documented way to expose 8000 on the tailnet without the sidecar" elif [ "$TS_ENABLED" = "true" ]; then @@ -398,7 +408,8 @@ else if [ -n "$BIND_HOST" ] && is_tailscale_ip "$BIND_HOST"; then row ok "Serve (port 8000)" "skipped — API_BIND=$BIND_HOST binds the API directly" elif ts_serve="$(tailscale serve status 2>&1)" && printf '%s' "$ts_serve" | grep -q "8000"; then - row ok "Serve (port 8000)" "forwarded to 127.0.0.1:8000 on this node" + _ts_name="$(tailnet_node_name || true)" + row ok "Serve (port 8000)" "forwarded to 127.0.0.1:8000${_ts_name:+ — reachable on the tailnet as http://$_ts_name:8000}" else row fail "Serve (port 8000)" "port 8000 is not forwarded, so nothing on the tailnet can reach the API. Run: sudo tailscale serve --bg --tcp=8000 tcp://127.0.0.1:8000" fi @@ -439,7 +450,17 @@ else if [ -n "$BIND_HOST" ] && is_tailscale_ip "$BIND_HOST"; then row ok "Serve (port 8000)" "skipped — API_BIND=$BIND_HOST binds the API directly, without relying on tailscale serve" elif ts_serve="$($TS_EXEC serve status 2>&1)" && printf '%s' "$ts_serve" | grep -q "8000"; then - row ok "Serve (port 8000)" "proxied to the tailnet as http://${TS_HOSTNAME:-hummingbot-api}:8000" + # The name the node ACTUALLY got, not the one .env asked for. + # Tailscale suffixes a taken hostname, so on a shared tailnet + # that already has a "hummingbot-api" this node is + # "hummingbot-api-1" -- and printing the requested name here + # sends people to a machine that is not this one. + _ts_name="$(tailnet_node_name $TS_EXEC || true)" + if [ -n "$_ts_name" ] && [ "$_ts_name" != "${TS_HOSTNAME:-hummingbot-api}" ]; then + row ok "Serve (port 8000)" "proxied to the tailnet as http://$_ts_name:8000 (TAILSCALE_HOSTNAME asked for '${TS_HOSTNAME:-hummingbot-api}'; that name was already taken on this tailnet, so use the one above)" + else + row ok "Serve (port 8000)" "proxied to the tailnet as http://${_ts_name:-${TS_HOSTNAME:-hummingbot-api}}:8000" + fi else row fail "Serve (port 8000)" "the node is on the tailnet but port 8000 is not proxied — with API_BIND=127.0.0.1 nothing can reach the API at all. Check tailscale-serve.json is mounted, then \`make deploy\`" fi diff --git a/tailnet-state.sh b/tailnet-state.sh index 6509e099..3f136aa3 100644 --- a/tailnet-state.sh +++ b/tailnet-state.sh @@ -194,3 +194,31 @@ tailnet_needs_userspace() { # and a composite state that reported "sidecar" there would skip it. tailnet_has_native } + +# The name this node is ACTUALLY registered under, or empty if it cannot be +# asked. +# +# Tailscale suffixes a hostname that is already taken: ask for +# "hummingbot-api" on a tailnet that has one, and the node comes up as +# "hummingbot-api-1". TAILSCALE_HOSTNAME therefore records what was +# REQUESTED, and is the wrong thing to print in a URL -- on a shared team +# tailnet it names somebody else's machine, which may well answer, so the +# mistake surfaces as an authentication error rather than a name error. +# +# `--self=true --peers=false` prints exactly one line, whose second field is +# the DNS label the control plane assigned. No JSON parser needed, which +# matters in a sidecar image that has neither python nor jq. +# +# Pass the command that reaches the right daemon: +# tailnet_node_name # this host +# tailnet_node_name docker exec hummingbot-tailscale tailscale # sidecar +tailnet_node_name() { + local line + if [ "$#" -eq 0 ]; then + set -- tailscale + fi + line="$("$@" status --self=true --peers=false 2>/dev/null | grep -v '^[[:space:]]*$' | head -1)" || return 1 + [ -n "$line" ] || return 1 + # 100.74.198.21 hummingbot-api-2 michaeld@ linux - + printf '%s' "$line" | awk '{print $2}' +} From 54b36b5005f161cd3dcfb748773d663426d380fb Mon Sep 17 00:00:00 2001 From: david-hummingbot Date: Mon, 7 Sep 2026 08:35:27 +0800 Subject: [PATCH 13/13] setup: survive an unset $USER, and summarise the mode actually chosen Two things that only show up at the ends of the run. `$USER` is set by login shells and is not set by several legitimate ways to run this as root: `docker exec`, cloud-init, a systemd unit, some CI runners. Under `set -euo pipefail` the bare "$USER" in the docker-group messages is then a fatal unbound variable, and the install dies before its first prompt: ./setup.sh: line 265: USER: unbound variable make: *** [Makefile:9: .setup-complete] Error 1 on the root path, where the group question is moot to begin with. `id -un` always answers. (Condor's delegation passes HBAPI_SKIP_DEPS=1 and skips the block entirely, so this only ever bit the standalone install.) The closing summary described a sidecar deployment regardless of mode. In host mode no sidecar starts and no node by that name ever registers, so both lines named things that were not about to happen: Docker deploy: Tailscale sidecar starts automatically with 'make deploy' Condor URL: http://hummingbot-api:8000 Each mode now says what it will do. Host mode can name its node, because the daemon is up by definition; sidecar mode cannot -- the node does not exist yet -- so it prints the requested name with the suffixing rule alongside it, and points at `make doctor` for the settled answer, rather than stating a URL that may belong to another machine. --- setup.sh | 44 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/setup.sh b/setup.sh index bed4f3cd..a220fb68 100755 --- a/setup.sh +++ b/setup.sh @@ -243,6 +243,17 @@ ensure_curl_on_linux() { # -------------------------- # Docker Install / Validation # -------------------------- +# Who is running this, for the docker-group messages below. +# +# $USER is set by login shells, and is NOT set by several ways this script is +# legitimately run as root: `docker exec`, cloud-init, a systemd unit, some CI +# runners. Under `set -euo pipefail` a bare "$USER" is then a fatal unbound +# variable, and the install died before its first prompt -- on the root path, +# where the group question is moot anyway. `id -un` always answers. +current_user() { + printf '%s' "${USER:-$(id -un)}" +} + check_user_in_docker_group() { # Check if current user is already in docker group if [[ "${EUID}" -eq 0 ]]; then @@ -251,7 +262,7 @@ check_user_in_docker_group() { fi if has_cmd getent && getent group docker >/dev/null 2>&1; then - if id -nG "$USER" 2>/dev/null | grep -qw docker; then + if id -nG "$(current_user)" 2>/dev/null | grep -qw docker; then return 0 fi fi @@ -262,14 +273,14 @@ check_user_in_docker_group() { add_user_to_docker_group() { # Only add user to docker group if not already a member if check_user_in_docker_group; then - echo "[OK] User '$USER' is already in the 'docker' group." + echo "[OK] User '$(current_user)' is already in the 'docker' group." return 0 fi if has_cmd getent && getent group docker >/dev/null 2>&1; then if [[ "${EUID}" -ne 0 ]]; then echo "[INFO] Adding current user to 'docker' group (may require re-login)..." - sudo usermod -aG docker "$USER" >/dev/null 2>&1 || true + sudo usermod -aG docker "$(current_user)" >/dev/null 2>&1 || true echo "[OK] User added to docker group. You may need to log out and back in for this to take effect." fi fi @@ -776,9 +787,30 @@ echo " make doctor # Verifies dependencies, .env, containers, port exposure if [ "$TAILSCALE_ENABLED" = true ]; then echo "" echo "Tailscale:" - echo " Docker deploy: Tailscale sidecar starts automatically with 'make deploy'" - echo " Source run: Tailscale installs and connects automatically with 'make run'" - echo " Condor URL: http://$TAILSCALE_HOSTNAME:8000" + # Say what THIS mode will do. In host mode no sidecar starts and no node by + # that name ever registers, so the sidecar summary described a deployment + # that was not about to happen. + if [ "$TAILSCALE_MODE" = host ]; then + echo " Mode: host — port 8000 is served on the tailnet node this machine already has" + _ts_name="$(tailnet_node_name 2>/dev/null || true)" + if [ -n "$_ts_name" ]; then + echo " Condor URL: http://$_ts_name:8000" + else + echo " Condor URL: run 'make doctor' after 'make deploy' — it prints this node's name" + fi + else + echo " Mode: sidecar — the API gets a tailnet node of its own" + echo " Docker deploy: Tailscale sidecar starts automatically with 'make deploy'" + echo " Source run: Tailscale installs and connects automatically with 'make run'" + # Deliberately NOT printed as a settled URL. The node does not exist yet, + # and Tailscale suffixes a hostname that is already taken -- ask for + # "hummingbot-api" on a tailnet that has one and you get + # "hummingbot-api-1". Printing the requested name as the URL is how an + # install ends up pointed at somebody else's machine. + echo " Condor URL: http://$TAILSCALE_HOSTNAME:8000 — unless that name is already taken on" + echo " your tailnet, in which case this node registers as" + echo " $TAILSCALE_HOSTNAME-1, -2, ... 'make doctor' prints the real one." + fi echo " Status: make tailscale-status" fi echo ""