diff --git a/Makefile b/Makefile index f1161a74..1d8e45e1 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ run: emqx-auth sudo tailscale up --authkey="$${TAILSCALE_AUTH_KEY}" --hostname="$${TAILSCALE_HOSTNAME:-hummingbot-api}" --accept-dns=true; \ fi; \ tailscale serve status 2>/dev/null | grep -q ":8000" || \ - sudo tailscale serve --bg http:8000 http://localhost:8000; \ + sudo tailscale serve --bg --tcp=8000 tcp://127.0.0.1:8000; \ echo "[INFO] Binding uvicorn to 127.0.0.1 (tailscale serve exposes port 8000 on tailnet)"; \ conda run --no-capture-output -n hummingbot-api uvicorn main:app --reload --host 127.0.0.1 --port 8000; \ else \ @@ -33,15 +33,43 @@ run: emqx-auth fi # Deploy with Docker -# When TAILSCALE_ENABLED=true: adds the Tailscale sidecar compose override +# +# Tailnet ownership is resolved HERE, not only at setup time. A setup-time +# decision is bypassed by every normal day-2 command: `make reset` deletes +# .env while a sidecar may still be running, and this target is routinely run +# on its own against an .env written weeks earlier. Whatever setup.sh recorded +# is a preference; what the host actually looks like right now is the fact. +# +# none plain compose, no overlay +# host this machine is already on the tailnet -- expose 8000 via +# `tailscale serve` on the existing node instead of joining twice +# sidecar the API gets its own tailnet node, in userspace mode when a +# native daemon is already present deploy: $(SETUP_SENTINEL) emqx-auth @set -a; [ -f .env ] && . ./.env; set +a; \ - if [ "$${TAILSCALE_ENABLED:-false}" = "true" ]; then \ - echo "[INFO] Deploying with Tailscale sidecar..."; \ - docker compose -f docker-compose.yml -f docker-compose.tailscale.yml up -d; \ - else \ - docker compose up -d; \ - fi + . ./tailnet-state.sh; \ + mode="$$(tailnet_mode)" || exit 1; \ + case "$$mode" in \ + none) \ + docker compose up -d ;; \ + host) \ + echo "[INFO] Tailscale: reusing this host's existing node (mode=host)."; \ + docker compose up -d; \ + if command -v tailscale >/dev/null 2>&1; then \ + tailscale serve status 2>/dev/null | grep -q ":8000" || \ + sudo tailscale serve --bg --tcp=8000 tcp://127.0.0.1:8000 || \ + echo "[WARN] Could not configure serve for :8000 — run it yourself: sudo tailscale serve --bg --tcp=8000 tcp://127.0.0.1:8000"; \ + else \ + echo "[WARN] mode=host but no tailscale CLI on PATH — port 8000 is loopback-only."; \ + fi ;; \ + sidecar) \ + if tailnet_needs_userspace; then \ + echo "[INFO] A tailscaled already owns this host's tailnet device."; \ + echo "[INFO] Starting the sidecar in userspace mode so both can coexist."; \ + export TS_USERSPACE=true; \ + fi; \ + docker compose -f docker-compose.yml -f docker-compose.tailscale.yml up -d ;; \ + esac # Verify dependencies, .env, containers, port exposure and API access. # Read-only; exits non-zero when a check actually fails. @@ -202,11 +230,72 @@ build: # - removes all .yml files under bots/credentials/master_account/ reset: @echo "[INFO] Checking for running hummingbot-api services..." - @if docker ps --format '{{.Names}}' 2>/dev/null | grep -qx 'hummingbot-api'; then \ - echo "[INFO] Docker containers running — stopping and wiping volumes..."; \ - docker compose down -v; \ + @# Both compose files, always. The Tailscale sidecar and its state volume are + @# declared ONLY in the overlay, so a base-file-only `down -v` leaves the + @# container running as an orphan -- still holding the host's tailscale0 + @# device -- while .env is deleted, so the next `make setup` has no record + @# that it exists. --remove-orphans covers a sidecar left by an older layout. + @# Unconditional when Docker is available, rather than gated on finding a + @# named container. `make run` (source) starts only emqx and postgres, so a + @# name check for hummingbot-api/the sidecar skipped the teardown entirely + @# while reset still deleted .env and the credential state -- leaving the + @# broker's volume, and the accounts seeded from the OLD BROKER_PASSWORD, + @# to be reused by a setup that generates a new one. That is the same + @# .env/broker desync `make emqx-auth-reset` exists to repair. + @# `down` on a project with nothing running is a no-op, so there is no + @# case worth guarding against. + @# Reset is all-or-nothing. If the volumes cannot be dropped, deleting .env + @# and the credential state anyway is worse than not resetting at all: the + @# next setup generates a new BROKER_PASSWORD while the retained EMQX volume + @# still holds the account seeded from the old one, so the broker comes up + @# healthy and rejects the API's correct credentials. Recovering needs + @# `make emqx-auth-reset` -- which also needs the Docker that was missing. + @if ! docker info >/dev/null 2>&1; then \ + if [ "$(ALLOW_PARTIAL_RESET)" = "1" ]; then \ + echo "[WARN] Docker unavailable; removing local files only (ALLOW_PARTIAL_RESET=1)."; \ + echo "[WARN] Container volumes are UNTOUCHED. Run 'make emqx-auth-reset' once Docker is back,"; \ + echo "[WARN] or the broker will keep rejecting the credentials the next setup generates."; \ + else \ + echo "[ERROR] Docker is not available, so container volumes cannot be wiped." >&2; \ + echo "[ERROR] Refusing to reset: removing .env while the broker keeps its old" >&2; \ + echo "[ERROR] accounts leaves an install that authenticates against nothing." >&2; \ + echo "[ERROR] Start Docker (or fix its permissions) and re-run 'make reset'." >&2; \ + echo "[ERROR] To remove local files anyway: make reset ALLOW_PARTIAL_RESET=1" >&2; \ + exit 1; \ + fi; \ + else \ + echo "[INFO] Stopping containers and wiping volumes..."; \ + docker compose -f docker-compose.yml -f docker-compose.tailscale.yml down -v --remove-orphans; \ + fi + @# Verify the teardown actually took, rather than trusting that it ran. + @# Compose derives the project name from COMPOSE_PROJECT_NAME, else the + @# directory name -- so a stack deployed under a different name, or before + @# this directory was renamed, is untouched by the `down` above, which then + @# succeeds as a no-op. Deleting .env after that leaves the broker holding + @# accounts seeded from a password the next setup will never generate, and + @# it surfaces as bots that cannot authenticate against a healthy broker. + @# + @# Any surviving emqx-data volume is treated as ours to be safe about: + @# aborting costs one environment variable, proceeding costs a silent + @# authentication failure that needs `make emqx-auth-reset` to unpick. + @# The query's own failure must not read as "no leftovers": that is the + @# permissive answer, and it would delete .env on a teardown nobody could + @# verify. An unanswerable check is treated exactly like a positive one. + @if leftover="$$(docker volume ls -q --filter 'label=com.docker.compose.volume=emqx-data' 2>/dev/null)"; then \ + :; \ else \ - echo "[INFO] No Docker containers running."; \ + leftover=''; \ + fi; \ + if [ -n "$$leftover" ] && [ "$(ALLOW_PARTIAL_RESET)" != "1" ]; then \ + echo "[ERROR] A broker data volume survived the teardown:" >&2; \ + for v in $$leftover; do echo " $$v" >&2; done; \ + echo "[ERROR] It belongs to a compose project under a different name — this stack was" >&2; \ + echo "[ERROR] deployed with another COMPOSE_PROJECT_NAME, or this directory has been" >&2; \ + echo "[ERROR] renamed since. Removing .env now would leave that broker holding accounts" >&2; \ + echo "[ERROR] the next setup cannot reproduce." >&2; \ + echo "[ERROR] Remove it first: docker volume rm " >&2; \ + echo "[ERROR] Or reset local files anyway: make reset ALLOW_PARTIAL_RESET=1" >&2; \ + exit 1; \ fi @if pgrep -f "uvicorn main[:]app" >/dev/null 2>&1; then \ echo "[INFO] Source uvicorn process found — stopping..."; \ diff --git a/docker-compose.tailscale.yml b/docker-compose.tailscale.yml index 22e3e8ed..921d99ea 100644 --- a/docker-compose.tailscale.yml +++ b/docker-compose.tailscale.yml @@ -26,7 +26,13 @@ services: environment: - TS_AUTHKEY=${TAILSCALE_AUTH_KEY} - TS_STATE_DIR=/var/lib/tailscale - - TS_USERSPACE=false + # Kernel mode by default -- but `make deploy` sets TS_USERSPACE=true when + # this host already runs its own tailscaled, because two kernel-mode + # daemons in one network namespace cannot both own tailscale0 and the + # loser dies silently inside a container that still reports "running". + # Netstack coexists with a kernel-mode daemon unconditionally and can + # still serve a loopback-bound port, which is all this sidecar does. + - TS_USERSPACE=${TS_USERSPACE:-false} - TS_HOSTNAME=${TAILSCALE_HOSTNAME:-hummingbot-api} - TS_SERVE_CONFIG=/config/tailscale-serve.json volumes: diff --git a/doctor.sh b/doctor.sh index 7f267278..1ad23d70 100755 --- a/doctor.sh +++ b/doctor.sh @@ -17,6 +17,12 @@ set -uo pipefail cd "$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" || exit 1 +# Kept in step with the Makefile's EMQX_AUTH_FILE. +EMQX_BOOTSTRAP=".emqx/auth-bootstrap.csv" +# Defaults so `set -u` holds when there is no .env to read them from. +BROKER_PASSWORD="" +TS_ENABLED="" + # ── Palette ────────────────────────────────────────── if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then @@ -190,6 +196,7 @@ else TS_ENABLED="$(env_get TAILSCALE_ENABLED)" TS_HOSTNAME="$(env_get TAILSCALE_HOSTNAME)" BIND_HOST="$(env_get API_BIND)" + BROKER_PASSWORD="$(env_get BROKER_PASSWORD)" GATEWAY_PASSPHRASE="$(env_get GATEWAY_PASSPHRASE)" # Credentials that used to be shipped as defaults. This API can place @@ -229,7 +236,17 @@ else # is enabled, so unset is always the safe case here -- only an explicit, # wider value is worth a second look. if [ -z "$BIND_HOST" ] || [ "$BIND_HOST" = "127.0.0.1" ]; then - row ok "API_BIND" "127.0.0.1 (default) — port 8000 is loopback-only" + # Loopback is right, and it is also the reason a Condor on ANOTHER + # machine cannot reach this API. Nothing in either installer says so at + # the point that choice is made, and the symptom -- a connection that + # times out from the other box while everything here reports healthy -- + # does not point at a bind address. Say it here, where someone is + # already looking for the answer. + if [ "$TS_ENABLED" = "true" ]; then + row ok "API_BIND" "127.0.0.1 (default) — port 8000 is loopback-only, and reaches the tailnet via tailscale serve" + else + row ok "API_BIND" "127.0.0.1 (default) — port 8000 is loopback-only. Only this machine can reach the API; a Condor running elsewhere needs Tailscale, an SSH tunnel, or an explicit API_BIND here" + fi elif [ "$TS_ENABLED" = "true" ] && is_tailscale_ip "$BIND_HOST"; then row ok "API_BIND" "$BIND_HOST looks like a tailscale IP — the documented way to expose 8000 on the tailnet without the sidecar" elif [ "$TS_ENABLED" = "true" ]; then @@ -267,6 +284,31 @@ else row fail "$label" "container $cname is not running — the API needs it; \`make deploy\`, or \`docker compose up emqx postgres -d\` for a source run" fi done + + # The broker's accounts come from this file, and EMQX imports it only at + # first start. Three ways it goes wrong, all of which leave a broker that + # is "healthy" with authentication enabled and NO accounts -- every MQTT + # connection refused while the API's HTTP health check still passes: + # + # missing -- deployed with plain `docker compose up` instead of + # `make deploy`, so the emqx-auth prerequisite never ran + # directory -- same, but Docker then created the absent bind-mount + # source as a root-owned directory, which also blocks the + # `make emqx-auth` that would have fixed it + # stale -- .env was rewritten afterwards, so the password the API + # uses no longer matches the one the broker was seeded with + if [ -d "$EMQX_BOOTSTRAP" ]; then + row fail "Broker accounts" "$EMQX_BOOTSTRAP is a DIRECTORY, not a file — Docker created it because the file was missing at deploy time. EMQX imported no accounts and will refuse every connection. Fix: sudo rm -rf $EMQX_BOOTSTRAP && make deploy" + elif [ ! -f "$EMQX_BOOTSTRAP" ]; then + row fail "Broker accounts" "$EMQX_BOOTSTRAP is missing — the broker was started without \`make emqx-auth\`, so it has no accounts and will refuse every connection. Fix: make deploy" + else + csv_pass="$(awk -F, 'NR==2{print $2}' "$EMQX_BOOTSTRAP" 2>/dev/null)" + if [ -n "$BROKER_PASSWORD" ] && [ "$csv_pass" != "$BROKER_PASSWORD" ]; then + row fail "Broker accounts" "$EMQX_BOOTSTRAP does not match BROKER_PASSWORD in .env — the API authenticates with one password against a broker seeded with another. Re-seeding needs the volume dropped: make emqx-auth-reset" + else + row ok "Broker accounts" "bootstrap file present and matches .env" + fi + fi fi echo "" @@ -330,33 +372,98 @@ section "Tailscale" if [ "${TS_ENABLED:-}" != "true" ]; then row ok "Tailscale" "not enabled (TAILSCALE_ENABLED is not true)" else - TS_EXEC="" - if container_running hummingbot-tailscale; then - TS_EXEC="docker exec hummingbot-tailscale tailscale" - row ok "Sidecar" "hummingbot-tailscale container is running" - elif has_cmd tailscale; then - TS_EXEC="tailscale" - row ok "Client" "installed locally" - else - row fail "Tailscale" "TAILSCALE_ENABLED=true but neither the sidecar container nor a local tailscale client is present — \`make deploy\` (Docker) or \`make run\` (source)" + # Report against the mode actually in force, resolved the same way + # `make deploy` resolves it -- .env records a preference, the live host + # decides. Reporting "check tailscale-serve.json is mounted" at someone + # running host mode, which has no sidecar and no serve JSON, is worse than + # saying nothing. + # shellcheck source=tailnet-state.sh + . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/tailnet-state.sh" + # tailnet_mode reads the ENVIRONMENT; doctor deliberately never sources + # .env (a password containing shell metacharacters is enough to break + # that), so hand it the values env_get already parsed. Without this it + # sees nothing set and reports every install as the default mode. + # A rejected TAILSCALE_MODE must be reported as the error it is. Falling + # back to a default here would show a mode the deploy is going to refuse + # to run, sending someone to debug the wrong thing. + if ! TS_MODE="$( + export TAILSCALE_MODE="$(env_get TAILSCALE_MODE)" TAILSCALE_ENABLED="$TS_ENABLED" + tailnet_mode 2>/dev/null + )"; then + TS_MODE="" fi - if [ -n "$TS_EXEC" ]; then - if ts_status="$($TS_EXEC status 2>&1)"; then - row ok "Tailnet" "$(printf '%s' "$ts_status" | grep -v '^[[:space:]]*$' | head -1)" + if [ -z "$TS_MODE" ]; then + row fail "Mode" "TAILSCALE_MODE=$(env_get TAILSCALE_MODE) is not one of none, host or sidecar — \`make deploy\` will refuse to run until this is corrected in .env" + elif [ "$TS_MODE" = none ]; then + row warn "Mode" "TAILSCALE_ENABLED=true but TAILSCALE_MODE=none — nothing will put the API on the tailnet. Set TAILSCALE_MODE to host or sidecar, or TAILSCALE_ENABLED=false" + elif [ "$TS_MODE" = host ]; then + row ok "Mode" "host — served on this machine's existing tailnet node" + if has_cmd tailscale; then + if ts_status="$(tailscale status 2>&1)"; then + row ok "Tailnet" "$(printf '%s' "$ts_status" | grep -v '^[[:space:]]*$' | head -1)" + else + row fail "Tailnet" "this host's tailscaled is not connected — \`sudo tailscale up\`" + fi + if [ -n "$BIND_HOST" ] && is_tailscale_ip "$BIND_HOST"; then + row ok "Serve (port 8000)" "skipped — API_BIND=$BIND_HOST binds the API directly" + elif ts_serve="$(tailscale serve status 2>&1)" && printf '%s' "$ts_serve" | grep -q "8000"; then + _ts_name="$(tailnet_node_name || true)" + row ok "Serve (port 8000)" "forwarded to 127.0.0.1:8000${_ts_name:+ — reachable on the tailnet as http://$_ts_name:8000}" + else + row fail "Serve (port 8000)" "port 8000 is not forwarded, so nothing on the tailnet can reach the API. Run: sudo tailscale serve --bg --tcp=8000 tcp://127.0.0.1:8000" + fi else - row fail "Tailnet" "not connected — check TAILSCALE_AUTH_KEY (keys expire) and re-run \`make deploy\`" + row fail "Tailscale" "mode=host but no tailscale client on PATH — install it, or set TAILSCALE_MODE=sidecar to give the API its own node" fi - # Joining the tailnet is not the same as being reachable on it: with - # API_BIND left at its 127.0.0.1 default, TS_SERVE_CONFIG (tailscale - # serve) is what forwards the tailnet's :8000 to the loopback bind -- - # skip this check if API_BIND is set directly to a tailscale IP instead. - if [ -n "$BIND_HOST" ] && is_tailscale_ip "$BIND_HOST"; then - row ok "Serve (port 8000)" "skipped — API_BIND=$BIND_HOST binds the API directly, without relying on tailscale serve" - elif ts_serve="$($TS_EXEC serve status 2>&1)" && printf '%s' "$ts_serve" | grep -q "8000"; then - row ok "Serve (port 8000)" "proxied to the tailnet as http://${TS_HOSTNAME:-hummingbot-api}:8000" + else + row ok "Mode" "sidecar — the API has its own tailnet node" + TS_EXEC="" + if container_running hummingbot-tailscale; then + TS_EXEC="docker exec hummingbot-tailscale tailscale" + row ok "Sidecar" "hummingbot-tailscale container is running" + elif has_cmd tailscale; then + TS_EXEC="tailscale" + row ok "Client" "installed locally" else - row fail "Serve (port 8000)" "the node is on the tailnet but port 8000 is not proxied — with API_BIND=127.0.0.1 nothing can reach the API at all. Check tailscale-serve.json is mounted, then \`make deploy\`" + row fail "Tailscale" "TAILSCALE_ENABLED=true but neither the sidecar container nor a local tailscale client is present — \`make deploy\` (Docker) or \`make run\` (source)" + fi + + if [ -n "$TS_EXEC" ]; then + if ts_status="$($TS_EXEC status 2>&1)"; then + row ok "Tailnet" "$(printf '%s' "$ts_status" | grep -v '^[[:space:]]*$' | head -1)" + elif container_running hummingbot-tailscale && tailnet_has_native; then + # A container that is "running" while its daemon is dead is the + # signature of two kernel-mode tailscaled contending for one + # tailscale0: the loser exits with "device or resource busy", + # containerboot stays up, restart:unless-stopped never fires. + # Blaming the auth key here sent people to reissue a key that + # was never the problem. + row fail "Tailnet" "the sidecar container is up but its daemon is not — another tailscaled already owns this host's tailscale0. Set TAILSCALE_MODE=host, or re-run \`make deploy\`, which starts the sidecar in userspace mode when a native daemon is present" + else + row fail "Tailnet" "not connected — check TAILSCALE_AUTH_KEY (keys expire) and re-run \`make deploy\`" + fi + # Joining the tailnet is not the same as being reachable on it: with + # API_BIND left at its 127.0.0.1 default, TS_SERVE_CONFIG (tailscale + # serve) is what forwards the tailnet's :8000 to the loopback bind -- + # skip this check if API_BIND is set directly to a tailscale IP instead. + if [ -n "$BIND_HOST" ] && is_tailscale_ip "$BIND_HOST"; then + row ok "Serve (port 8000)" "skipped — API_BIND=$BIND_HOST binds the API directly, without relying on tailscale serve" + elif ts_serve="$($TS_EXEC serve status 2>&1)" && printf '%s' "$ts_serve" | grep -q "8000"; then + # The name the node ACTUALLY got, not the one .env asked for. + # Tailscale suffixes a taken hostname, so on a shared tailnet + # that already has a "hummingbot-api" this node is + # "hummingbot-api-1" -- and printing the requested name here + # sends people to a machine that is not this one. + _ts_name="$(tailnet_node_name $TS_EXEC || true)" + if [ -n "$_ts_name" ] && [ "$_ts_name" != "${TS_HOSTNAME:-hummingbot-api}" ]; then + row ok "Serve (port 8000)" "proxied to the tailnet as http://$_ts_name:8000 (TAILSCALE_HOSTNAME asked for '${TS_HOSTNAME:-hummingbot-api}'; that name was already taken on this tailnet, so use the one above)" + else + row ok "Serve (port 8000)" "proxied to the tailnet as http://${_ts_name:-${TS_HOSTNAME:-hummingbot-api}}:8000" + fi + else + row fail "Serve (port 8000)" "the node is on the tailnet but port 8000 is not proxied — with API_BIND=127.0.0.1 nothing can reach the API at all. Check tailscale-serve.json is mounted, then \`make deploy\`" + fi fi fi fi diff --git a/setup.sh b/setup.sh index 62d20277..a220fb68 100755 --- a/setup.sh +++ b/setup.sh @@ -94,8 +94,9 @@ prompt_required_secret_tty() { echo "[WARN] This value cannot be empty" >&2 continue fi - if [[ "$value" =~ [[:space:]] ]]; then - echo "[WARN] Spaces are not supported here -- .env is read by three different parsers that disagree on quoting" >&2 + if env_value_unsafe "$value"; then + echo "[WARN] Not supported here: spaces and \` \$ \" ' \\ ; & | < > ( ) ~" >&2 + echo "[WARN] .env is read by three different parsers, and the Makefile sources it -- these would execute rather than parse." >&2 continue fi confirm="$(prompt_secret_tty "Confirm: ")" @@ -108,6 +109,27 @@ prompt_required_secret_tty() { done } +# Characters that are unsafe in a .env value. +# +# The Makefile's run/deploy/emqx-auth targets read .env with `. ./.env`, so a +# value is not merely parsed there -- it is EXECUTED. A password containing +# $(...) or a `;` runs as the deploying user. Spaces were already rejected +# below because ".env is read by three different parsers that disagree on +# quoting"; this is the same argument, for the characters where the +# disagreement is arbitrary code rather than a truncated value. +# +# Rejected: whitespace and ` $ " ' \ ; & | < > ( ) ~ +# Still allowed: A-Z a-z 0-9 ! @ # % ^ * - _ = + [ ] { } : , . ? / +# +# Validated HERE because setup.sh is the only writer of .env -- one author +# means one place to enforce this, for typed and caller-supplied values alike. +env_value_unsafe() { + case "$1" in + *[[:space:]]*|*'`'*|*'$'*|*'"'*|*"'"*|*'\'*|*';'*|*'&'*|*'|'*|*'<'*|*'>'*|*'('*|*')'*|*'~'*) return 0 ;; + *) return 1 ;; + esac +} + resolve_script_dir() { local src="${BASH_SOURCE[0]}" while [ -h "$src" ]; do @@ -221,6 +243,17 @@ ensure_curl_on_linux() { # -------------------------- # Docker Install / Validation # -------------------------- +# Who is running this, for the docker-group messages below. +# +# $USER is set by login shells, and is NOT set by several ways this script is +# legitimately run as root: `docker exec`, cloud-init, a systemd unit, some CI +# runners. Under `set -euo pipefail` a bare "$USER" is then a fatal unbound +# variable, and the install died before its first prompt -- on the root path, +# where the group question is moot anyway. `id -un` always answers. +current_user() { + printf '%s' "${USER:-$(id -un)}" +} + check_user_in_docker_group() { # Check if current user is already in docker group if [[ "${EUID}" -eq 0 ]]; then @@ -229,7 +262,7 @@ check_user_in_docker_group() { fi if has_cmd getent && getent group docker >/dev/null 2>&1; then - if id -nG "$USER" 2>/dev/null | grep -qw docker; then + if id -nG "$(current_user)" 2>/dev/null | grep -qw docker; then return 0 fi fi @@ -240,14 +273,14 @@ check_user_in_docker_group() { add_user_to_docker_group() { # Only add user to docker group if not already a member if check_user_in_docker_group; then - echo "[OK] User '$USER' is already in the 'docker' group." + echo "[OK] User '$(current_user)' is already in the 'docker' group." return 0 fi if has_cmd getent && getent group docker >/dev/null 2>&1; then if [[ "${EUID}" -ne 0 ]]; then echo "[INFO] Adding current user to 'docker' group (may require re-login)..." - sudo usermod -aG docker "$USER" >/dev/null 2>&1 || true + sudo usermod -aG docker "$(current_user)" >/dev/null 2>&1 || true echo "[OK] User added to docker group. You may need to log out and back in for this to take effect." fi fi @@ -393,12 +426,29 @@ pull_hummingbot_image() { # -------------------------- echo "[INFO] OS=${OS} ARCH=${ARCH}" +# HBAPI_SKIP_DEPS=1 skips the apt/Docker installation pass only -- for a caller +# that has already established both (Condor's installer checks `docker info` +# before it ever gets here). Without it, delegation means every Condor install +# re-runs an apt-get build-dep pass and a Docker install probe that cannot +# change anything, adding minutes to a run that already satisfied them. +# +# Deliberately NOT skipped: the Hummingbot image pull below. Dependency checks +# are the caller's to vouch for; a missing bot image is a functional gap no +# caller currently covers. +if [ "${HBAPI_SKIP_DEPS:-0}" = "1" ]; then + echo "[INFO] Skipping dependency install (HBAPI_SKIP_DEPS=1) — caller vouches for docker + build deps." + DOCKER_ALREADY_PRESENT=true + COMPOSE_ALREADY_PRESENT=true +else + if is_linux; then install_linux_build_deps fi ensure_docker_and_compose +fi + # Show summary of what was done echo "" if [ "$DOCKER_ALREADY_PRESENT" = true ] && [ "$COMPOSE_ALREADY_PRESENT" = true ]; then @@ -437,15 +487,106 @@ if [ -f ".env" ]; then exit 0 fi +# -------------------------- +# Non-interactive mode +# -------------------------- +# Lets another installer (Condor's setup-environment.sh, CI, a provisioning +# script) drive this setup without a human. It exists because the alternative +# -- a caller hand-writing .env itself -- is how the schema drifted before: +# a second author shipped BROKER_PASSWORD=password and omitted +# BROKER_DASHBOARD_PASSWORD entirely, leaving the broker with credentials that +# did not match its own bootstrap file. +# +# Only the values a human would TYPE are accepted here. Everything this script +# GENERATES -- both broker passwords above all -- stays generated in both +# modes, so a caller cannot supply a weak one. That is the whole point: .env +# has exactly one author regardless of who started the run. +# +# The prompts below read from /dev/tty, so they cannot be fed by a pipe; this +# is the supported way to answer them programmatically. +HBAPI_NONINTERACTIVE="${HBAPI_NONINTERACTIVE:-0}" + +# Captured before the defaults below reset them -- these three are the only +# Tailscale settings a caller may supply, and the reset would otherwise +# silently discard them. +_env_ts_enabled="${TAILSCALE_ENABLED:-}" +_env_ts_key="${TAILSCALE_AUTH_KEY:-}" +_env_ts_host="${TAILSCALE_HOSTNAME:-}" +# Optional caller tuning. Emitted only when supplied, so a standalone install's +# .env is unchanged and this repo's own defaults keep applying. +_env_bt_concurrent="${HBAPI_BACKTESTING_MAX_CONCURRENT:-}" +_env_bt_timeout="${HBAPI_BACKTESTING_TIMEOUT_SECONDS:-}" + # Clear screen before prompting user (only if running interactively) -if [[ -t 0 ]] && [[ -c /dev/tty ]]; then - if has_cmd clear; then - clear - else - printf "\033c" - fi +if [ "$HBAPI_NONINTERACTIVE" != "1" ] && [[ -t 0 ]] && [[ -c /dev/tty ]]; then + # `clear` exits non-zero when TERM is unset (a tty without a termcap entry -- + # CI runners, `script` wrappers, some IDE terminals). Under `set -e` that + # aborted the whole setup before a single prompt, so fall back to the escape + # sequence rather than trusting the exit status. + clear 2>/dev/null || printf "\033c" fi +TAILSCALE_ENABLED=false +TAILSCALE_AUTH_KEY="" +TAILSCALE_HOSTNAME="hummingbot-api" + +if [ "$HBAPI_NONINTERACTIVE" = "1" ]; then + echo "Hummingbot API Setup (non-interactive)" + echo "" + # Fail loudly rather than writing a half-configured .env: these three guard + # an API that can place orders and read balances, and there is no safe + # default for any of them. + : "${HBAPI_USERNAME:?HBAPI_NONINTERACTIVE=1 requires HBAPI_USERNAME}" + : "${HBAPI_PASSWORD:?HBAPI_NONINTERACTIVE=1 requires HBAPI_PASSWORD}" + : "${HBAPI_CONFIG_PASSWORD:?HBAPI_NONINTERACTIVE=1 requires HBAPI_CONFIG_PASSWORD}" + USERNAME="$HBAPI_USERNAME" + PASSWORD="$HBAPI_PASSWORD" + CONFIG_PASSWORD="$HBAPI_CONFIG_PASSWORD" + for _v in USERNAME PASSWORD CONFIG_PASSWORD; do + eval "_val=\$$_v" + if env_value_unsafe "$_val"; then + echo "[ERROR] $_v contains a character that is unsafe in .env: spaces or \` \$ \" ' \\ ; & | < > ( ) ~" >&2 + echo "[ERROR] The Makefile sources .env, so these would execute rather than parse." >&2 + exit 1 + fi + done + + # TAILSCALE_ENABLED is the caller-facing switch and keeps its historical + # name. TAILSCALE_MODE is accepted as a forward-looking alias so callers can + # already say what they mean (none|host|sidecar); only "none" differs from + # enabled=true today, the host/sidecar split lands with the ownership work. + case "${TAILSCALE_MODE:-}" in + none) TAILSCALE_ENABLED=false ;; + host|sidecar) TAILSCALE_ENABLED=true ;; + "") case "$_env_ts_enabled" in + [Tt]rue|[Yy]es|1) TAILSCALE_ENABLED=true ;; + *) TAILSCALE_ENABLED=false ;; + esac ;; + *) echo "[ERROR] TAILSCALE_MODE must be none, host or sidecar (got: $TAILSCALE_MODE)" >&2; exit 1 ;; + esac + + if [ "$TAILSCALE_ENABLED" = true ]; then + TAILSCALE_AUTH_KEY="$_env_ts_key" + TAILSCALE_HOSTNAME="${_env_ts_host:-hummingbot-api}" + # An auth key registers a NEW node, so only the sidecar needs one. In host + # mode we serve on a node this machine already has, and demanding a + # credential for that would fail installs that are entirely correct -- + # which is exactly the co-located Condor case. + if [ "${TAILSCALE_MODE:-}" != "host" ]; then + if [ -z "$TAILSCALE_AUTH_KEY" ]; then + echo "[ERROR] Tailscale requested but TAILSCALE_AUTH_KEY is empty" >&2 + echo " (not needed when TAILSCALE_MODE=host — this host would reuse its own node)" >&2 + exit 1 + fi + if [[ ! "$TAILSCALE_AUTH_KEY" =~ ^tskey-auth- ]]; then + echo "[ERROR] TAILSCALE_AUTH_KEY must start with 'tskey-auth-'" >&2 + exit 1 + fi + fi + fi + echo "[INFO] Credentials taken from the environment; broker passwords generated locally." +else + echo "Hummingbot API Setup" echo "" echo "Set API credentials (use a strong username, password, and config password):" @@ -458,34 +599,83 @@ CONFIG_PASSWORD="$(prompt_required_secret_tty "Config password: ")" # -------------------------- # Tailscale Configuration # -------------------------- -TAILSCALE_ENABLED=false -TAILSCALE_AUTH_KEY="" -TAILSCALE_HOSTNAME="hummingbot-api" if prompt_yes_no "Use Tailscale for secure private networking? [y/N]: " "n"; then - echo "" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo " How to get a Tailscale auth key:" - echo " 1. Create a free account at https://tailscale.com" - echo " 2. Go to: https://tailscale.com/admin/settings/keys" - echo " 3. Click 'Generate auth key'" - echo " 4. Check 'Reusable' for multiple server deployments" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "" - while true; do - TAILSCALE_AUTH_KEY="$(prompt_tty "Tailscale auth key (tskey-auth-...): " "")" - if [[ -z "$TAILSCALE_AUTH_KEY" ]]; then - echo "[WARN] Auth key cannot be empty" - continue - fi - if [[ ! "$TAILSCALE_AUTH_KEY" =~ ^tskey-auth- ]]; then - echo "[WARN] Auth key must start with 'tskey-auth-'" - continue - fi - break - done - # Hostname defaults to "hummingbot-api" — override via TAILSCALE_HOSTNAME in .env if needed TAILSCALE_ENABLED=true + # Ask what the answer depends on before asking for an auth key. A machine + # that already runs tailscaled needs no key -- it is on the tailnet, and + # joining a second time from the same host would only contend for the same + # device. Prompting anyway would demand a credential to do something we are + # about to decide not to do. + # shellcheck source=tailnet-state.sh + . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/tailnet-state.sh" + if [ "$(tailnet_state)" != none ]; then + TAILSCALE_MODE=host + echo "" + echo "[OK] This machine is already on a tailnet — no auth key needed." + echo " Port 8000 will be served on the node it already has." + echo " Want the API to have its own tailnet identity instead?" + echo " Set TAILSCALE_MODE=sidecar and TAILSCALE_AUTH_KEY in .env, then 'make deploy'." + else + TAILSCALE_MODE=sidecar + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " How to get a Tailscale auth key:" + echo " 1. Create a free account at https://tailscale.com" + echo " 2. Go to: https://tailscale.com/admin/settings/keys" + echo " 3. Click 'Generate auth key'" + echo " 4. Check 'Reusable' for multiple server deployments" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "" + while true; do + TAILSCALE_AUTH_KEY="$(prompt_tty "Tailscale auth key (tskey-auth-...): " "")" + if [[ -z "$TAILSCALE_AUTH_KEY" ]]; then + echo "[WARN] Auth key cannot be empty" + continue + fi + if [[ ! "$TAILSCALE_AUTH_KEY" =~ ^tskey-auth- ]]; then + echo "[WARN] Auth key must start with 'tskey-auth-'" + continue + fi + break + done + # Hostname defaults to "hummingbot-api" — override via TAILSCALE_HOSTNAME in .env if needed + fi +fi + +fi # end interactive / non-interactive split + +# -------------------------- +# Tailnet ownership +# -------------------------- +# Recorded in .env as a preference; `make deploy` re-resolves it against the +# live host on every run, so this is a starting point, not a guarantee. +_setup_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tailnet-state.sh +. "$_setup_here/tailnet-state.sh" + +if [ "$TAILSCALE_ENABLED" != true ]; then + TAILSCALE_MODE=none +elif [ -n "${TAILSCALE_MODE:-}" ]; then + : # caller was explicit (validated earlier); respect it +else + case "$(tailnet_state)" in + native) + TAILSCALE_MODE=host + echo "" + echo "[INFO] This machine is already on a tailnet." + echo "[INFO] Port 8000 will be served on the node it already has, rather than" + echo "[INFO] joining a second time — two daemons cannot share one tailnet device." + echo "[INFO] Want a separate identity anyway? Set TAILSCALE_MODE=sidecar in .env." + ;; + sidecar) + TAILSCALE_MODE=sidecar + echo "[INFO] Reusing the existing hummingbot-tailscale sidecar." + ;; + *) + TAILSCALE_MODE=sidecar + ;; + esac fi # Broker credentials are never typed by the user — the API and the bots are the only clients — @@ -545,11 +735,34 @@ GATEWAY_PASSPHRASE=$CONFIG_PASSWORD BOTS_PATH=$(pwd) # Tailscale +# TAILSCALE_ENABLED is the switch and keeps its historical meaning. +# TAILSCALE_MODE says HOW: host = share this machine's existing tailnet node, +# sidecar = give the API a node of its own. Deploy re-resolves this against +# the live host every time, so an .env that was right in June cannot put a +# second kernel-mode daemon on a box that grew one in July. +# +# NOTE: this heredoc is unquoted so plain variables expand. Command +# substitution in these comments would EXECUTE while .env is written -- a +# backtick here once ran a make target mid-setup and seeded the broker with +# the wrong password. Keep substitution out of the comments. TAILSCALE_ENABLED=$TAILSCALE_ENABLED +TAILSCALE_MODE=$TAILSCALE_MODE TAILSCALE_AUTH_KEY=$TAILSCALE_AUTH_KEY TAILSCALE_HOSTNAME=$TAILSCALE_HOSTNAME EOF +# Appended rather than templated: these are the caller's tuning, not this +# repo's defaults, and an install that did not ask for them should not carry +# them at all. +if [ -n "$_env_bt_concurrent" ] || [ -n "$_env_bt_timeout" ]; then + { + echo "" + echo "# Backtesting (set by the installer that drove this setup)" + [ -n "$_env_bt_concurrent" ] && echo "BACKTESTING_MAX_CONCURRENT=$_env_bt_concurrent" + [ -n "$_env_bt_timeout" ] && echo "BACKTESTING_TIMEOUT_SECONDS=$_env_bt_timeout" + } >> .env +fi + # Holds the API password, the config/Gateway passphrase and (when set) a # Tailscale auth key. The usual 644 umask makes all of that readable by every # other account on the box. @@ -574,9 +787,30 @@ echo " make doctor # Verifies dependencies, .env, containers, port exposure if [ "$TAILSCALE_ENABLED" = true ]; then echo "" echo "Tailscale:" - echo " Docker deploy: Tailscale sidecar starts automatically with 'make deploy'" - echo " Source run: Tailscale installs and connects automatically with 'make run'" - echo " Condor URL: http://$TAILSCALE_HOSTNAME:8000" + # Say what THIS mode will do. In host mode no sidecar starts and no node by + # that name ever registers, so the sidecar summary described a deployment + # that was not about to happen. + if [ "$TAILSCALE_MODE" = host ]; then + echo " Mode: host — port 8000 is served on the tailnet node this machine already has" + _ts_name="$(tailnet_node_name 2>/dev/null || true)" + if [ -n "$_ts_name" ]; then + echo " Condor URL: http://$_ts_name:8000" + else + echo " Condor URL: run 'make doctor' after 'make deploy' — it prints this node's name" + fi + else + echo " Mode: sidecar — the API gets a tailnet node of its own" + echo " Docker deploy: Tailscale sidecar starts automatically with 'make deploy'" + echo " Source run: Tailscale installs and connects automatically with 'make run'" + # Deliberately NOT printed as a settled URL. The node does not exist yet, + # and Tailscale suffixes a hostname that is already taken -- ask for + # "hummingbot-api" on a tailnet that has one and you get + # "hummingbot-api-1". Printing the requested name as the URL is how an + # install ends up pointed at somebody else's machine. + echo " Condor URL: http://$TAILSCALE_HOSTNAME:8000 — unless that name is already taken on" + echo " your tailnet, in which case this node registers as" + echo " $TAILSCALE_HOSTNAME-1, -2, ... 'make doctor' prints the real one." + fi echo " Status: make tailscale-status" fi echo "" diff --git a/tailnet-state.sh b/tailnet-state.sh new file mode 100644 index 00000000..3f136aa3 --- /dev/null +++ b/tailnet-state.sh @@ -0,0 +1,224 @@ +#!/bin/bash +# Who, if anyone, already owns this host's tailnet interface? +# +# Sourced by setup.sh (to pick a default mode) and by the Makefile (to enforce +# one at deploy time). Prints exactly one word: +# +# sidecar our own hummingbot-tailscale container is running +# native a tailscaled is running on the host itself +# none nothing here owns a tailnet interface +# +# The predicate is deliberately about the RESOURCE, not about which product +# installed it. "Is Condor here?" is the wrong question: the most common real +# conflict is a VPS that already runs Tailscale for the admin's own SSH +# access, and no product check sees that. tailscale0 is the thing actually +# contended, it is path-independent, and on a host where the other component +# lives elsewhere there is simply nothing local to find. +# +# Why it matters: two kernel-mode tailscaled in one network namespace is +# fatal and silent. The second dies with +# tstun.New("tailscale0"): device or resource busy +# while its container stays "running", so restart:unless-stopped never fires +# and `docker ps` shows it healthy. + +# Is our own sidecar container running? +tailnet_sidecar_running() { + command -v docker >/dev/null 2>&1 && + docker ps --format '{{.Names}}' 2>/dev/null | grep -qx 'hummingbot-tailscale' +} + +# Is a tailscaled running on the HOST ITSELF -- something other than our +# sidecar, and therefore something the sidecar would contend with? +# +# This is the question that actually decides userspace mode, and it cannot be +# answered by "is a sidecar running": both can be true at once, which is the +# normal state on a co-located Condor install after the first deploy. +# +# A bare `pgrep -x tailscaled` cannot tell a host daemon from the sidecar's +# own: on a native-Linux host the PID namespace is the parent of every +# container's, so container processes DO appear in host process lists (under +# Docker Desktop / WSL2 they do not). The cgroup is the portable +# discriminator, and reading it needs no docker permissions -- which matters +# when the installing user cannot run `docker ps` at all. +# procps is not guaranteed on a minimal host, and its absence must not read +# as "no daemon running" -- that is the permissive answer, and it would skip +# the userspace downgrade. /proc is always there on Linux and needs no tools. +_tailnet_tailscaled_pids() { + if command -v pgrep >/dev/null 2>&1; then + pgrep -x tailscaled 2>/dev/null + return + fi + local d + for d in /proc/[0-9]*; do + [ -r "$d/comm" ] || continue + if [ "$(cat "$d/comm" 2>/dev/null)" = tailscaled ]; then + printf '%s\n' "${d#/proc/}" + fi + done +} + +# Does the tailnet device exist on this host? +# +# /sys/class/net is present on every Linux kernel and reading it needs no +# iproute2, so a host without `ip` still gets a real answer rather than the +# permissive one -- previously a missing `ip` read as "device free" and sent +# a caller straight into the collision this is meant to detect. +# +# No /sys and no tools means this is not Linux, and a network_mode: host +# container cannot hold the host's device there in the first place, so +# "absent" is the correct answer rather than a guess. +tailnet_device_present() { + [ -e /sys/class/net/tailscale0 ] && return 0 + command -v ip >/dev/null 2>&1 && ip link show tailscale0 >/dev/null 2>&1 && return 0 + command -v ifconfig >/dev/null 2>&1 && ifconfig tailscale0 >/dev/null 2>&1 && return 0 + return 1 +} + +_tailnet_in_container() { + [ -r "/proc/$1/cgroup" ] && + grep -qE '(docker|containerd|libpod|kubepods)' "/proc/$1/cgroup" 2>/dev/null +} + +# A tailscaled that is NOT inside a container. No /proc (macOS) means we +# cannot say it is containerised, and there a container's processes are not +# visible here anyway -- so treating it as a host daemon is right. +_tailnet_host_tailscaled() { + local pid + for pid in $(_tailnet_tailscaled_pids); do + _tailnet_in_container "$pid" && continue + return 0 + done + return 1 +} + +# A containerised tailscaled that is NOT our own sidecar -- someone else's +# host-networked Tailscale container, which contends for tailscale0 exactly +# like a host daemon would. Identified by container id in the cgroup path, +# since the sidecar's tailscaled is a child of containerboot and so does not +# match the container's own main PID. +_tailnet_foreign_container_tailscaled() { + local ours pid + ours="$(docker inspect -f '{{.Id}}' hummingbot-tailscale 2>/dev/null || true)" + for pid in $(_tailnet_tailscaled_pids); do + _tailnet_in_container "$pid" || continue + if [ -n "$ours" ] && grep -q "$ours" "/proc/$pid/cgroup" 2>/dev/null; then + continue + fi + return 0 + done + return 1 +} + +tailnet_has_native() { + # 1. The host's own tailscaled socket. Definitive: a sidecar keeps its + # socket inside its container and cannot answer here. + if command -v tailscale >/dev/null 2>&1 && tailscale status >/dev/null 2>&1; then + return 0 + fi + # 2. A tailscaled process outside any container. Also definitive, and it + # still works when the host has no tailscale CLI installed. + _tailnet_host_tailscaled && return 0 + # 3. tailscale0 exists. A kernel-mode sidecar runs with network_mode: + # host and creates that device in the host's own namespace, so OUR + # sidecar is the one owner that is not a conflict. Anything else -- + # another host-networked Tailscale container, or an owner that cannot + # be identified -- contends just as a host daemon would. + # + # The asymmetry decides the unknown case: answering "native" when it is + # only our own sidecar costs an unnecessary userspace downgrade, and + # userspace serves port 8000 perfectly well. Answering "not native" + # when something else owns the device costs a wedged sidecar and an + # unreachable API. So anything unproven resolves toward native. + if tailnet_device_present; then + if tailnet_sidecar_running && ! _tailnet_foreign_container_tailscaled; then + return 1 + fi + return 0 + fi + return 1 +} + +# Native is reported FIRST when both are present. Ownership of the device is +# the fact that matters, and the sidecar is the thing that has to yield to it; +# reporting "sidecar" there would hide the contention it is meant to reveal. +tailnet_state() { + tailnet_has_native && { echo native; return; } + tailnet_sidecar_running && { echo sidecar; return; } + echo none +} + +# Resolve the effective mode from .env plus what is actually on the host. +# +# TAILSCALE_MODE is authoritative when set. TAILSCALE_ENABLED remains the +# historical switch and keeps working: enabled-but-unspecified means "put the +# API on the tailnet, you pick how", and how depends on whether this host +# already has a daemon. +# +# Echoes: none | host | sidecar +tailnet_mode() { + local declared="${TAILSCALE_MODE:-}" + local enabled="${TAILSCALE_ENABLED:-false}" + local state; state="$(tailnet_state)" + + case "$declared" in + none|host|sidecar) echo "$declared"; return ;; + "") : ;; + *) echo "[ERROR] TAILSCALE_MODE must be none, host or sidecar (got: $declared)" >&2; return 1 ;; + esac + + case "$enabled" in + [Tt]rue|[Yy]es|1) : ;; + *) echo none; return ;; + esac + + # Enabled, mode unspecified. A host that already runs its own tailscaled + # gets `host`: joining the tailnet a second time from the same machine + # would contend for the same device for no gain. + if [ "$state" = native ]; then echo host; else echo sidecar; fi +} + +# Should the sidecar run in userspace (netstack) mode? +# +# This is the safety net that makes correctness independent of detection being +# right. Netstack creates no TUN device and touches no host routes or +# iptables, so it coexists with a kernel-mode daemon unconditionally -- +# verified on a live tailnet: a userspace node registered alongside a +# kernel-mode one in the same netns and served a loopback-bound port to the +# other node, with a single tailscale0 between them. +# +# The cost is real but small: a userspace node cannot route for the host, only +# accept inbound. Serving port 8000 is exactly that, so nothing is lost here. +tailnet_needs_userspace() { + # Deliberately NOT `tailnet_state = native`: when a native daemon and our + # sidecar are both up, that is precisely when the downgrade is required, + # and a composite state that reported "sidecar" there would skip it. + tailnet_has_native +} + +# The name this node is ACTUALLY registered under, or empty if it cannot be +# asked. +# +# Tailscale suffixes a hostname that is already taken: ask for +# "hummingbot-api" on a tailnet that has one, and the node comes up as +# "hummingbot-api-1". TAILSCALE_HOSTNAME therefore records what was +# REQUESTED, and is the wrong thing to print in a URL -- on a shared team +# tailnet it names somebody else's machine, which may well answer, so the +# mistake surfaces as an authentication error rather than a name error. +# +# `--self=true --peers=false` prints exactly one line, whose second field is +# the DNS label the control plane assigned. No JSON parser needed, which +# matters in a sidecar image that has neither python nor jq. +# +# Pass the command that reaches the right daemon: +# tailnet_node_name # this host +# tailnet_node_name docker exec hummingbot-tailscale tailscale # sidecar +tailnet_node_name() { + local line + if [ "$#" -eq 0 ]; then + set -- tailscale + fi + line="$("$@" status --self=true --peers=false 2>/dev/null | grep -v '^[[:space:]]*$' | head -1)" || return 1 + [ -n "$line" ] || return 1 + # 100.74.198.21 hummingbot-api-2 michaeld@ linux - + printf '%s' "$line" | awk '{print $2}' +}