Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 108 additions & 11 deletions condor/doctor.py
Original file line number Diff line number Diff line change
Expand Up @@ -351,8 +351,13 @@ def _listening_process(port: int) -> str:
return ""


def _bind_host(addr: str) -> str:
"""The host half of a ``host:port`` bind, with IPv6 brackets left intact."""
return addr.rsplit(":", 1)[0] if re.search(r":\d+$", addr) else addr


def _is_public_bind(addr: str) -> bool:
host = addr.rsplit(":", 1)[0] if re.search(r":\d+$", addr) else addr
host = _bind_host(addr)
return host in ("0.0.0.0", "*", "::", "[::]", "")


Expand All @@ -377,31 +382,80 @@ def check_dashboard_port() -> list[Check]:
held_by = f" (held by {owner})" if owner else ""

public = [b for b in binds if _is_public_bind(b)]
if USE_TAILSCALE:

# Local mode stays on loopback whether or not Tailscale is on: the
# dashboard has no login, and a tailnet is a smaller audience than the
# internet, not an authenticated one. So the Tailscale expectations below
# do not apply here -- a loopback bind is the correct end state, not a
# fallback that failed.
from utils.config import LOCAL_MODE

if USE_TAILSCALE and not LOCAL_MODE:
if public:
return [
Check(
"Dashboard port",
FAIL,
f"USE_TAILSCALE=true but {WEB_PORT} is bound to "
f"{public[0]} (all interfaces){held_by} — should be "
"127.0.0.1-only",
f"{public[0]} (all interfaces){held_by} — should be this "
"node's tailnet address only",
)
]

# A tailnet bind is the goal: the port is on the tailnet and on no
# public interface. A loopback bind is NOT the same thing and must not
# report OK -- that is the fail-closed fallback, and it means the
# dashboard is reachable from this machine and nowhere else. Treating
# it as success is exactly what hid a dashboard that was reachable
# from nowhere for the whole time `tailscale serve` was broken.
from utils.tailscale import is_tailnet_ip

hosts = [_bind_host(b) for b in binds]
if any(is_tailnet_ip(h) for h in hosts):
tailnet = next(h for h in hosts if is_tailnet_ip(h))
return [
Check(
"Dashboard port",
OK,
f"{tailnet}:{WEB_PORT} — tailnet only{held_by}",
)
]
return [
Check(
"Dashboard port", OK, f"127.0.0.1:{WEB_PORT} only (Tailscale){held_by}"
"Dashboard port",
WARN,
f"USE_TAILSCALE=true but {WEB_PORT} is bound to "
f"{hosts[0]}{held_by}, not a tailnet address — reachable from "
"this machine only. Condor falls back to loopback when it "
"cannot read the tailnet IP: check `tailscale status`, then "
"restart Condor.",
)
]

if public:
# Local mode has no login, so a public bind there is a different order
# of problem -- and it can only happen by setting WEB_HOST explicitly.
# "Enable Tailscale" is also the wrong hint when it is already on.
remedy = (
"unset WEB_HOST to go back to loopback"
if LOCAL_MODE
else (
"firewall it"
if USE_TAILSCALE
else "Enable Tailscale (`make setup`) or firewall it"
)
)
return [
Check(
"Dashboard port",
WARN,
f"{WEB_PORT} is reachable on all interfaces{held_by} — fine on a "
"trusted LAN, risky on a public VPS. Enable Tailscale "
"(`make setup`) or firewall it.",
FAIL if LOCAL_MODE else WARN,
f"{WEB_PORT} is reachable on all interfaces{held_by} — "
+ (
"local mode has no login at all, so this is open to "
f"anyone who can route to it. {remedy}."
if LOCAL_MODE
else f"fine on a trusted LAN, risky on a public VPS. {remedy}."
),
)
]
return [Check("Dashboard port", OK, f"127.0.0.1:{WEB_PORT} only{held_by}")]
Expand Down Expand Up @@ -478,9 +532,52 @@ async def _probe_and_close(cm, name: str) -> None:
await client.close()


_HB_CONTAINER = "hummingbot-api"

_LOCAL_HOSTS = ("localhost", "127.0.0.1", "::1", "0.0.0.0")

_HB_CONTAINER = "hummingbot-api"
# Where a co-located hummingbot-api would be, relative to Condor's own root.
_HB_API_DIR = Path(__file__).resolve().parent.parent.parent / "hummingbot-api"


def _hb_api_is_here() -> bool:
"""Is there a hummingbot-api install on THIS machine?

``hummingbot-api`` is also the MagicDNS name a co-located API answers to
when it has its own tailnet node, and treating that name as local is what
makes a down co-located stack report `docker compose ps` instead of
"check the host is reachable".

But it is equally the name Condor writes for a *remote* API over Tailscale
-- that is the default in its own installer -- and there the local-stack
advice names a directory that does not exist and a stack that was never
supposed to be here. The name alone cannot tell the two apart; whether the
install is on this disk can.
"""
try:
return (_HB_API_DIR / ".env").is_file() or (
_HB_API_DIR / "docker-compose.yml"
).is_file()
except OSError:
return False


def _is_local_host(host: str) -> bool:
"""Whether ``host`` names a service on this machine.

A tailnet address counts when it is *this node's own* -- reaching the API
at the tailnet IP of the box you are standing on is still a local stack.
"""
if host in _LOCAL_HOSTS:
return True
if host == _HB_CONTAINER and _hb_api_is_here():
return True
try:
from utils.tailscale import is_tailnet_ip, tailnet_ip

return is_tailnet_ip(host) and host == tailnet_ip()
except Exception:
return False


def _local_stack_diagnosis() -> list[Check]:
Expand Down Expand Up @@ -583,7 +680,7 @@ def check_hummingbot_api() -> list[Check]:
checks.append(Check(label, OK, "reachable, authenticated"))
except Exception as e:
checks.append(Check(label, FAIL, f"{e} — {_connection_hint(host, e)}"))
local_failure = local_failure or host in _LOCAL_HOSTS
local_failure = local_failure or _is_local_host(host)

if local_failure:
checks.extend(_local_stack_diagnosis())
Expand Down
68 changes: 55 additions & 13 deletions main.py
Original file line number Diff line number Diff line change
Expand Up @@ -1097,21 +1097,63 @@ async def _run_dual(application: Application) -> None:
await application.updater.start_polling(allowed_updates=Update.ALL_TYPES)
await application.start()

# Create and start the web server. WEB_HOST (utils.config.resolve_web_host)
# already resolves to loopback for Tailscale same as it does for local mode
# -- here we only have to make sure `tailscale serve` is actually proxying
# the tailnet to that loopback bind, or it would be reachable nowhere at
# all.
# Create and start the web server. With Tailscale on, WEB_HOST
# (utils.config.resolve_web_host) is normally this node's own tailnet
# address, which needs no proxy: the port is on the tailnet and on no
# public interface, full stop.
#
# `tailscale serve` is only the fallback for when that address could not be
# resolved -- daemon not up yet, node not registered -- in which case the
# bind failed closed to loopback and a proxy is the only way anything
# reaches it. Serve needs the daemon socket, which refuses unprivileged
# callers, so this often cannot succeed either; say so plainly rather than
# leaving the dashboard silently unreachable.
web_app = create_app()
if USE_TAILSCALE:
from utils.tailscale import ensure_serve

if not await ensure_serve(WEB_PORT):
logger.error(
"Dashboard bound to 127.0.0.1 only; tailnet forwarding could "
"not be confirmed, so it will NOT be reachable remotely "
"until `tailscale serve` is fixed (see error above)."
if USE_TAILSCALE and LOCAL_MODE:
# Local mode keeps loopback even with Tailscale on: the dashboard has
# no login, and a tailnet is a smaller audience than the internet, not
# an authenticated one. Intended, so no proxy and no warning.
logger.info(
"Dashboard on http://%s:%s — local mode stays on loopback even "
"with Tailscale enabled (no login on this dashboard).",
WEB_HOST,
WEB_PORT,
)
elif USE_TAILSCALE:
from utils.tailscale import is_tailnet_ip

if is_tailnet_ip(WEB_HOST):
logger.info(
"Dashboard on http://%s:%s — reachable on the tailnet only.",
WEB_HOST,
WEB_PORT,
)
elif os.environ.get("WEB_HOST", "").strip():
# An explicit WEB_HOST is a deliberate opt-out, not a failed
# lookup. Warning about a tailnet address we were never asked to
# find would report someone's own choice as a fault.
logger.info(
"Dashboard on http://%s:%s — WEB_HOST was set explicitly.",
WEB_HOST,
WEB_PORT,
)
else:
from utils.tailscale import ensure_serve

logger.warning(
"USE_TAILSCALE is set but this node's tailnet address could "
"not be determined, so the dashboard bound to %s. Falling "
"back to `tailscale serve`.",
WEB_HOST,
)
if not await ensure_serve(WEB_PORT):
logger.error(
"Dashboard bound to %s; tailnet forwarding could not be "
"confirmed, so it will NOT be reachable remotely. Check "
"`tailscale status`, then restart Condor to pick up the "
"tailnet address.",
WEB_HOST,
)
server = uvicorn.Server(_web_server_config(web_app))

# Start WebSocket manager
Expand Down
Loading
Loading