Skip to content
7 changes: 6 additions & 1 deletion docs/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ parsers all resolve automatically from the checkpoint and the GPU.
|---|---|---|
| `--host` | 127.0.0.1 | Bind address |
| `--port` | 1919 | Bind port |
| `--api-key` | `$FREETOKEN_API_KEY`, else off | Require the key on every route except `/health`, as `Authorization: Bearer <key>` or `x-api-key: <key>`; other requests get a 401 |
| `--gpu` | GPU 0 | GPU to run on: a UUID from `nvidia-smi -L` or an `nvidia-smi` index; see [below](#choosing-a-gpu) |
| `--max-running-requests` | 4 | Max concurrently running requests |
| `--max-output-tokens` | 32768 | Default output budget for requests that omit one |
Expand Down Expand Up @@ -131,14 +132,17 @@ ft shell --model ~/models/Qwen3.6-35B-A3B # serve + chat in one process
```

- Attach mode talks to `--server URL` (default `http://127.0.0.1:1919`)
- `--api-key KEY` (default `$FREETOKEN_API_KEY`) attaches to a server started with `--api-key`; with `--model` the shell uses that server's own key
- `/help` inside the shell lists the commands (`/think`, `/cache`, `/reset`).

## ft ctl

```bash
ft ctl [--base-url http://127.0.0.1:1919] [--timeout 10] [--json] <subcommand>
ft ctl [--base-url http://127.0.0.1:1919] [--api-key KEY] [--timeout 10] [--json] <subcommand>
```

`--api-key` defaults to `$FREETOKEN_API_KEY`.

| Subcommand | Endpoint | Purpose |
|---|---|---|
| `health` | `GET /health` | Server status, model, load progress |
Expand Down Expand Up @@ -166,6 +170,7 @@ Code and Hermes need no declaration.
| Flag | Meaning |
|---|---|
| `--server URL` | Server to point the agent at (default `http://127.0.0.1:1919`) |
| `--api-key KEY` | Key of a server started with `--api-key` (default `$FREETOKEN_API_KEY`). The agent gets it in its environment; Hermes and OpenClaw keep it in their config files |
| `--dry-run` | Print the planned config changes and command, touch nothing |
| `-y`, `--yes` | Approve install/config prompts |
| `--config` | Configure without launching |
Expand Down
37 changes: 32 additions & 5 deletions python/freetoken/control_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
from collections.abc import Sequence
from typing import Any

from freetoken.launch import resolve_api_key


DEFAULT_BASE_URL = "http://127.0.0.1:1919"
# (CLI dest, pool, rebuild body key). moe/mamba are slot counts and pass straight through;
Expand Down Expand Up @@ -74,6 +76,7 @@ def _request_json(
body: dict[str, Any] | None = None,
query: dict[str, Any] | None = None,
timeout: float = 10.0,
api_key: str | None,
) -> dict[str, Any]:
with _open_request(
method,
Expand All @@ -83,6 +86,7 @@ def _request_json(
query=query,
accept="application/json",
timeout=timeout,
api_key=api_key,
) as response:
raw = response.read()

Expand All @@ -104,6 +108,7 @@ def _request_sse_text(
*,
body: dict[str, Any] | None = None,
timeout: float = 10.0,
api_key: str | None,
) -> str:
with _open_request(
method,
Expand All @@ -112,6 +117,7 @@ def _request_sse_text(
body=body,
accept="text/event-stream",
timeout=timeout,
api_key=api_key,
) as response:
return _read_sse_text(response)

Expand All @@ -125,9 +131,12 @@ def _open_request(
query: dict[str, Any] | None = None,
accept: str,
timeout: float,
api_key: str | None,
) -> Any:
data = None
headers = {"Accept": accept}
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
if body is not None:
data = json.dumps(body).encode("utf-8")
headers["Content-Type"] = "application/json"
Expand Down Expand Up @@ -284,6 +293,10 @@ def _build_parser(prog: str) -> argparse.ArgumentParser:
default=DEFAULT_BASE_URL,
help=f"FreeToken server URL (default: {DEFAULT_BASE_URL})",
)
parser.add_argument(
"--api-key",
help="API key of a server started with --api-key (default: $FREETOKEN_API_KEY)",
)
parser.add_argument("--timeout", type=float, default=10.0, help="HTTP timeout in seconds")
parser.add_argument("--json", action="store_true", help="Print raw JSON response")

Expand Down Expand Up @@ -355,14 +368,19 @@ def main(argv: Sequence[str] | None = None, *, prog: str = "ft ctl") -> int:
args = parser.parse_args(list(argv) if argv is not None else None)
except SystemExit as exc:
return int(exc.code) if isinstance(exc.code, int) else 2
args.api_key = resolve_api_key(args.api_key)

try:
if args.command == "health":
doc = _request_json("GET", args.base_url, "/health", timeout=args.timeout)
doc = _request_json(
"GET", args.base_url, "/health", timeout=args.timeout, api_key=args.api_key
)
_print_doc(doc, _format_health, raw_json=args.json)
return 0
if args.command == "stats":
doc = _request_json("GET", args.base_url, "/v1/stats", timeout=args.timeout)
doc = _request_json(
"GET", args.base_url, "/v1/stats", timeout=args.timeout, api_key=args.api_key
)
_print_doc(doc, _format_stats, raw_json=args.json)
return 0
if args.command == "generate":
Expand All @@ -376,6 +394,7 @@ def main(argv: Sequence[str] | None = None, *, prog: str = "ft ctl") -> int:
"ignore_eos": args.ignore_eos,
},
timeout=args.timeout,
api_key=args.api_key,
)
if args.json:
print(json.dumps({"text": text}, ensure_ascii=False, indent=2, sort_keys=True))
Expand All @@ -390,13 +409,16 @@ def main(argv: Sequence[str] | None = None, *, prog: str = "ft ctl") -> int:
"/v1/requests",
query=query,
timeout=args.timeout,
api_key=args.api_key,
)
_print_doc(doc, _format_requests, raw_json=args.json)
return 0
if args.command == "cache" and args.cache_command in (None, "status"):
if args.cache_command is None and _cache_targets(args):
return _run_cache_rebuild(args)
doc = _request_json("GET", args.base_url, "/v1/cache/status", timeout=args.timeout)
doc = _request_json(
"GET", args.base_url, "/v1/cache/status", timeout=args.timeout, api_key=args.api_key
)
_print_doc(doc, _format_cache_status, raw_json=args.json)
return 0
if args.command == "cache" and args.cache_command == "rebuild":
Expand Down Expand Up @@ -444,7 +466,9 @@ def _run_cache_rebuild(args: argparse.Namespace) -> int:
print("error: cache rebuild requires at least one cache target", file=sys.stderr)
return 2
# The token targets can only be converted against the live geometry, so read it first.
status = _request_json("GET", args.base_url, "/v1/cache/status", timeout=args.timeout)
status = _request_json(
"GET", args.base_url, "/v1/cache/status", timeout=args.timeout, api_key=args.api_key
)
body = _rebuild_body(targets, status.get("geometry") or {})
body["timeout"] = args.wait
doc = _request_json(
Expand All @@ -453,6 +477,7 @@ def _run_cache_rebuild(args: argparse.Namespace) -> int:
"/v1/cache/rebuild",
body=body,
timeout=args.wait + args.timeout,
api_key=args.api_key,
)
if args.json or doc.get("status") != "ok":
_print_doc(doc, _format_rebuild, raw_json=args.json)
Expand All @@ -462,7 +487,9 @@ def _run_cache_rebuild(args: argparse.Namespace) -> int:
# /cache prints. Best-effort -- the rebuild already succeeded.
print("status=ok")
try:
print(_format_cache_status(_request_json("GET", args.base_url, "/v1/cache/status")))
print(_format_cache_status(
_request_json("GET", args.base_url, "/v1/cache/status", api_key=args.api_key)
))
except ControlCliError:
pass
return 0
Expand Down
11 changes: 9 additions & 2 deletions python/freetoken/daemon/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,9 @@ Target a non-default daemon with `--url http://host:1900` (or `$FREETOKEN_DAEMON
| Method / path | Notes |
| --- | --- |
| `GET /health` | Daemon self-health; always answers, never gated by `--token`. |
| `POST /engine/start` `{model,port,args[]}` | Idempotent on the full `(model,port,args)`; a differing config on the same port → `409`. |
| `POST /engine/start` `{model,port,args[],apiKey?}` | Idempotent on the full `(model,port,args,apiKey)`; a differing config on the same port → `409`. |
| `POST /engine/stop` `{force?:false}` | Close admission, drain/abort, durably enqueue the final-accounting receipt, then `SIGTERM`→grace→`SIGKILL`. A prepare/outbox failure preserves the engine. |
| `POST /engine/switch` `{model,port,args[],force?:false}` | One serialized stop-accounting-start transaction. |
| `POST /engine/switch` `{model,port,args[],apiKey?,force?:false}` | One serialized stop-accounting-start transaction. |
| `GET /engine/status` | `{running,pid,model,port,uptimeS,lastExitCode,…}`; outlives any single serve. |
| `GET /engine/logs?since=` | SSE, ANSI-stripped, tqdm-`\r` collapsed, ring replay, `id:<seq>`, `Last-Event-ID` resume. |
| `GET /engine/metrics` | `{ramBytes,vramBytes}` — the serve tree's own footprint only. |
Expand All @@ -75,6 +75,13 @@ Target a non-default daemon with `--url http://host:1900` (or `$FREETOKEN_DAEMON
Set `--token` (or `$FREETOKEN_DAEMON_TOKEN`) to require an `X-FT-Token` header on everything
except `/health`.

`apiKey` is the serve's own `--api-key`. The daemon hands it to the serve as `FREETOKEN_API_KEY` in
its environment (never on its command line), sends it on its own calls to the serve, and keeps it in
the owner-only state file so a re-adopted serve stays reachable. The daemon's own
`FREETOKEN_API_KEY` is never passed down: a serve requires a key only when its start request carries
one, and an `--api-key` inside `args` is rejected with a 400 because the daemon would not know it.
`ft daemon start|switch --api-key KEY` (default `$FREETOKEN_API_KEY`) sets it.

The daemon reaches the serve's destructive `POST /v1/admin/prepare-stop` endpoint only over
loopback; the serve rejects non-loopback callers even when its inference API is bound to
`0.0.0.0`. `force` is never implicit: it is an explicit recovery choice that may record null
Expand Down
22 changes: 21 additions & 1 deletion python/freetoken/daemon/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ class StartBody(BaseModel):
model: str
port: int | None = None
args: list[str] = []
apiKey: str | None = None


class StopBody(BaseModel):
Expand Down Expand Up @@ -161,6 +162,15 @@ def resolve_port(explicit: int | None) -> int:
st = manager.status()
return st.get("port") or default_serve_port

def reject_api_key_arg(args: list[str]) -> None:
# the daemon must know the key to reach the serve, so the key only travels as apiKey
# argparse also accepts any unambiguous prefix, and no other serve flag starts with --ap
flags = (a.split("=", 1)[0] for a in args)
if any(len(f) >= 4 and "--api-key".startswith(f) for f in flags):
raise HTTPException(
status_code=400, detail="pass the serve's API key as apiKey, not in args"
)

def accounting_error(exc: Exception) -> JSONResponse:
code = (
"accounting_outbox_failed"
Expand Down Expand Up @@ -192,9 +202,17 @@ async def health():

@app.post("/engine/start", dependencies=auth)
async def engine_start(body: StartBody):
reject_api_key_arg(body.args)
port = resolve_port(body.port)
try:
return await run(lifecycle_pool, manager.start, body.model, port, list(body.args))
return await run(
lifecycle_pool,
manager.start,
body.model,
port,
list(body.args),
body.apiKey or None,
)
except Conflict as exc:
st = manager.status()
return JSONResponse(
Expand Down Expand Up @@ -236,6 +254,7 @@ async def shutdown_daemon(request: Request, body: StopBody | None = None):

@app.post("/engine/switch", dependencies=auth)
async def engine_switch(body: SwitchBody):
reject_api_key_arg(body.args)
port = resolve_port(body.port)
try:
return await run(
Expand All @@ -245,6 +264,7 @@ async def engine_switch(body: SwitchBody):
port,
list(body.args),
body.force,
body.apiKey or None,
)
except (AccountingPrepareError, AccountingOutboxError) as exc:
return accounting_error(exc)
Expand Down
8 changes: 8 additions & 0 deletions python/freetoken/daemon/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
from collections.abc import Sequence
from typing import Any

from freetoken.launch import resolve_api_key

DEFAULT_URL = "http://127.0.0.1:1900"
DEFAULT_TIMEOUT = 10.0
# prepare-stop (15s transport budget) + default SIGTERM grace (10s) + reap wait (10s),
Expand Down Expand Up @@ -144,6 +146,10 @@ def _build_parser(prog: str) -> argparse.ArgumentParser:
sp = sub.add_parser(name, parents=[common], help=f"POST /engine/{name}")
sp.add_argument("model", help="Model path/id")
sp.add_argument("--port", type=int, default=None, help="Serve port")
sp.add_argument(
"--api-key",
help="API key the serve requires, set in its environment (default: $FREETOKEN_API_KEY)",
)
if name == "switch":
sp.add_argument(
"--force",
Expand Down Expand Up @@ -181,6 +187,8 @@ def main(argv: Sequence[str] | None = None, *, prog: str = "ft daemon") -> int:
body: dict[str, Any] = {"model": args.model, "args": list(args.serve_args)}
if args.port is not None:
body["port"] = args.port
if api_key := resolve_api_key(args.api_key):
body["apiKey"] = api_key
if args.verb == "switch" and args.force:
body["force"] = True
method, path = "POST", f"/engine/{args.verb}"
Expand Down
16 changes: 12 additions & 4 deletions python/freetoken/daemon/pidfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,16 @@
open for the daemon's whole life. flock (not a bare port-bind) avoids TIME_WAIT races and is
released automatically if the daemon dies, so a crashed daemon never wedges its own restart.

``ServeStateStore`` persists ``{model, port, pid, args, starttime, logPath}`` as JSON on every
``ServeStateStore`` persists ``{model, port, pid, args, starttime, log_path, api_key}`` as JSON on every
lifecycle change, so a restarted daemon can re-adopt a still-running serve. ``starttime`` +
``args`` are what make re-adoption PID-reuse-safe and config-exact."""
``args`` are what make re-adoption PID-reuse-safe and config-exact.
The file is owner-only because it holds the serve's API key."""

from __future__ import annotations

import json
import os
from dataclasses import asdict, dataclass
from dataclasses import asdict, dataclass, field


class AlreadyRunning(RuntimeError):
Expand Down Expand Up @@ -63,6 +64,7 @@ class ServeState:
args: list[str]
starttime: int | None = None
log_path: str | None = None
api_key: str | None = field(default=None, repr=False)


class ServeStateStore:
Expand All @@ -72,7 +74,12 @@ def __init__(self, path: str) -> None:
def save(self, state: ServeState) -> None:
os.makedirs(os.path.dirname(self.path) or ".", exist_ok=True)
tmp = f"{self.path}.tmp"
with open(tmp, "w") as fh:
# unlink first: O_CREAT applies the 0o600 mode only to a file it creates
try:
os.remove(tmp)
except FileNotFoundError:
pass
with os.fdopen(os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as fh:
json.dump(asdict(state), fh)
fh.flush()
os.fsync(fh.fileno())
Expand Down Expand Up @@ -102,6 +109,7 @@ def load(self) -> ServeState | None:
args=list(doc.get("args") or []),
starttime=doc.get("starttime"),
log_path=doc.get("log_path"),
api_key=doc.get("api_key"),
)
except (KeyError, TypeError, ValueError):
return None
18 changes: 12 additions & 6 deletions python/freetoken/daemon/proxy.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ def __init__(
now: Callable[[], float] = time.monotonic,
opener: Callable[[str, float], dict] | None = None,
prepare_opener: Callable[[str, float], dict] | None = None,
api_key: Callable[[], str | None] = lambda: None,
) -> None:
self._host = host
self._ttl = ttl_s
Expand All @@ -57,6 +58,7 @@ def __init__(
self._now = now
self._opener = opener or self._urlopen
self._prepare_opener = prepare_opener or self._urlopen_prepare
self._api_key = api_key
self._lock = threading.Lock()
self._cache: dict[tuple[str, int], tuple[float, dict]] = {}

Expand Down Expand Up @@ -109,19 +111,23 @@ def _fetch(self, path: str, port: int) -> dict:
result["reachable"] = True
return result

@staticmethod
def _urlopen(url: str, timeout: float) -> dict:
req = urllib.request.Request(url, headers={"Accept": "application/json"}, method="GET")
def _headers(self) -> dict[str, str]:
headers = {"Accept": "application/json"}
if key := self._api_key():
headers["Authorization"] = f"Bearer {key}"
return headers

def _urlopen(self, url: str, timeout: float) -> dict:
req = urllib.request.Request(url, headers=self._headers(), method="GET")
with urllib.request.urlopen(req, timeout=timeout) as resp:
raw = resp.read()
return json.loads(raw.decode("utf-8"))

@staticmethod
def _urlopen_prepare(url: str, timeout: float) -> dict:
def _urlopen_prepare(self, url: str, timeout: float) -> dict:
req = urllib.request.Request(
url,
data=b"{}",
headers={"Accept": "application/json", "Content-Type": "application/json"},
headers={**self._headers(), "Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req, timeout=timeout) as resp:
Expand Down
Loading
Loading