diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b15f59ec..61e755d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,12 @@ jobs: # changelog, matching the real release. fetch-depth: 0 + # This repository is public, so a committed account id, ARN, endpoint + # host, address or resource id would be a disclosure. Checked before + # anything else so it fails fast. + - name: No cloud identifiers + run: scripts/check-no-cloud-identifiers.sh + - uses: actions/setup-go@v7 with: go-version-file: go.mod diff --git a/.github/workflows/remote.yml b/.github/workflows/remote.yml new file mode 100644 index 00000000..8a9cde7f --- /dev/null +++ b/.github/workflows/remote.yml @@ -0,0 +1,61 @@ +name: Remote deployment + +# The CDK project under remote/ that `outfit remote` drives. It is TypeScript +# rather than Go, so it has its own workflow; the paths filter keeps it off +# pull requests that do not touch it. +on: + push: + branches: [main] + paths: + - "remote/**" + - ".github/workflows/remote.yml" + pull_request: + paths: + - "remote/**" + - ".github/workflows/remote.yml" + workflow_dispatch: + +permissions: + contents: read + +defaults: + run: + working-directory: remote + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: pnpm/action-setup@v4 + with: + version: 10 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: remote/pnpm-lock.yaml + + - name: Install + run: pnpm install --frozen-lockfile + + - name: Typecheck + run: pnpm build + + # The stack assertions: they synthesize each stack and assert over the + # resulting template (lambdas, IAM scoping, the deploy-config parameter), + # alongside the pure unit tests for the Lambdas' own logic. + - name: Test + run: pnpm test + + # Synthesizing the app itself covers what the assertions cannot: bin/app.ts + # wiring, and esbuild bundling every Lambda. No AWS credentials are needed + # — the stacks take their AZs from config rather than looking them up — so + # the context values here are placeholders. + # Invoked directly rather than through `pnpm synth`: pnpm appends extra + # arguments to the end of the script's command line, so the context values + # never reach cdk and the synth fails on the required `runner`. + - name: Synthesize + run: npx cdk synth -c allowedCidr=203.0.113.7/32 -c runner=llamacpp --quiet diff --git a/AGENTS.md b/AGENTS.md index 3d3c817e..bdf0f263 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,18 +22,21 @@ Run a single test: `go test -run TestWriteConfig_Idempotent ./...` The binary lives under `cmd/`; the domain logic is split into `internal/` packages so each concern is isolated and independently testable. -- `cmd/outfit/main.go` — CLI: command dispatch (`add`/`remove`/`list`/`show`/`apply`/`unapply`/`alias`/`unalias`/`serve`/`export`/`harness`/`init-providers`/`completion`, plus the hidden `__complete`), flag parsing (`parseSelection` registers both long and short flags against the same vars, and returns the chosen harness name separately so it never leaks into a `Selection`), and user-facing output. `add` and `apply` share `applySelection`, the core that writes one provider selection; `remove` and `unapply` share `removeSelection`, its inverse. Both resolve the active harness (`harness.Resolve`) and route through it. `show` reports the active harness's current config via `harness.State` (the same read `export` reconstructs an Outfit from). `serve` is harness-agnostic (it launches `llama-server`, not a config write). `harness` launches the active harness (forwarding trailing args), or manages the stored default with `--get`/`--set`; its `--outfit`/`-O` routes through the same `applySelection` before launching, so a launch can dress the harness first. That flag's value is optional (`outfitPathFlag` implements `IsBoolFlag`) — bare it means the default `./Outfit`, so a path has to be attached (`--outfit=`), since positional args belong to the harness; the exception is a *leading* positional that names an Outfit, which is applied and not forwarded (see "Alias registry" below). `alias`/`unalias` manage the registry that lets a name stand in for a path, and `writeAliases` renders it for both `alias --list` and `show`. Tests: `main_test.go` (dispatch/add/remove/list), `apply_test.go` (`apply`/`unapply`/`export`), `harness_test.go` (the `harness` command, `show`, and Pi-routed add/export/remove), `serve_test.go`, `alias_test.go`, `complete_test.go`. +- `cmd/outfit/main.go` — CLI: command dispatch (`add`/`remove`/`list`/`show`/`apply`/`unapply`/`alias`/`unalias`/`serve`/`export`/`harness`/`init-providers`/`remote`/`completion`, plus the hidden `__complete`), flag parsing (`parseSelection` registers both long and short flags against the same vars, and returns the chosen harness name separately so it never leaks into a `Selection`), and user-facing output. `add` and `apply` share `applySelection`, the core that writes one provider selection; `remove` and `unapply` share `removeSelection`, its inverse. Both resolve the active harness (`harness.Resolve`) and route through it. `show` reports the active harness's current config via `harness.State` (the same read `export` reconstructs an Outfit from). `serve` is harness-agnostic (it launches `llama-server`, not a config write). `harness` launches the active harness (forwarding trailing args), or manages the stored default with `--get`/`--set`; its `--outfit`/`-O` routes through the same `applySelection` before launching, so a launch can dress the harness first. That flag's value is optional (`outfitPathFlag` implements `IsBoolFlag`) — bare it means the default `./Outfit`, so a path has to be attached (`--outfit=`), since positional args belong to the harness; the exception is a *leading* positional that names an Outfit, which is applied and not forwarded (see "Alias registry" below). `alias`/`unalias` manage the registry that lets a name stand in for a path, and `writeAliases` renders it for both `alias --list` and `show`. Tests: `main_test.go` (dispatch/add/remove/list), `apply_test.go` (`apply`/`unapply`/`export`), `harness_test.go` (the `harness` command, `show`, and Pi-routed add/export/remove), `serve_test.go`, `alias_test.go`, `complete_test.go`. - `cmd/outfit/complete.go` + `completion.bash` — tab completion, and the only reason the CLI is no longer a single source file: `//go:embed` needs the script beside it. `outfit completion bash` prints the script; the script calls the hidden `outfit __complete `, which prints one candidate per line and a final `:nofile`/`:file` directive. The `commands` table there mirrors `run`'s switch and each `cmdX`'s `FlagSet` (the `flag` package cannot enumerate a `FlagSet`); `TestCompletionCoversDispatch` scans `main.go` and fails when a new command is not completable. - `internal/config` — outfit's own config file, `${XDG_CONFIG_HOME:-~/.config}/outfit/config.json`: the default-harness preference and the Outfit alias registry. A **leaf package** — stdlib imports only, and it must stay that way (in particular it must never import `internal/outfit` to parse an Outfit), because that is what lets `readOutfit` resolve a name without dragging in the harness abstraction. Every write goes through `Update`, a read-modify-write of the whole document; unknown top-level keys round-trip through `File.extra`. -- `internal/outfit` — the `Outfit` file format and the shared `Selection` type: a flat, Dockerfile-style description of one provider selection (`PROVIDER`/`FAMILY`/`MODEL`/`ALIAS`/`CONTEXT`/`OUTPUT`/`BASEURL`/`PRESET`). `Parse` reads it into a `Selection` (keywords case-insensitive via `canonicalKeyword`, UPPERCASE canonical, `#` comments); `Format` renders one back out for `export`. `apply` defaults to `./Outfit` (`outfit.DefaultFile`). The harness is deliberately **not** a keyword — it is a runtime choice, so an Outfit stays portable. +- `internal/outfit` — the `Outfit` file format and the shared `Selection` type: a flat, Dockerfile-style description of one provider selection (`PROVIDER`/`FAMILY`/`MODEL`/`ALIAS`/`CONTEXT`/`OUTPUT`/`BASEURL`/`PRESET`/`REMOTE`). `Parse` reads it into a `Selection` (keywords case-insensitive via `canonicalKeyword`, UPPERCASE canonical, `#` comments); `Format` renders one back out for `export`. `apply` defaults to `./Outfit` (`outfit.DefaultFile`). The harness is deliberately **not** a keyword — it is a runtime choice, so an Outfit stays portable. - `internal/harness` — the harness abstraction. The `Harness` interface (`Name`/`Command`/`ConfigPath`/`Apply`/`Remove`/`State`), a `registry` of the opencode and pi adapters (`adapters.go`, which wrap `catalog` + `opencode`/`pi` + `contextsize`), runtime selection via `Resolve` with precedence **`--harness`/`-H` flag > `OUTFIT_HARNESS` env > stored preference > opencode**, and preference load/save (`LoadPreference`/`SavePreference`), which delegate to `internal/config` — that package, not this one, owns the config file. **Start here when adding a third harness:** implement the interface and register it. -- `internal/catalog` — the embedded provider catalogue (`//go:embed providers.yaml`), its types, and the two block builders: `BuildProviderBlock` turns a provider+family+model selection into an opencode provider block; `BuildPiProvider` turns the same into a Pi provider entry. `MatchFamily` does the reverse for `export` (configured models -> family name). The catalogue can be overridden at runtime via the `--providers` flag or `OUTFIT_PROVIDERS` env var (flag > env > embedded; see `ResolveCatalogPath`/`LoadFrom`). `providers.yaml` lives here. +- `internal/catalog` — the embedded provider catalogue (`//go:embed providers.yaml`), its types, and the two block builders (`apiKeyOptional` marks a provider that also works unauthenticated — `llamacpp` names both a keyless local server and the authenticated remote deployment — so the Pi builder writes its keyless placeholder instead of a `$VAR` that resolves to nothing; every other provider keeps the reference, which Pi resolves at run time): `BuildProviderBlock` turns a provider+family+model selection into an opencode provider block; `BuildPiProvider` turns the same into a Pi provider entry. `MatchFamily` does the reverse for `export` (configured models -> family name). The catalogue can be overridden at runtime via the `--providers` flag or `OUTFIT_PROVIDERS` env var (flag > env > embedded; see `ResolveCatalogPath`/`LoadFrom`). `providers.yaml` lives here. - `internal/opencode` — opencode config IO: JSONC read/merge/write, env/key resolution, JSON-Pointer helpers. `LoadConfigState` reads the config back (configured providers, their model keys, the default model, context/output limits) for `export`. - `internal/pi` — Pi config IO: resolves `~/.pi/agent/models.json` (note: `~/.pi/agent/`, **not** XDG), deep-merges one managed provider entry into it preserving everything else (other providers, unknown fields), `Remove`, and reads provider state back for `export`. Written `0600`. +- `cmd/outfit/remote.go` — the `remote` command group (`start`/`stop`/`status`/`deploy`), the CLI's only nested subcommand: each subcommand is a thin wrapper over `internal/remote`. `deploy` is the exception that carries logic: `deployConfigFor` turns an Outfit (plus its preset) into a `remote.DeployConfig` — `PROVIDER` picks the runner (`llamacpp`/`vllm`; other providers are not self-hosted engines, so there is nothing to deploy), `MODEL` or the preset's `hf` names the weights as `repo:QUANT`, `CONTEXT` or the preset's `ctx-size` the window, `ALIAS` the served name, and the preset's remaining flags become `serveArgs`. The `[*]` globals are layered under the section (so `ngl`/`jinja` written there survive), and `cloudOwnedFlags` drops what the cloud sets itself (host/port/model/hf/api-key/ctx-size/alias/metrics), matched on `preset.CanonicalKey` so short aliases count. Config discovery (`resolveRemoteConfig`): an explicit `[path]` argument must name an Outfit with a `REMOTE` instruction; with no argument, `./Outfit` is consulted when present (its `REMOTE` resolved relative to the Outfit, like `PRESET`), falling back to the per-user `remote.json`. Tests: `remote_test.go` (dispatch plus full flows against an `httptest.Server`, with `stubAWSEnv` pinning the AWS credential chain to static env credentials). +- `internal/remote` — controls the scale-to-zero GPU inference instance defined in [`remote/`](remote/) by calling its start/stop/deploy Lambda Function URLs (IAM auth, so requests are SigV4-signed via `aws-sdk-go-v2` — the repo's only AWS/network dependency). `Deploy` is the only call with a body, and SigV4 covers the payload hash, so `call`/`sign` take the bytes and hash exactly what is sent. `DeployConfig` deliberately carries no weights prefix: the Lambda derives the S3 layout and seeds the weights when they are absent, replying `{seeding, seedInstanceId}`. Config in `${XDG_CONFIG_HOME:-~/.config}/outfit/remote.json` (`start_url`/`stop_url`/`deploy_url`/`region`), overridable via `OUTFIT_REMOTE_*` env vars; `deploy_url` is optional so configs predating deploy still work for start/stop, and only `Deploy` complains when it is missing; the region falls back to `AWS_REGION`, then to the region embedded in the Function URL host. `Start` retries while the endpoint reports 503/starting, honouring `retry_after_seconds`. - `internal/contextsize` — parses human-friendly sizes (`128k`, `1.5m`, `200000`); `Apply` sets `limit.context` and `limit.output` on opencode model blocks; `DefaultOutput` returns a quarter of the context (the fallback when `OUTPUT` is unset). - `internal/preset` — parses llama.cpp [preset `.ini`](https://github.com/ggml-org/llama.cpp/blob/master/docs/preset.md) files into named sections of flags, and renders a `llama-server` command for `serve` (`Parse`/`Flags`/`FormatCommand`). - `internal/catalog/providers.yaml` — externalised provider/model-family data (URLs, model ids, key env vars, and a `pi:` block per Pi-capable provider). **Add providers/models here, not in Go.** Embedded at build time but kept external for maintenance. - `examples/` — runnable guides, each a directory with a README and an `Outfit`. +- `remote/` — the cloud GPU deployment `outfit remote` drives: a TypeScript CDK project (AWS Lambdas, EC2 Image Builder, S3 weights) with its own `package.json`, tests and docs, built and tested by pnpm rather than Go. It is the only non-Go part of the repository; CI runs its checks separately. **This repository is public, so nothing identifying a deployment — account ids, ARNs, Function URL hosts, allocated addresses, bucket names, resource ids — may be committed here or anywhere else in the repo.** `scripts/check-no-cloud-identifiers.sh` enforces that in CI; deployment state belongs in the gitignored files under `remote/` (`.env`, `Outfit`, `remote.json`, `cdk-outputs.json`). ## Architecture notes (the important part) @@ -50,15 +53,27 @@ The binary lives under `cmd/`; the domain logic is split into `internal/` packag - `remove` guards every `remove` op with `Find` (RFC 6902 errors on missing paths) and clears the top-level default `model` when it pointed at something removed. - Output is written `0600` (and `chmod`ed, since `WriteFile` won't change perms on an existing file) because it may hold an API key. -**Pi config (`internal/pi`).** Pi's `models.json` is plain JSON of the form `{"providers": {"": {baseUrl, api, apiKey, models: [...]}}}`. The whole file is read into a generic map so unknown top-level keys, sibling providers, and unknown provider fields all round-trip untouched; only the managed provider is merged in (models unioned by `id`). `apiKey` is written as a `$ENV_VAR` interpolation (never the resolved secret), and **keyless local providers get a dummy literal placeholder** (`piPlaceholderAPIKey`) because Pi hides a provider's models from `/model` until *some* auth is configured. Pi has no top-level default-model setting, so `export` relies on the provider selection alone and `add` tells the user which model to pick. +**Pi config (`internal/pi`).** Pi's `models.json` is plain JSON of the form `{"providers": {"": {baseUrl, api, apiKey, models: [...]}}}`. The whole file is read into a generic map so unknown top-level keys, sibling providers, and unknown provider fields all round-trip untouched; only the managed provider is merged in (models unioned by `id`). `apiKey` is written as a `$ENV_VAR` interpolation (never the resolved secret), and **keyless local providers get a dummy literal placeholder** (`piPlaceholderAPIKey`) because Pi hides a provider's models from `/model` until *some* auth is configured. The reference is otherwise always written, even when the variable is currently unset, because Pi resolves it at run time — the key can be exported after the Outfit is applied. The one exception is `apiKeyOptional` (see the traps below), where an unset variable means a genuinely keyless server rather than a key yet to be supplied. Pi has no top-level default-model setting, so `export` relies on the provider selection alone and `add` tells the user which model to pick. -**Key/option resolution.** `catalog.BuildProviderBlock` injects `options.apiKey` (opencode) from the provider's `apiKeyEnv` (resolved from `.env` next to the tool, then the environment), validating `apiKeyPrefix` and erroring when `apiKeyRequired`. `optionsFromEnv` injects other options (e.g. `baseURL`, `region`) when their env vars are set. The opencode top-level `model` is `/`. Base-URL precedence for both builders: `--base-url`/`-u` flag > `OUTFIT_BASE_URL` > the catalogue's `pi.baseUrl`/`options.baseURL`. +**Key/option resolution.** `catalog.BuildProviderBlock` sets `options.apiKey` (opencode) to the provider's `apiKeyEnv` as an `{env:VAR}` reference — never the secret — validating `apiKeyPrefix` and erroring when `apiKeyRequired` against the value it resolves. Resolution is `opencode.EnvResolver(dir)`: a `.env` **beside the Outfit being applied**, then the environment — `dir` is empty for a flags-only `add`, which falls back to the working directory. It deliberately is not relative to the binary or the source file (the old `runtime.Caller` form resolved to the build machine's path, so an installed binary never found one). The option is omitted only for a provider with no key variable, or an `apiKeyOptional` one at a local endpoint. Both harnesses therefore rely on the variable being set when the agent runs, which `harnessEnv` in the CLI arranges at launch. `optionsFromEnv` injects other options (e.g. `baseURL`, `region`) when their env vars are set. The opencode top-level `model` is `/`. Base-URL precedence for both builders: `--base-url`/`-u` flag > `OUTFIT_BASE_URL` > the catalogue's `pi.baseUrl`/`options.baseURL`. **`serve`.** `outfit serve` launches `llama-server` for an Outfit without touching any harness config. With a `PRESET` it flattens the matching `.ini` section (plus `[*]` defaults) into flags via `internal/preset`; otherwise it derives a command from `MODEL`/`ALIAS`/`CONTEXT`/`BASEURL`. Anything the Outfit states overrides the preset. `--dry-run`/`-n` prints the command without running it. Invariants to keep: opencode merges stay idempotent and never drop unrelated config or comments; Pi merges preserve sibling providers and unknown fields; output stays `0600`; the harness never appears in an Outfit, and neither does an alias — both are machine-local, so an Outfit stays portable; outfit's own `config.json` is only ever written read-modify-write, so a new setting cannot clobber an existing one; `__complete` never returns an error and never writes to stderr, whatever the state of the config or catalogue; and every family's `defaultModel` must be one of its `models` (enforced by `TestCatalogIntegrity`). +## Traps + +These are mistakes already made here; each was silent rather than loud, which is what makes them worth writing down. + +**The catalogue is embedded at build time.** `providers.yaml` is `//go:embed`-ed, so a previously-built `outfit` binary keeps applying the *old* catalogue no matter what the file says. Rebuild before testing any catalogue change, or you will "verify" a fix that is not in the binary you ran. (`--providers`/`OUTFIT_PROVIDERS` reads a file at run time and sidesteps this.) + +**A preset section is not the whole preset.** `Preset.Select` returns only the named `[section]`; the `[*]` defaults live separately in `Preset.Global`. Anything that consumes a section's `Params` directly — rather than going through `Args`/`Command`, which layer both — silently drops whatever the user put in `[*]`. That is usually the settings they consider obvious enough to write once, like `ngl` and `jinja`, so the failure surfaces later as a model running on CPU or refusing tool calls. + +**Match preset flags by canonical name.** Preset keys have short aliases (`ngl` is `n-gpu-layers`, `c` is `ctx-size`, `hf` is `hf-repo`, `a` is `alias`). Comparing raw keys against a list of flag names therefore matches only the spelling you happened to think of. Use `preset.CanonicalKey`, which is exported for exactly this; `Flags` already dedupes layers by canonical name, last layer winning. + +**`contextsize.Parse` is decimal.** `128k` is 128000, not 131072 — a `CONTEXT` written that way is not the power-of-two window it looks like. It also *overrides* a preset's `ctx-size` (both in `serve` and in `remote deploy`), so the Outfit, not the preset, decides the window whenever it states one. + ## Reference -- opencode config schema: https://opencode.ai/docs/config/. The catalogue follows it: `amazon-bedrock` is the Bedrock provider id, custom providers (`ollama`, `llamacpp`, `openai-compatible`) carry an `npm` package plus `options.baseURL`. opencode also supports `{env:VAR}` substitution in the config, but this tool deliberately embeds the resolved key instead (a global config can't rely on a project-local `.env`). +- opencode config schema: https://opencode.ai/docs/config/. The catalogue follows it: `amazon-bedrock` is the Bedrock provider id, custom providers (`ollama`, `llamacpp`, `openai-compatible`) carry an `npm` package plus `options.baseURL`. The key is written as opencode's `{env:VAR}` substitution rather than the resolved secret, so no secret lands on disk; `outfit harness` passes the keys it can resolve to the agent it launches, which is what makes a config outfit wrote usable without exporting anything by hand. - Pi custom-models schema: https://github.com/earendil-works/pi (`packages/coding-agent/docs/models.md`). `api` is one of `openai-completions`/`openai-responses`/`anthropic-messages`/`google-generative-ai`; `apiKey` supports `$ENV_VAR` interpolation. Not every provider maps to Pi — those without a `pi:` block (e.g. `amazon-bedrock`) error under the pi harness. diff --git a/README.md b/README.md index d2ae5eec..c336f4e7 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,9 @@ outfit harness [] [-H ] [--outfit[=]] [args...] # launch the harness (a leading Outfit or alias is # applied first; --get shows it; --set stores it) outfit completion # tab completion (bash, zsh, powershell) +outfit remote [path] + # control the remote GPU inference instance + # (deploy sets what it serves, from the Outfit) ``` Short flags: `-p` (provider), `-f` (model-family), `-m` (model), `-a` (alias), `-c` (context), `-o` (output), `-u` (base-url), `-H` (harness), `-O` (outfit), and under `alias`: `-n` (name), `-l` (list), `-F` (force). @@ -284,10 +287,40 @@ the chosen section into the command instead — with anything the `Outfit` state cover: launching a *single* model. Details in [`docs/commands/serve.md`](docs/commands/serve.md). +## Remote inference instance + +Running a model on your own cloud GPU box? [`remote/`](remote/) deploys one. +`outfit remote` drives its scale-to-zero lifecycle: the instance only exists +while you are using it, and stops itself after a period of idleness. + +```sh +outfit remote start # boot the instance, wait for the model to load, + # then print OPENAI_BASE_URL / OPENAI_API_KEY exports +outfit remote status # instance state and endpoint health +outfit remote stop # stop now instead of waiting for the idle timer +``` + +Configuration is found in one of two places. A project's `Outfit` file can +name it with a `REMOTE` instruction (`REMOTE remote.json`, resolved relative +to the Outfit — like `PRESET`, so the pair travel together); otherwise the +per-user `${XDG_CONFIG_HOME:-~/.config}/outfit/remote.json` is used. Either +way, paste the `OutfitRemoteConfig` output of the `remote/` deployment there: + +```json +{"start_url": "https://...lambda-url...on.aws/", "stop_url": "https://...", "region": "eu-west-1"} +``` + +Each field can be overridden with `OUTFIT_REMOTE_START_URL`, +`OUTFIT_REMOTE_STOP_URL` and `OUTFIT_REMOTE_REGION`. Requests are SigV4-signed +with your AWS credentials (env, profile or SSO — the standard chain), which +must be allowed `lambda:InvokeFunctionUrl`. A cold `start` takes a few +minutes while the instance boots and loads the model; `--timeout` (default +15m) caps the wait. + ## Keys and endpoints Each provider declares which environment variable holds its key (`outfit -list` shows them). Values are looked up in `.env` next to the tool first, then +list` shows them). Values are looked up in a `.env` beside the `Outfit` first, then your shell environment. Local providers like Ollama and llama.cpp need no key; Bedrock authenticates through your AWS credentials. diff --git a/cmd/outfit/complete.go b/cmd/outfit/complete.go index 180143ab..a78c6382 100644 --- a/cmd/outfit/complete.go +++ b/cmd/outfit/complete.go @@ -77,6 +77,10 @@ type command struct { // positionals how many arguments that applies to. positional candidateKind positionals int + // subcommands is the fixed set a nested command dispatches on. When set, + // the first positional completes to one of these and any later one falls + // through to positional. + subcommands []string } // selectionFlags are the flags add and remove share (see parseSelection). @@ -162,6 +166,14 @@ var commands = map[string]command{ // harness's own, and outfit has nothing to say about them. positional: kindAlias, positionals: 1, }, + "remote": { + flags: []string{"--timeout", "--dry-run", "-n"}, + values: map[string]candidateKind{"--timeout": kindNone}, + // `outfit remote [outfit]`: the subcommand, then optionally the + // Outfit naming the endpoint. + subcommands: []string{"start", "stop", "status", "deploy"}, + positional: kindAlias, positionals: 2, + }, "completion": {positional: kindShell, positionals: 1}, "version": {}, "help": {}, @@ -233,7 +245,11 @@ func completions(args []string) ([]string, string) { if strings.HasPrefix(cur, "-") { return cmd.flags, directiveNoFile } - if positionalsUsed(cmd, words[1:]) >= cmd.positionals { + used := positionalsUsed(cmd, words[1:]) + if len(cmd.subcommands) > 0 && used == 0 { + return cmd.subcommands, directiveNoFile + } + if used >= cmd.positionals { return nil, directiveNoFile } return candidatesFor(cmd.positional, words) diff --git a/cmd/outfit/harness_test.go b/cmd/outfit/harness_test.go index bb68af20..e1f4c8d8 100644 --- a/cmd/outfit/harness_test.go +++ b/cmd/outfit/harness_test.go @@ -649,3 +649,38 @@ func TestCmdExportRemove_PiRoundTrip(t *testing.T) { t.Error("ollama should have been removed from the Pi config") } } + +// Neither harness stores the secret — opencode substitutes {env:VAR}, Pi +// resolves $VAR — so a key kept only in outfit's .env has to reach the agent +// outfit launches, or the user would have to export it by hand. +func TestHarnessEnv_CarriesResolvableKeys(t *testing.T) { + t.Setenv("OPENAI_API_KEY", "") + fromDotEnv := func(name string) string { + if name == "OPENAI_API_KEY" { + return "sk-from-dotenv" + } + return "" + } + + var found string + for _, kv := range harnessEnv("", fromDotEnv) { + if strings.HasPrefix(kv, "OPENAI_API_KEY=") { + found = kv + } + } + if found != "OPENAI_API_KEY=sk-from-dotenv" { + t.Errorf("launched agent's env has %q, want the key from .env", found) + } +} + +// An explicit export is the user's own decision and must win. +func TestHarnessEnv_DoesNotOverrideTheEnvironment(t *testing.T) { + t.Setenv("OPENAI_API_KEY", "sk-exported") + fromDotEnv := func(string) string { return "sk-from-dotenv" } + + for _, kv := range harnessEnv("", fromDotEnv) { + if kv == "OPENAI_API_KEY=sk-from-dotenv" { + t.Error("the .env value overrode an exported one") + } + } +} diff --git a/cmd/outfit/main.go b/cmd/outfit/main.go index 625735ea..4b47f74f 100644 --- a/cmd/outfit/main.go +++ b/cmd/outfit/main.go @@ -25,6 +25,8 @@ // outfit harness [-H name] [-O[=path]] # launch the harness, optionally applying an // # Outfit first (--get shows it; --set stores the default) // outfit completion # print the tab-completion script +// outfit remote start|stop|status|deploy # control the remote GPU inference +// # instance (deploy sets what it serves) // // Short flags: -p (provider), -f (model-family), -m (model), -a (alias), // -c (context), -o (output), -u (base-url), -H (harness), -O (outfit). @@ -60,6 +62,7 @@ import ( "github.com/lucinate-ai/outfit/internal/config" "github.com/lucinate-ai/outfit/internal/contextsize" "github.com/lucinate-ai/outfit/internal/harness" + "github.com/lucinate-ai/outfit/internal/opencode" "github.com/lucinate-ai/outfit/internal/outfit" "github.com/lucinate-ai/outfit/internal/preset" ) @@ -112,6 +115,8 @@ func run(args []string) error { case "__complete": // Hidden: the completion script's way of asking what could come next. return cmdComplete(rest) + case "remote": + return cmdRemote(rest) case "version", "-v", "--version": fmt.Println(version) return nil @@ -142,6 +147,9 @@ Usage: outfit harness [] [-H ] [--outfit[=]] [args...] (launch the harness; available: %s) outfit completion (tab completion: bash, zsh, powershell) + outfit remote [path] + (control the remote GPU instance; deploy sets + what it serves, from the Outfit) outfit version (or -v/--version) Flags: @@ -204,6 +212,12 @@ completion: prints a tab-completion script for bash, zsh, or powershell. Add source <(outfit completion bash) to your ~/.bashrc (zsh: swap in zsh) and TAB then completes commands, flags, providers, harnesses, and your registered aliases. +remote: runs the model on a cloud GPU that exists only while you use it, from + the same Outfit. deploy says what to serve (PROVIDER picks the engine, + just as it does for serve); start boots it and prints the exports your + agent needs; status reports its state; stop shuts it down rather than + waiting for the idle timer. The endpoint's URLs come from the Outfit's + REMOTE file, or ~/.config/outfit/remote.json. `, strings.Join(harness.Names(), ", ")) } @@ -249,13 +263,16 @@ func cmdAdd(args []string) error { if err != nil { return err } - return applySelection(sel, h) + return applySelection(sel, h, "") } // applySelection writes a single provider selection into the active harness's // config. It is the shared core of `add` and `apply`: both resolve a selection // (from flags or an Outfit file) and hand it here. -func applySelection(sel outfit.Selection, h harness.Harness) error { +// envDir is the directory of the Outfit the selection came from, which is where +// a `.env` holding its API key belongs; it is empty when no Outfit is involved +// (an `outfit add` from flags), leaving only the process environment. +func applySelection(sel outfit.Selection, h harness.Harness, envDir string) error { if sel.Family == "" && sel.Model == "" && sel.Alias == "" { return fmt.Errorf("a provider selection needs a model family, a model, or an alias") } @@ -291,7 +308,7 @@ func applySelection(sel outfit.Selection, h harness.Harness) error { } } - summary, err := h.Apply(p, sel, contextSize, outputSize) + summary, err := h.Apply(p, sel, contextSize, outputSize, opencode.EnvResolver(envDir)) if err != nil { return err } @@ -414,7 +431,7 @@ func cmdApply(args []string) error { if rest := fs.Args(); len(rest) > 0 { path = rest[0] } - sel, _, err := readOutfit("outfit apply ", path) + sel, outfitPath, err := readOutfit("outfit apply ", path) if err != nil { return err } @@ -423,7 +440,7 @@ func cmdApply(args []string) error { if output != "" { sel.Output = output } - return applySelection(sel, h) + return applySelection(sel, h, filepath.Dir(outfitPath)) } // cmdUnapply reads an Outfit file and removes what it selects — the inverse of @@ -650,12 +667,7 @@ func cmdServe(args []string) error { var argv []string if sel.Preset != "" { - // A relative PRESET is resolved against the Outfit's directory, so an - // Outfit and its preset can travel together. - presetPath := sel.Preset - if !filepath.IsAbs(presetPath) { - presetPath = filepath.Join(filepath.Dir(outfitPath), presetPath) - } + presetPath := resolvePresetPath(sel.Preset, outfitPath) data, err := os.ReadFile(presetPath) if err != nil { return fmt.Errorf("reading preset %s: %w", presetPath, err) @@ -708,6 +720,16 @@ func cmdServe(args []string) error { return nil } +// resolvePresetPath resolves an Outfit's PRESET value: a relative one is taken +// against the Outfit's own directory, so an Outfit and its preset travel +// together (the same rule REMOTE uses). +func resolvePresetPath(presetValue, outfitPath string) string { + if filepath.IsAbs(presetValue) { + return presetValue + } + return filepath.Join(filepath.Dir(outfitPath), presetValue) +} + // outfitServeParams turns the llama-server settings an Outfit states into preset // params: the provider-native MODEL supplies the model source (hf for a Hugging // Face repo, model for a .gguf path); ALIAS, CONTEXT, and BASEURL fill in the @@ -1053,8 +1075,13 @@ func cmdHarness(args []string) error { } } + // A .env beside the applied Outfit is where its keys live, so the launched + // agent is given the same ones. Without an Outfit there is no such file and + // only the environment is passed on. + var envDir string if outfitPath.set { - if err := applyBeforeLaunch(outfitPath, providers, h, rest); err != nil { + var err error + if envDir, err = applyBeforeLaunch(outfitPath, providers, h, rest); err != nil { return err } } @@ -1065,6 +1092,7 @@ func cmdHarness(args []string) error { cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr + cmd.Env = harnessEnv(providers, opencode.EnvResolver(envDir)) if err := cmd.Run(); err != nil { if errors.Is(err, exec.ErrNotFound) || errors.Is(err, os.ErrNotExist) { return fmt.Errorf("%s not found — install the %s harness or add it to your PATH", bin, h.Name()) @@ -1079,6 +1107,36 @@ func cmdHarness(args []string) error { return nil } +// harnessEnv is the environment for the agent outfit launches: this process's, +// plus any provider API key outfit can resolve that the environment does not +// already carry. Neither harness stores the secret itself — opencode +// substitutes {env:VAR} and Pi resolves $VAR, both when they run — so without +// this a key kept only in outfit's .env would never reach the agent, and the +// user would have to export it by hand. +// +// A variable already set in the environment is left alone, so an explicit +// export always wins. A catalogue that cannot be loaded is not fatal: launching +// the agent matters more than the keys, and it will report its own auth error. +func harnessEnv(providersPath string, resolve func(string) string) []string { + env := os.Environ() + cat, err := catalog.LoadFrom(catalog.ResolveCatalogPath(providersPath)) + if err != nil { + return env + } + seen := map[string]bool{} + for _, name := range cat.SortedProviderNames() { + key := cat.Providers[name].APIKeyEnv + if key == "" || seen[key] || os.Getenv(key) != "" { + continue + } + seen[key] = true + if value := resolve(key); value != "" { + env = append(env, key+"="+value) + } + } + return env +} + // flagsTerminated reports whether flag parsing stopped at an explicit bare // `--`. The flag package consumes that terminator without reporting it, so the // only reliable trace is the last argument it swallowed — scanning args for a @@ -1117,26 +1175,29 @@ func namesAnOutfitOrAlias(arg string) bool { // is about to be launched — exactly the work `outfit apply` does, so one command // can dress the harness and then run it. rest is what will be forwarded to the // harness, inspected only to catch a path that was meant for the flag. -func applyBeforeLaunch(f outfitPathFlag, providers string, h harness.Harness, rest []string) error { +// It returns the applied Outfit's directory, which is where its `.env` lives, +// so the launched agent can be given the same keys the apply resolved. +func applyBeforeLaunch(f outfitPathFlag, providers string, h harness.Harness, rest []string) (string, error) { // The flag's value has to be attached, so `--outfit ./dev/Outfit` (or // `--outfit q3`) would otherwise apply ./Outfit and quietly hand the path // or alias to the harness. if f.path == "" && len(rest) > 0 && namesAnOutfitOrAlias(rest[0]) { - return fmt.Errorf("--outfit takes its path attached: --outfit=%s (a bare --outfit applies ./%s)", rest[0], outfit.DefaultFile) + return "", fmt.Errorf("--outfit takes its path attached: --outfit=%s (a bare --outfit applies ./%s)", rest[0], outfit.DefaultFile) } sel, path, err := readOutfit("outfit harness --outfit=", f.path) if err != nil { - return err + return "", err } // As for apply, --providers overrides the catalogue the selection resolves // against (an Outfit never names one). sel.Providers = providers fmt.Printf("Applying %s\n\n", path) - if err := applySelection(sel, h); err != nil { - return err + envDir := filepath.Dir(path) + if err := applySelection(sel, h, envDir); err != nil { + return "", err } fmt.Println() - return nil + return envDir, nil } // namesAnOutfit reports whether arg points at an Outfit file, or at a directory diff --git a/cmd/outfit/main_test.go b/cmd/outfit/main_test.go index f4c5dc7d..4dbfdf30 100644 --- a/cmd/outfit/main_test.go +++ b/cmd/outfit/main_test.go @@ -33,6 +33,25 @@ func captureStdout(t *testing.T, fn func()) string { return buf.String() } +// captureStderr runs fn with os.Stderr redirected and returns what it wrote. +func captureStderr(t *testing.T, fn func()) string { + t.Helper() + old := os.Stderr + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stderr = w + defer func() { os.Stderr = old }() + + fn() + w.Close() + + var buf bytes.Buffer + io.Copy(&buf, r) + return buf.String() +} + // readConfigMap reads a config file, standardises the JSONC, and unmarshals it // into a map for assertions. func readConfigMap(t *testing.T, path string) map[string]any { diff --git a/cmd/outfit/remote.go b/cmd/outfit/remote.go new file mode 100644 index 00000000..ccd0fefb --- /dev/null +++ b/cmd/outfit/remote.go @@ -0,0 +1,439 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/lucinate-ai/outfit/internal/contextsize" + "github.com/lucinate-ai/outfit/internal/outfit" + "github.com/lucinate-ai/outfit/internal/preset" + "github.com/lucinate-ai/outfit/internal/remote" +) + +// cmdRemote dispatches the remote subcommands, which control the +// scale-to-zero GPU inference instance defined in this repo's remote/: +// start boots it and prints the endpoint exports, stop shuts it down +// immediately (its stop Lambda also runs on a schedule to auto-stop on +// idle), status reports instance state and endpoint health, and deploy sets +// what the instance will serve from the Outfit itself. Each subcommand takes +// an optional Outfit path; see resolveRemoteConfig for how the remote config +// is found. +func cmdRemote(args []string) error { + if len(args) == 0 { + return fmt.Errorf("usage: outfit remote [path]") + } + sub, rest := args[0], args[1:] + switch sub { + case "start": + return cmdRemoteStart(rest) + case "stop": + return cmdRemoteStop(rest) + case "status": + return cmdRemoteStatus(rest) + case "deploy": + return cmdRemoteDeploy(rest) + default: + return fmt.Errorf( + "unknown remote subcommand %q (expected start, stop, status or deploy)", sub) + } +} + +// resolveRemoteConfig loads the remote config, preferring an Outfit's REMOTE +// instruction over the per-user file. An explicit [path] argument must name +// an Outfit (or a directory holding one) with a REMOTE instruction. With no +// argument, ./Outfit is consulted when present; the per-user config +// (~/.config/outfit/remote.json) is the fallback, so `outfit remote` still +// works outside any project. A relative REMOTE resolves against the Outfit's +// directory, so an Outfit and its remote config travel together — the same +// rule PRESET uses. +func resolveRemoteConfig(outfitArg string) (remote.Config, error) { + if outfitArg != "" { + sel, outfitPath, err := readOutfit("remote", outfitArg) + if err != nil { + return remote.Config{}, err + } + if sel.Remote == "" { + return remote.Config{}, fmt.Errorf("%s has no REMOTE instruction", outfitPath) + } + return remote.LoadConfigFile(remoteConfigPath(sel.Remote, outfitPath), os.Getenv) + } + if defaultOutfitExists() { + sel, outfitPath, err := readOutfit("remote", "") + if err != nil { + return remote.Config{}, err + } + if sel.Remote != "" { + return remote.LoadConfigFile(remoteConfigPath(sel.Remote, outfitPath), os.Getenv) + } + } + return remote.LoadConfig(os.Getenv) +} + +// defaultOutfitExists reports whether the working directory holds a file +// named exactly "Outfit". A plain os.Stat would do, except that on +// case-insensitive filesystems (macOS, Windows) it also matches a file named +// "outfit" — such as the binary `make build` drops in this repo's root — so +// the directory listing is checked for the exact name instead. +func defaultOutfitExists() bool { + entries, err := os.ReadDir(".") + if err != nil { + return false + } + for _, entry := range entries { + if entry.Name() == outfit.DefaultFile && !entry.IsDir() { + return true + } + } + return false +} + +func remoteConfigPath(remoteValue, outfitPath string) string { + if filepath.IsAbs(remoteValue) { + return remoteValue + } + return filepath.Join(filepath.Dir(outfitPath), remoteValue) +} + +// outfitArg returns the optional positional Outfit path after the flags. +func outfitArg(fs *flag.FlagSet) string { + if rest := fs.Args(); len(rest) > 0 { + return rest[0] + } + return "" +} + +// heartbeatEvery is how often a start that is still waiting says so. The start +// endpoint blocks until the model is serving, which on a cold start is minutes, +// so without this the command looks hung. +const heartbeatEvery = 30 * time.Second + +// startProgress reports what a slow start is doing. Everything it writes goes +// to stderr, so `outfit remote start | grep '^export '` still yields just the +// exports while the user watching the terminal still sees progress. +type startProgress struct { + mu sync.Mutex + since time.Time + done chan struct{} + stop sync.Once +} + +func newStartProgress(every time.Duration) *startProgress { + p := &startProgress{since: time.Now(), done: make(chan struct{})} + go func() { + ticker := time.NewTicker(every) + defer ticker.Stop() + for { + select { + case <-p.done: + return + case <-ticker.C: + p.line(fmt.Sprintf("still starting (%s elapsed)", p.elapsed())) + } + } + }() + return p +} + +func (p *startProgress) elapsed() time.Duration { + return time.Since(p.since).Round(time.Second) +} + +// line prints one progress line. Serialised, so a heartbeat landing at the same +// moment as a retry notice cannot interleave with it. +func (p *startProgress) line(msg string) { + p.mu.Lock() + defer p.mu.Unlock() + fmt.Fprintln(os.Stderr, msg) +} + +func (p *startProgress) close() { + p.stop.Do(func() { close(p.done) }) +} + +func cmdRemoteStart(args []string) error { + fs := flag.NewFlagSet("remote start", flag.ContinueOnError) + var timeout time.Duration + fs.DurationVar(&timeout, "timeout", 15*time.Minute, "overall time to wait for the endpoint") + if err := fs.Parse(args); err != nil { + return err + } + cfg, err := resolveRemoteConfig(outfitArg(fs)) + if err != nil { + return err + } + + progress := newStartProgress(heartbeatEvery) + defer progress.close() + progress.line(fmt.Sprintf( + "Starting the endpoint. It boots, fetches the weights, and loads them into the GPU,\nwhich takes several minutes from cold; waiting up to %s.", timeout)) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + resp, err := remote.Start(ctx, cfg, progress.line) + if err != nil { + return err + } + progress.close() + progress.line(fmt.Sprintf("ready after %s", progress.elapsed())) + + // stdout carries only the result, so `eval "$(outfit remote start)"` works. + fmt.Printf("export OPENAI_BASE_URL=%s\n", resp.BaseURL) + fmt.Printf("export OPENAI_API_KEY=%s\n", resp.APIKey) + return nil +} + +func cmdRemoteStop(args []string) error { + fs := flag.NewFlagSet("remote stop", flag.ContinueOnError) + if err := fs.Parse(args); err != nil { + return err + } + cfg, err := resolveRemoteConfig(outfitArg(fs)) + if err != nil { + return err + } + resp, err := remote.Stop(context.Background(), cfg) + if err != nil { + return err + } + fmt.Printf("state: %s\n", resp.State) + return nil +} + +func cmdRemoteStatus(args []string) error { + fs := flag.NewFlagSet("remote status", flag.ContinueOnError) + if err := fs.Parse(args); err != nil { + return err + } + cfg, err := resolveRemoteConfig(outfitArg(fs)) + if err != nil { + return err + } + resp, err := remote.Status(context.Background(), cfg) + if err != nil { + return err + } + fmt.Printf("state: %s\n", resp.State) + if resp.Healthy != nil { + fmt.Printf("healthy: %t\n", *resp.Healthy) + } + if resp.BaseURL != "" { + fmt.Printf("base_url: %s\n", resp.BaseURL) + } + return nil +} + +// runnerFor maps an Outfit's PROVIDER to the inference runner the cloud should +// run. PROVIDER already names the engine — `outfit serve` starts that engine +// locally, `outfit remote deploy` asks the cloud for the same one — so no +// separate keyword is needed. Providers that are not self-hosted engines have +// nothing to deploy. +func runnerFor(provider string) (string, error) { + switch provider { + case "llamacpp", "vllm": + return provider, nil + default: + return "", fmt.Errorf( + "PROVIDER %q cannot be deployed: remote deploy runs a self-hosted engine, so use llamacpp or vllm", + provider) + } +} + +// splitModelQuant splits a model reference into the Hugging Face repo and an +// optional quant tag, as used by llama.cpp's -hf (org/model:QUANT). Repo ids +// cannot contain a colon, so the first one separates them. +func splitModelQuant(model string) (repo, quant string) { + if i := strings.Index(model, ":"); i >= 0 { + return model[:i], model[i+1:] + } + return model, "" +} + +// cloudOwnedFlags are the llama-server settings the cloud sets itself, from the +// deploy config and the instance's own environment. They are dropped from a +// preset before it becomes serveArgs, so a preset written for a local run does +// not fight the deployment — binding to 127.0.0.1, say, or serving from a local +// .gguf path or an HF repo that the instance does not use (it syncs the weights +// from S3 instead). Keyed by canonical name, so short aliases match too. +var cloudOwnedFlags = map[string]bool{ + "host": true, "port": true, + "model": true, "model-url": true, + "hf-repo": true, "hf-file": true, "hf-token": true, + "api-key": true, "api-key-file": true, + "ctx-size": true, "alias": true, "metrics": true, +} + +// isCloudOwned reports whether the cloud sets this preset key itself. +func isCloudOwned(key string) bool { + return cloudOwnedFlags[preset.CanonicalKey(key)] +} + +// dropCloudOwned returns the params the deployment should keep. +func dropCloudOwned(params []preset.Param) []preset.Param { + var kept []preset.Param + for _, p := range params { + if !isCloudOwned(p.Key) { + kept = append(kept, p) + } + } + return kept +} + +// deployConfigFor turns an Outfit (plus its preset, if any) into the config the +// deploy Lambda accepts. The Outfit is the single source of truth: PROVIDER +// picks the runner, MODEL or the preset's hf names the weights, CONTEXT the +// window, ALIAS the served name, and whatever else the preset sets becomes the +// runner's own flags. +func deployConfigFor(sel outfit.Selection, outfitPath string) (remote.DeployConfig, error) { + var dc remote.DeployConfig + + runner, err := runnerFor(sel.Provider) + if err != nil { + return dc, err + } + dc.Runner = runner + + // The preset supplies the model and the runner flags when the Outfit does + // not state them, so a single preset can drive both serve and deploy. The + // [*] globals are a separate layer that the chosen section overrides — + // exactly as preset.Args does for a local serve — so settings written there + // (commonly ngl and jinja) are not lost. + var global, params []preset.Param + if sel.Preset != "" { + presetPath := resolvePresetPath(sel.Preset, outfitPath) + data, err := os.ReadFile(presetPath) + if err != nil { + return dc, fmt.Errorf("reading preset %s: %w", presetPath, err) + } + pre, err := preset.Parse(data) + if err != nil { + return dc, fmt.Errorf("%s: %w", presetPath, err) + } + sec, err := pre.Select(sel.Alias) + if err != nil { + return dc, fmt.Errorf("%s: %w", presetPath, err) + } + global, params = pre.Global, sec.Params + } + + model := sel.Model + if model == "" { + model = presetValue("hf", global, params) + } + if model == "" { + return dc, fmt.Errorf( + "nothing to deploy: set MODEL (an HF repo like org/model:QUANT) in %s, or hf in its preset", + outfitPath) + } + if isModelPath(model) { + return dc, fmt.Errorf( + "cannot deploy the local model file %q: the cloud downloads weights from Hugging Face, so name a repo (org/model:QUANT)", + model) + } + dc.ModelID, dc.Quant = splitModelQuant(model) + + context := sel.Context + if context == "" { + context = presetValue("ctx-size", global, params) + } + if context == "" { + return dc, fmt.Errorf("no context size: set CONTEXT in %s, or ctx-size in its preset", outfitPath) + } + n, err := contextsize.Parse(context) + if err != nil { + return dc, err + } + dc.ContextSize = n + + // The served name is what a coding agent asks for. ALIAS is the friendly + // name; without one the repo id is served under its own name. + dc.ServedModelName = sel.Alias + if dc.ServedModelName == "" { + dc.ServedModelName = dc.ModelID + } + + dc.ServeArgs = preset.Flags(dropCloudOwned(global), dropCloudOwned(params)) + if dc.ServeArgs == nil { + dc.ServeArgs = []string{} + } + return dc, nil +} + +// presetValue returns a preset param's value across layers, or "" when it is +// not set. Later layers win, matching preset.Flags — so a section overrides the +// [*] globals. +func presetValue(key string, layers ...[]preset.Param) string { + want := preset.CanonicalKey(key) + value := "" + for _, layer := range layers { + for _, p := range layer { + if preset.CanonicalKey(p.Key) == want { + value = p.Value + } + } + } + return value +} + +func cmdRemoteDeploy(args []string) error { + fs := flag.NewFlagSet("remote deploy", flag.ContinueOnError) + var dryRun bool + fs.BoolVar(&dryRun, "dry-run", false, "print the config that would be deployed, without sending it") + fs.BoolVar(&dryRun, "n", false, "print the config without sending it (shorthand)") + if err := fs.Parse(args); err != nil { + return err + } + + // deploy reads the Outfit for what to serve, so unlike the other + // subcommands it always needs one — the per-user remote config alone is not + // enough. + sel, outfitPath, err := readOutfit("outfit remote deploy ", outfitArg(fs)) + if err != nil { + return err + } + dc, err := deployConfigFor(sel, outfitPath) + if err != nil { + return err + } + + fmt.Printf("Deploying from %s\n", outfitPath) + fmt.Printf(" runner: %s\n", dc.Runner) + fmt.Printf(" model: %s", dc.ModelID) + if dc.Quant != "" { + fmt.Printf(" (%s)", dc.Quant) + } + fmt.Println() + fmt.Printf(" context: %d\n", dc.ContextSize) + fmt.Printf(" served: %s\n", dc.ServedModelName) + if len(dc.ServeArgs) > 0 { + fmt.Printf(" args: %s\n", strings.Join(dc.ServeArgs, " ")) + } + if dryRun { + return nil + } + + cfg, err := resolveRemoteConfig(outfitArg(fs)) + if err != nil { + return err + } + resp, err := remote.Deploy(context.Background(), cfg, dc) + if err != nil { + return err + } + + fmt.Println() + fmt.Println("deployed") + if resp.Seeding { + fmt.Printf("seeding the weights on %s — this takes ~15-20 min.\n", resp.SeedInstanceID) + fmt.Println("Wait for it to finish before `outfit remote start`, or the instance will") + fmt.Println("start against an incomplete download.") + } else { + fmt.Println("weights already in place — `outfit remote start` will serve this.") + } + return nil +} diff --git a/cmd/outfit/remote_deploy_test.go b/cmd/outfit/remote_deploy_test.go new file mode 100644 index 00000000..30ef9e45 --- /dev/null +++ b/cmd/outfit/remote_deploy_test.go @@ -0,0 +1,410 @@ +package main + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/lucinate-ai/outfit/internal/remote" +) + +// writeDeployOutfit writes an Outfit (and optionally a preset) into a temp dir +// and returns the Outfit's path. +func writeDeployOutfit(t *testing.T, outfitBody, presetBody string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "Outfit") + if err := os.WriteFile(path, []byte(outfitBody), 0o600); err != nil { + t.Fatal(err) + } + if presetBody != "" { + if err := os.WriteFile(filepath.Join(dir, "preset.ini"), []byte(presetBody), 0o600); err != nil { + t.Fatal(err) + } + } + return path +} + +const qwenPreset = `[*] +host = 127.0.0.1 +port = 8080 +jinja = 1 +ngl = 99 + +[qwen3.6-27b] +hf = unsloth/Qwen3.6-27B-MTP-GGUF:UD-Q6_K_XL +ctx-size = 131072 +fa = on +spec-type = draft-mtp +spec-draft-n-max = 2 +` + +func deployConfigFrom(t *testing.T, outfitBody, presetBody string) remote.DeployConfig { + t.Helper() + path := writeDeployOutfit(t, outfitBody, presetBody) + sel, _, err := readOutfit("test", path) + if err != nil { + t.Fatal(err) + } + dc, err := deployConfigFor(sel, path) + if err != nil { + t.Fatalf("deployConfigFor: %v", err) + } + return dc +} + +func TestDeployConfigFor_FromPreset(t *testing.T) { + dc := deployConfigFrom(t, + "PROVIDER llamacpp\nALIAS qwen3.6-27b\nCONTEXT 131072\nPRESET ./preset.ini\n", qwenPreset) + + if dc.Runner != "llamacpp" { + t.Errorf("runner = %q, want llamacpp", dc.Runner) + } + if dc.ModelID != "unsloth/Qwen3.6-27B-MTP-GGUF" || dc.Quant != "UD-Q6_K_XL" { + t.Errorf("model = %q quant = %q, want the repo and quant split", dc.ModelID, dc.Quant) + } + if dc.ContextSize != 131072 { + t.Errorf("contextSize = %d, want 131072", dc.ContextSize) + } + if dc.ServedModelName != "qwen3.6-27b" { + t.Errorf("servedModelName = %q, want the ALIAS", dc.ServedModelName) + } + + args := strings.Join(dc.ServeArgs, " ") + // From the [*] globals: losing these would serve on CPU, and without tool + // calling, which is the whole point of the deployment. + for _, want := range []string{"--jinja", "--n-gpu-layers 99"} { + if !strings.Contains(args, want) { + t.Errorf("serveArgs %q is missing the [*] global %q", args, want) + } + } + // From the section. + for _, want := range []string{"--flash-attn on", "--spec-type draft-mtp", "--spec-draft-n-max 2"} { + if !strings.Contains(args, want) { + t.Errorf("serveArgs %q is missing %q", args, want) + } + } + // Set by the cloud, so they must not be echoed back. + for _, unwanted := range []string{"--host", "--port", "--ctx-size", "--alias", "--hf-repo", "127.0.0.1", "8080"} { + if strings.Contains(args, unwanted) { + t.Errorf("serveArgs %q should not carry the cloud-owned %q", args, unwanted) + } + } +} + +func TestDeployConfigFor_ContextAndModelFallBackToPreset(t *testing.T) { + // No CONTEXT and no MODEL in the Outfit: both come from the preset, so one + // preset can drive a local serve and a deploy. + dc := deployConfigFrom(t, "PROVIDER llamacpp\nALIAS qwen3.6-27b\nPRESET ./preset.ini\n", qwenPreset) + if dc.ContextSize != 131072 { + t.Errorf("contextSize = %d, want the preset's ctx-size", dc.ContextSize) + } + if dc.ModelID != "unsloth/Qwen3.6-27B-MTP-GGUF" { + t.Errorf("modelId = %q, want the preset's hf", dc.ModelID) + } +} + +func TestDeployConfigFor_OutfitOverridesPreset(t *testing.T) { + dc := deployConfigFrom(t, + "PROVIDER llamacpp\nALIAS qwen3.6-27b\nMODEL unsloth/Other-GGUF:Q4_K_M\nCONTEXT 32768\nPRESET ./preset.ini\n", + qwenPreset) + if dc.ModelID != "unsloth/Other-GGUF" || dc.Quant != "Q4_K_M" { + t.Errorf("MODEL should win over the preset's hf, got %q:%q", dc.ModelID, dc.Quant) + } + if dc.ContextSize != 32768 { + t.Errorf("CONTEXT should win over the preset's ctx-size, got %d", dc.ContextSize) + } +} + +func TestDeployConfigFor_SectionOverridesGlobals(t *testing.T) { + preset := "[*]\nngl = 10\n\n[m]\nhf = org/model:Q4\nctx-size = 4096\nngl = 99\n" + dc := deployConfigFrom(t, "PROVIDER llamacpp\nALIAS m\nPRESET ./preset.ini\n", preset) + args := strings.Join(dc.ServeArgs, " ") + if !strings.Contains(args, "--n-gpu-layers 99") || strings.Contains(args, "--n-gpu-layers 10") { + t.Errorf("the section should override the [*] global, got %q", args) + } +} + +func TestDeployConfigFor_VllmNeedsNoPreset(t *testing.T) { + dc := deployConfigFrom(t, "PROVIDER vllm\nMODEL Qwen/Qwen3.6-27B-FP8\nCONTEXT 32k\n", "") + if dc.Runner != "vllm" { + t.Errorf("runner = %q, want vllm", dc.Runner) + } + if dc.Quant != "" { + t.Errorf("quant = %q, want empty for a safetensors repo", dc.Quant) + } + if dc.ServedModelName != "Qwen/Qwen3.6-27B-FP8" { + t.Errorf("servedModelName = %q, want the model id when there is no ALIAS", dc.ServedModelName) + } + if dc.ServeArgs == nil { + t.Error("serveArgs should marshal as [] rather than null") + } +} + +func TestDeployConfigFor_Rejects(t *testing.T) { + cases := []struct { + name, outfitBody, presetBody, want string + }{ + { + name: "a provider that is not a self-hosted engine", + outfitBody: "PROVIDER openai-compatible\nMODEL org/m\nCONTEXT 32k\n", + want: "cannot be deployed", + }, + { + name: "a local gguf path", + outfitBody: "PROVIDER llamacpp\nMODEL ./models/qwen.gguf\nCONTEXT 32k\n", + want: "cannot deploy the local model file", + }, + { + name: "no model anywhere", + outfitBody: "PROVIDER llamacpp\nCONTEXT 32k\n", + want: "nothing to deploy", + }, + { + name: "no context anywhere", + outfitBody: "PROVIDER llamacpp\nMODEL org/m:Q4\n", + want: "no context size", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + path := writeDeployOutfit(t, tc.outfitBody, tc.presetBody) + sel, _, err := readOutfit("test", path) + if err != nil { + t.Fatal(err) + } + _, err = deployConfigFor(sel, path) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("want an error containing %q, got %v", tc.want, err) + } + }) + } +} + +func TestSplitModelQuant(t *testing.T) { + for _, tc := range []struct{ in, repo, quant string }{ + {"org/model:Q4_K_M", "org/model", "Q4_K_M"}, + {"org/model", "org/model", ""}, + {"unsloth/Qwen3.6-27B-MTP-GGUF:UD-Q6_K_XL", "unsloth/Qwen3.6-27B-MTP-GGUF", "UD-Q6_K_XL"}, + } { + repo, quant := splitModelQuant(tc.in) + if repo != tc.repo || quant != tc.quant { + t.Errorf("splitModelQuant(%q) = %q,%q; want %q,%q", tc.in, repo, quant, tc.repo, tc.quant) + } + } +} + +func TestRemoteDeploy_PostsTheConfig(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + + var got remote.DeployConfig + var gotMethod, gotAuth string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotMethod = r.Method + gotAuth = r.Header.Get("Authorization") + body, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(body, &got) + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"deployed":true,"seeding":true,"seedInstanceId":"i-123"}`)) + })) + defer server.Close() + + dir := t.TempDir() + t.Chdir(dir) + if err := os.WriteFile("Outfit", []byte( + "PROVIDER llamacpp\nALIAS qwen3.6-27b\nCONTEXT 131072\nPRESET ./preset.ini\nREMOTE ./remote.json\n", + ), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile("preset.ini", []byte(qwenPreset), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := json.Marshal(remote.Config{ + StartURL: server.URL, + StopURL: server.URL, + DeployURL: server.URL, + Region: "us-east-1", + }) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile("remote.json", cfg, 0o600); err != nil { + t.Fatal(err) + } + + out := captureStdout(t, func() { + if err := cmdRemoteDeploy(nil); err != nil { + t.Errorf("cmdRemoteDeploy: %v", err) + } + }) + + if gotMethod != http.MethodPost { + t.Errorf("method = %q, want POST", gotMethod) + } + // A body-carrying request must still be signed, with the payload hash. + if !strings.Contains(gotAuth, "AWS4-HMAC-SHA256") { + t.Errorf("request was not SigV4-signed: %q", gotAuth) + } + if got.Runner != "llamacpp" || got.ModelID != "unsloth/Qwen3.6-27B-MTP-GGUF" || got.ContextSize != 131072 { + t.Errorf("posted config is wrong: %+v", got) + } + // The Lambda derives the prefix, so the body must not pin one. + if strings.Contains(strings.ToLower(out), "weightsprefix") { + t.Errorf("deploy should not mention a weights prefix, got:\n%s", out) + } + if !strings.Contains(out, "seeding the weights on i-123") { + t.Errorf("seeding should be reported with the instance id, got:\n%s", out) + } +} + +func TestRemoteDeploy_NeedsDeployURL(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + dir := t.TempDir() + t.Chdir(dir) + if err := os.WriteFile("Outfit", []byte( + "PROVIDER llamacpp\nMODEL org/m:Q4\nCONTEXT 32k\nREMOTE ./remote.json\n", + ), 0o600); err != nil { + t.Fatal(err) + } + // A config written before `remote deploy` existed: start/stop still work, so + // only deploy should complain, and it should say what to add. + cfg, _ := json.Marshal(remote.Config{StartURL: "https://x.lambda-url.us-east-1.on.aws/", StopURL: "https://y.lambda-url.us-east-1.on.aws/", Region: "us-east-1"}) + if err := os.WriteFile("remote.json", cfg, 0o600); err != nil { + t.Fatal(err) + } + err := cmdRemoteDeploy(nil) + if err == nil || !strings.Contains(err.Error(), "deploy_url") { + t.Errorf("want an error naming deploy_url, got %v", err) + } +} + +func TestRemoteDeploy_DryRunSendsNothing(t *testing.T) { + isolateConfig(t) + called := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called = true + })) + defer server.Close() + + dir := t.TempDir() + t.Chdir(dir) + if err := os.WriteFile("Outfit", []byte( + "PROVIDER llamacpp\nALIAS qwen3.6-27b\nPRESET ./preset.ini\nREMOTE ./remote.json\n", + ), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile("preset.ini", []byte(qwenPreset), 0o600); err != nil { + t.Fatal(err) + } + cfg, _ := json.Marshal(remote.Config{StartURL: server.URL, StopURL: server.URL, DeployURL: server.URL, Region: "us-east-1"}) + if err := os.WriteFile("remote.json", cfg, 0o600); err != nil { + t.Fatal(err) + } + + out := captureStdout(t, func() { + if err := cmdRemoteDeploy([]string{"--dry-run"}); err != nil { + t.Errorf("cmdRemoteDeploy --dry-run: %v", err) + } + }) + if called { + t.Error("--dry-run must not call the deploy Lambda") + } + if !strings.Contains(out, "unsloth/Qwen3.6-27B-MTP-GGUF") { + t.Errorf("--dry-run should print the config, got:\n%s", out) + } +} + +// Guard the assumption deployConfigFor relies on: PROVIDER names the engine. +func TestRunnerFor(t *testing.T) { + for _, provider := range []string{"llamacpp", "vllm"} { + if got, err := runnerFor(provider); err != nil || got != provider { + t.Errorf("runnerFor(%q) = %q, %v", provider, got, err) + } + } + for _, provider := range []string{"openai-compatible", "ollama", "openrouter", ""} { + if _, err := runnerFor(provider); err == nil { + t.Errorf("runnerFor(%q) should error", provider) + } + } +} + +// A cold start blocks in one request for minutes, so the command must say what +// it is doing rather than sit silent — and must say it on stderr, so piping the +// exports still works. +func TestRemoteStart_ReportsProgressWhileWaiting(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(120 * time.Millisecond) // stand in for a cold start + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"state":"ready","base_url":"http://198.51.100.1:8000/v1","api_key":"sk-test"}`)) + })) + defer server.Close() + writeRemoteConfig(t, server.URL) + + stderr := captureStderr(t, func() { + if err := cmdRemoteStart(nil); err != nil { + t.Errorf("cmdRemoteStart: %v", err) + } + }) + + if !strings.Contains(stderr, "Starting the endpoint") { + t.Errorf("no opening explanation on stderr, got:\n%s", stderr) + } + if !strings.Contains(stderr, "ready after") { + t.Errorf("no completion line on stderr, got:\n%s", stderr) + } +} + +func TestStartProgress_HeartbeatsAndStops(t *testing.T) { + stderr := captureStderr(t, func() { + p := newStartProgress(10 * time.Millisecond) + time.Sleep(60 * time.Millisecond) + p.close() + time.Sleep(40 * time.Millisecond) + }) + if !strings.Contains(stderr, "still starting") { + t.Errorf("expected a heartbeat while waiting, got:\n%s", stderr) + } + // Closing must stop it: 40ms of 10ms ticks after close would add ~4 more. + if got := strings.Count(stderr, "still starting"); got > 8 { + t.Errorf("heartbeat kept running after close (%d lines):\n%s", got, stderr) + } +} + +// Progress must not land on stdout, or `outfit remote start | grep '^export '` +// would pick it up. +func TestRemoteStart_StdoutCarriesOnlyTheResult(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"state":"ready","base_url":"http://198.51.100.1:8000/v1","api_key":"sk-test"}`)) + })) + defer server.Close() + writeRemoteConfig(t, server.URL) + + out := captureStdout(t, func() { + if err := cmdRemoteStart(nil); err != nil { + t.Errorf("cmdRemoteStart: %v", err) + } + }) + lines := strings.Split(strings.TrimSpace(out), "\n") + if len(lines) != 2 { + t.Fatalf("stdout should be exactly the two exports, got:\n%s", out) + } + for i, want := range []string{"export OPENAI_BASE_URL=", "export OPENAI_API_KEY="} { + if !strings.HasPrefix(lines[i], want) { + t.Errorf("stdout line %d = %q, want it to start with %q", i, lines[i], want) + } + } +} diff --git a/cmd/outfit/remote_test.go b/cmd/outfit/remote_test.go new file mode 100644 index 00000000..cd2a0a10 --- /dev/null +++ b/cmd/outfit/remote_test.go @@ -0,0 +1,222 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/lucinate-ai/outfit/internal/remote" +) + +// stubAWSEnv pins the default credential chain to static env credentials so +// SigV4 signing works offline without touching a real profile or IMDS. +func stubAWSEnv(t *testing.T) { + t.Helper() + t.Setenv("AWS_ACCESS_KEY_ID", "AKIATESTTESTTESTTEST") + t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret") + t.Setenv("AWS_SESSION_TOKEN", "") + t.Setenv("AWS_PROFILE", "") + t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "no-such-file")) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "no-such-file")) + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") +} + +// writeRemoteConfig stores a remote config pointing at the test server. +func writeRemoteConfig(t *testing.T, serverURL string) { + t.Helper() + path := remote.ConfigPath() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + data, err := json.Marshal(remote.Config{StartURL: serverURL, StopURL: serverURL, Region: "eu-west-1"}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } +} + +func TestRemoteDispatch(t *testing.T) { + if err := run([]string{"remote"}); err == nil || !strings.Contains(err.Error(), "usage") { + t.Errorf("bare remote should error with usage, got %v", err) + } + if err := run([]string{"remote", "bogus"}); err == nil || !strings.Contains(err.Error(), "bogus") { + t.Errorf("unknown subcommand should error, got %v", err) + } +} + +func TestRemote_Unconfigured(t *testing.T) { + isolateConfig(t) + for _, sub := range []string{"start", "stop", "status"} { // deploy needs an Outfit, covered separately + if err := run([]string{"remote", sub}); err == nil || !strings.Contains(err.Error(), "not configured") { + t.Errorf("remote %s without config should explain setup, got %v", sub, err) + } + } +} + +func TestRemoteStart_PrintsExports(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"state":"ready","base_url":"http://198.51.100.1:8000/v1","api_key":"sk-test"}`)) + })) + defer server.Close() + writeRemoteConfig(t, server.URL) + + out := captureStdout(t, func() { + if err := cmdRemoteStart(nil); err != nil { + t.Errorf("cmdRemoteStart: %v", err) + } + }) + if !strings.Contains(out, "export OPENAI_BASE_URL=http://198.51.100.1:8000/v1") || + !strings.Contains(out, "export OPENAI_API_KEY=sk-test") { + t.Errorf("start should print the endpoint exports, got:\n%s", out) + } +} + +func TestRemoteStatus_PrintsState(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"state":"running","healthy":true,"base_url":"http://198.51.100.1:8000/v1"}`)) + })) + defer server.Close() + writeRemoteConfig(t, server.URL) + + out := captureStdout(t, func() { + if err := cmdRemoteStatus(nil); err != nil { + t.Errorf("cmdRemoteStatus: %v", err) + } + }) + for _, want := range []string{"state: running", "healthy: true", "base_url: http://198.51.100.1:8000/v1"} { + if !strings.Contains(out, want) { + t.Errorf("status output missing %q:\n%s", want, out) + } + } +} + +func TestRemoteStop_PrintsState(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("stop should POST, got %s", r.Method) + } + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"state":"stopping"}`)) + })) + defer server.Close() + writeRemoteConfig(t, server.URL) + + out := captureStdout(t, func() { + if err := cmdRemoteStop(nil); err != nil { + t.Errorf("cmdRemoteStop: %v", err) + } + }) + if !strings.Contains(out, "state: stopping") { + t.Errorf("stop should print the state, got:\n%s", out) + } +} + +func TestRemote_OutfitDiscovery(t *testing.T) { + isolateConfig(t) // no per-user config exists, so success proves discovery + stubAWSEnv(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"state":"running","healthy":true,"base_url":"http://198.51.100.1:8000/v1"}`)) + })) + defer server.Close() + + dir := t.TempDir() + t.Chdir(dir) + outfitFile := "PROVIDER openai-compatible\nREMOTE remote.json\n" + if err := os.WriteFile("Outfit", []byte(outfitFile), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := json.Marshal(remote.Config{StartURL: server.URL, StopURL: server.URL, Region: "eu-west-1"}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile("remote.json", cfg, 0o600); err != nil { + t.Fatal(err) + } + + out := captureStdout(t, func() { + if err := cmdRemoteStatus(nil); err != nil { + t.Errorf("cmdRemoteStatus: %v", err) + } + }) + if !strings.Contains(out, "state: running") { + t.Errorf("status via Outfit REMOTE should work, got:\n%s", out) + } +} + +func TestRemote_ExplicitOutfitNeedsRemote(t *testing.T) { + isolateConfig(t) + t.Chdir(t.TempDir()) + if err := os.WriteFile("Outfit", []byte("PROVIDER ollama\n"), 0o600); err != nil { + t.Fatal(err) + } + err := cmdRemoteStatus([]string{"Outfit"}) + if err == nil || !strings.Contains(err.Error(), "no REMOTE") { + t.Errorf("explicit Outfit without REMOTE should error, got %v", err) + } +} + +func TestRemote_OutfitWithoutRemoteFallsBack(t *testing.T) { + isolateConfig(t) + stubAWSEnv(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"state":"stopped","healthy":false,"base_url":"http://198.51.100.1:8000/v1"}`)) + })) + defer server.Close() + writeRemoteConfig(t, server.URL) + + t.Chdir(t.TempDir()) + if err := os.WriteFile("Outfit", []byte("PROVIDER ollama\n"), 0o600); err != nil { + t.Fatal(err) + } + out := captureStdout(t, func() { + if err := cmdRemoteStatus(nil); err != nil { + t.Errorf("cmdRemoteStatus: %v", err) + } + }) + if !strings.Contains(out, "state: stopped") { + t.Errorf("an Outfit without REMOTE should fall back to the user config, got:\n%s", out) + } +} + +func TestRemote_IgnoresLowercaseOutfitFile(t *testing.T) { + // On case-insensitive filesystems a stat of "Outfit" matches a file named + // "outfit" (e.g. the built binary in this repo's root); discovery must not + // try to parse it. + isolateConfig(t) + stubAWSEnv(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"state":"stopped","healthy":false,"base_url":"http://198.51.100.1:8000/v1"}`)) + })) + defer server.Close() + writeRemoteConfig(t, server.URL) + + t.Chdir(t.TempDir()) + if err := os.WriteFile("outfit", []byte{0xcf, 0xfa, 0xed, 0xfe}, 0o755); err != nil { + t.Fatal(err) + } + out := captureStdout(t, func() { + if err := cmdRemoteStatus(nil); err != nil { + t.Errorf("cmdRemoteStatus: %v", err) + } + }) + if !strings.Contains(out, "state: stopped") { + t.Errorf("a lowercase outfit file should not shadow discovery, got:\n%s", out) + } +} diff --git a/docs/README.md b/docs/README.md index d486042e..0c2f0405 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,7 +15,7 @@ Four words carry the whole tool: Pi is also supported. Chosen at runtime, so the same selection works for either. See [`outfit harness`](commands/harness.md). - **Provider and family** — what `outfit` can configure, from a built-in - catalogue: OpenRouter, AWS Bedrock, Ollama, llama.cpp, or any + catalogue: OpenRouter, AWS Bedrock, Ollama, llama.cpp, vLLM, or any OpenAI-compatible endpoint, each with named model families. See [`outfit list`](commands/list.md). - **Outfit file** — a small, declarative file (like a `Dockerfile`, but for @@ -28,7 +28,11 @@ Four words carry the whole tool: - [Getting started](getting-started.md) — the end-to-end flow - [The `Outfit` file](outfit-file.md) — syntax and examples +- [Running on a cloud GPU](commands/remote.md) — the same Outfit, on a + machine that stops when you do - [Runnable examples](../examples/) — ready-to-apply Outfits with walkthroughs +- [Deploying your own cloud GPU endpoint](../remote/) — the AWS project behind + `outfit remote` ## Commands @@ -43,6 +47,7 @@ Four words carry the whole tool: | [`outfit alias`](commands/alias.md) | Name an `Outfit` so the name works anywhere a path does | | [`outfit unalias`](commands/unalias.md) | Drop a registered name | | [`outfit serve`](commands/serve.md) | Run `llama-server` for the model an `Outfit` names | +| [`outfit remote`](commands/remote.md) | Run the model on a cloud GPU that stops when you do | | [`outfit export`](commands/export.md) | Capture the current setup as an `Outfit` | | [`outfit harness`](commands/harness.md) | Launch the agent, optionally dressing it first | | [`outfit init-providers`](commands/init-providers.md) | Write the catalogue out to customise | @@ -61,6 +66,11 @@ Four words carry the whole tool: | `OLLAMA_BASE_URL`, `LLAMACPP_BASE_URL`, `OPENAI_BASE_URL` | Per-provider endpoint overrides | | `AWS_REGION` | Region for AWS Bedrock | -Keys are looked up in a `.env` file next to the tool first, then your shell -environment. Local providers (Ollama, llama.cpp) need no key; Bedrock uses -your AWS credentials. +Keys are looked up in a `.env` file **beside the `Outfit` being applied** first +(or in the current directory, for a command that takes no Outfit), then your +shell environment — so a project keeps its own key next to the file that needs +it, the same way `PRESET` and `REMOTE` travel with an Outfit. They are **never written into the agent's config** — outfit writes +a reference the agent resolves when it runs, and `outfit harness` passes the +keys it can resolve to the agent it launches. If you start the agent yourself, +set the variable in your own environment. Local providers (Ollama, llama.cpp on +localhost) need no key; Bedrock uses your AWS credentials. diff --git a/docs/commands/add.md b/docs/commands/add.md index 5bf373b4..e0dfdffc 100644 --- a/docs/commands/add.md +++ b/docs/commands/add.md @@ -51,7 +51,9 @@ outfit add -p llamacpp -m my-model -c 128k -o 32k - You need at least one of `--model-family`, `--model`, or `--alias` alongside the provider. -- API keys are read from a `.env` next to the tool, then your environment, and +- API keys are read from a `.env` beside the `Outfit` — or, for `outfit add`, + which has no Outfit, from a `.env` in the current directory — then your + environment, and never written anywhere they'll leak. A provider that requires a key tells you which variable to set. - On opencode, `add` sets the chosen model as the default. Pi has no diff --git a/docs/commands/list.md b/docs/commands/list.md index 9c700aa6..7b3b4fcf 100644 --- a/docs/commands/list.md +++ b/docs/commands/list.md @@ -21,7 +21,7 @@ This is the *could* view — what's available to configure. For what your agent ## Notes - A provider marked with a required API key won't configure until that - variable is set in a `.env` next to the tool or your environment. + variable is set in a `.env` beside your `Outfit`, or in your environment. - Not every provider maps to every harness — the listing names which harnesses each supports (AWS Bedrock, for instance, is opencode-only). diff --git a/docs/commands/remote.md b/docs/commands/remote.md new file mode 100644 index 00000000..4dedaad8 --- /dev/null +++ b/docs/commands/remote.md @@ -0,0 +1,112 @@ +# outfit remote + +Run a model too big for your laptop on a GPU in the cloud, from the same +[`Outfit` file](../outfit-file.md) you'd use locally — and only pay for it +while you're using it. + +```sh +outfit remote deploy # tell the endpoint what to serve +outfit remote start # boot it; prints the exports your agent needs (progress on stderr) +outfit remote status # is it up? is it healthy? +outfit remote stop # shut it down now, rather than waiting for the idle timer +``` + +The endpoint is the one [`remote/`](../../remote/) in this repository deploys: a +GPU instance that exists only while you're using it, and terminates itself once +you stop. `outfit remote` drives it; it doesn't create it. + +## The usual flow + +```sh +eval "$(outfit remote start)" # boots it (~10 min from cold) and sets + # OPENAI_BASE_URL and OPENAI_API_KEY +outfit apply # point your agent at it +outfit harness # work +outfit remote stop # done +``` + +Forgetting `stop` is not a disaster — the endpoint terminates itself after a +spell with no requests — but it's the difference between minutes and hours of +GPU time. + +## Pointing at your endpoint + +`outfit remote` needs the endpoint's control URLs, which its deployment prints. +Put them in a JSON file: + +```json +{ + "start_url": "https://....lambda-url.us-east-1.on.aws/", + "stop_url": "https://....lambda-url.us-east-1.on.aws/", + "deploy_url": "https://....lambda-url.us-east-1.on.aws/", + "region": "us-east-1" +} +``` + +Either name it from the Outfit, so a project carries its own endpoint: + +```dockerfile +REMOTE ./remote.json # resolved next to the Outfit, like PRESET +``` + +…or save it once as `~/.config/outfit/remote.json`, which is used whenever no +Outfit names one — so `outfit remote` works from anywhere. + +Requests are signed with **your** AWS credentials (the usual profile, SSO +session, or environment variables), and the endpoint's URLs require it. Outfit +stores no credentials of its own and needs no permission beyond invoking those +URLs. + +## Choosing what it serves + +`outfit remote deploy` reads the Outfit and its preset, and tells the endpoint +what to load. `PROVIDER` picks the engine, so the file that runs a model +locally under [`outfit serve`](serve.md) deploys the same model remotely: + +```dockerfile +PROVIDER llamacpp # the engine to run: llamacpp or vllm +ALIAS qwen3.6-27b # the name your agent asks for — and the name served +CONTEXT 131072 +PRESET ./preset.ini # the model and its flags +REMOTE ./remote.json +``` + +Everything the endpoint sets itself — host, port, where the weights live, the +API key, the context size, the alias — is dropped from the preset, so one +preset works both locally and remotely without edits. + +Deploying doesn't start anything. If the endpoint doesn't have those weights +yet it fetches them (about 15–20 minutes, entirely on its side) and says so; +wait for that before your first `start`, or the model won't be there. + +Switching model, quantisation, or engine is an edit to those two files and one +`deploy` — no redeployment of the infrastructure. + +```sh +outfit remote deploy --dry-run # see what would be sent +outfit remote deploy path/to/Outfit # deploy a different one +``` + +## Flags + +| Flag | Meaning | +| ---- | ------- | +| `--timeout` | How long `start` waits for the endpoint (default 15m) | +| `-n`, `--dry-run` | `deploy` only: print what would be sent, without sending it | + +## Notes + +- Every subcommand takes an optional Outfit path, or a + [registered alias](alias.md), and defaults to `./Outfit`. +- `deploy` always needs an Outfit — it's the thing being deployed. The others + fall back to your per-user config. +- `deploy_url` is optional: a config written before `deploy` existed still + works for `start`, `stop`, and `status`. +- Only a self-hosted engine can be deployed (`llamacpp` or `vllm`). A hosted + provider has nothing to deploy. + +## See also + +- [The `Outfit` file](../outfit-file.md) — including `REMOTE` +- [`outfit serve`](serve.md) — the same Outfit, run on your own machine +- [`outfit apply`](apply.md) — point your agent at the endpoint diff --git a/docs/outfit-file.md b/docs/outfit-file.md index 01d54869..1bf59d22 100644 --- a/docs/outfit-file.md +++ b/docs/outfit-file.md @@ -42,6 +42,32 @@ Every command that takes an Outfit path defaults to `./Outfit` in the current directory, accepts a directory that holds one, and takes a [registered alias](commands/alias.md) in place of a path. +## Running the model on a cloud GPU + +For a model too big for your machine, `REMOTE` names the config of a +scale-to-zero GPU endpoint — one that runs only while you're using it: + +```dockerfile +# Outfit +PROVIDER llamacpp # the engine to run there, as it would run here +ALIAS qwen3.6-27b +CONTEXT 131072 +PRESET ./preset.ini +REMOTE ./remote.json +``` + +[`outfit remote`](commands/remote.md) then reads the named file — resolved +relative to the Outfit, like `PRESET` — so a project can carry its own endpoint +instead of relying on the per-user +`${XDG_CONFIG_HOME:-~/.config}/outfit/remote.json`. With no path argument the +commands consult `./Outfit` when it exists and fall back to the per-user file; +an explicit path (`outfit remote status path/to/Outfit`) requires the Outfit +to carry a `REMOTE` instruction. + +Because `PROVIDER` names the engine, this is the same file that would run the +model locally with [`outfit serve`](commands/serve.md) — pointed at a bigger +machine. + ## Syntax One instruction per line: a keyword followed by a single value. @@ -56,6 +82,7 @@ One instruction per line: a keyword followed by a single value. | `OUTPUT` | no | `--output` | `OUTPUT 32k` | | `BASEURL` | no | `--base-url` | `BASEURL https://gateway/v1` | | `PRESET` | no | `outfit serve` | `PRESET ./preset.ini` | +| `REMOTE` | no | `outfit remote` | `REMOTE ./remote.json` | Rules: diff --git a/go.mod b/go.mod index 42a66656..826b17fd 100644 --- a/go.mod +++ b/go.mod @@ -4,4 +4,23 @@ go 1.24.13 require github.com/tailscale/hujson v0.0.0-20260302212456-ecc657c15afd -require gopkg.in/yaml.v3 v3.0.1 +require ( + github.com/aws/aws-sdk-go-v2 v1.43.0 + github.com/aws/aws-sdk-go-v2/config v1.32.31 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/aws/aws-sdk-go-v2/credentials v1.19.30 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 // indirect + github.com/aws/smithy-go v1.27.3 // indirect +) diff --git a/go.sum b/go.sum index 6f936df0..47a91a9f 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,31 @@ +github.com/aws/aws-sdk-go-v2 v1.43.0 h1:fharf/WhbRAVZ1du0QL7roNFxZ6T/sWr+4Ni617bwSI= +github.com/aws/aws-sdk-go-v2 v1.43.0/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= +github.com/aws/aws-sdk-go-v2/config v1.32.31 h1:n4nY9O3QKoHIkL85EX+V8RcMFtOhlpTFhGArg915PXk= +github.com/aws/aws-sdk-go-v2/config v1.32.31/go.mod h1:PN0NYDCCoOpGGsZ2+elDUidmHfQBPyYzN2GCgl8HEBs= +github.com/aws/aws-sdk-go-v2/credentials v1.19.30 h1:TTCvvzFU6gXa4iJecNG/0F/B0oYTiazoRECr2XyLHrY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.30/go.mod h1:jKxAp2AEncnliinzpgOSZDFv6+VjvWhjw/AtbfsWT9U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 h1:kfVL5wAunCJycL6MOQ6aNh6PlAYEymflcjuKmrWUA0o= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31/go.mod h1:nWfRNDAppujCQgOUd43lKT4yeLv9z3nJ3bw1G3BgQKo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 h1:Z8F3hfCY33IGpJjFAnv0wvtv1FIKj1GHmRDEYqy64tw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31/go.mod h1:aVyUoytEyOViR6jhq6jula0xkc5NfBE2hgeF6BvOrao= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 h1:hyOxUyXdh3AyjE93gBgsfziJag9ACwcs+ZpDBLzi8mw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31/go.mod h1:OERqI9k0draSLB8O8woxY3q25ZWTELRK4RRoLMuMZFo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 h1:0MrUL35H/Y4kdFfItoR5jCgtDQ4Z/8LudAoIHRfA4hE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32/go.mod h1:2tNZkuWz54arj8mHVf+8Y7cKkcD8Wr/fBpENgEXpjLc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 h1:w2SIhW92DZPFrSL4ksVCr8IYff5OZwIcxg8+95tzvAI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31/go.mod h1:wAhpCQbkov+IcvjozJbd2xRCoZybUEHNkcFunssNACg= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 h1:OHH5iTQvVGmfHjX/5Q+vFuA/Rf2x6/95aJ/75QCQSm4= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.0/go.mod h1:mCF3AK9PpL49oOrhniUXWAfhVBVQ/XbytoE5eccZUIs= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 h1:CaJyYhxBE0M/HJX/YvSaSmQlsI91VHB0lKU8LtLxL3A= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.0/go.mod h1:+e6BMRMPjBQoCw/WovYR9GLy2IU0z4Q77smOB1DraSg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 h1:tC323YV77QdafeBr6LUhLDTsboyuyHLNRwAyCP44kGU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0/go.mod h1:SfLK1sgviHmbI+MozR9iDwDjL4cdCVZtahsjoR+z7wg= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 h1:Pd6PNlp4t8PTXxqzstICl52Wsy78vpjFZ7PRUj44mJc= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.0/go.mod h1:rmQ0TnHzuLPmabgjPcsywhsSOmaBDgzR4zvDxSPsGdg= +github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY= +github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/tailscale/hujson v0.0.0-20260302212456-ecc657c15afd h1:Rf9uhF1+VJ7ZHqxrG8pJ6YacmHvVCmByDmGbAWCc/gA= diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go index af27c542..35df652b 100644 --- a/internal/catalog/catalog.go +++ b/internal/catalog/catalog.go @@ -5,6 +5,7 @@ package catalog import ( _ "embed" "fmt" + "net/url" "os" "sort" "strings" @@ -35,11 +36,14 @@ type Catalog struct { // Provider describes how to construct an opencode provider block. type Provider struct { - Description string `yaml:"description"` - Name string `yaml:"name"` - NPM string `yaml:"npm"` - APIKeyEnv string `yaml:"apiKeyEnv"` - APIKeyRequired bool `yaml:"apiKeyRequired"` + Description string `yaml:"description"` + Name string `yaml:"name"` + NPM string `yaml:"npm"` + APIKeyEnv string `yaml:"apiKeyEnv"` + APIKeyRequired bool `yaml:"apiKeyRequired"` + // APIKeyOptional marks a provider that also works unauthenticated (a local + // server). It only affects the Pi harness: see BuildPiProvider. + APIKeyOptional bool `yaml:"apiKeyOptional"` APIKeyPrefix string `yaml:"apiKeyPrefix"` Options map[string]any `yaml:"options"` OptionsFromEnv map[string]string `yaml:"optionsFromEnv"` @@ -199,13 +203,21 @@ func BuildProviderBlock(id string, p *Provider, familyName, modelOverride, baseU } if p.APIKeyEnv != "" { key := resolve(p.APIKeyEnv) + baseURL, _ := options["baseURL"].(string) switch { case key == "" && p.APIKeyRequired: return nil, "", fmt.Errorf("%s is not set; add it to your .env or environment", p.APIKeyEnv) case key != "" && p.APIKeyPrefix != "" && !strings.HasPrefix(key, p.APIKeyPrefix): return nil, "", fmt.Errorf("%s does not look right (expected it to start with %q)", p.APIKeyEnv, p.APIKeyPrefix) - case key != "": - options["apiKey"] = key + case key == "" && p.APIKeyOptional && IsLocalEndpoint(baseURL): + // A local server that needs no key: leave the option out entirely + // rather than point it at a variable nobody will set. + default: + // opencode substitutes {env:VAR} when it reads the config, so the + // secret is never written to disk. The variable has to be set when + // opencode runs; `outfit harness` passes on whatever it can resolve, + // including from its own .env. + options["apiKey"] = EnvRef(p.APIKeyEnv) } } if len(options) > 0 { @@ -295,14 +307,28 @@ func BuildPiProvider(id string, p *Provider, familyName, modelOverride, baseURLO prov.BaseURL, _ = p.Options["baseURL"].(string) } - if p.APIKeyEnv != "" { + // Pi only surfaces a provider's models in /model once auth is configured; + // with no apiKey at all the models load but stay unavailable. Keyless local + // servers (llama.cpp, Ollama, …) ignore the key, so write a dummy literal — + // the same placeholder pattern Pi's own docs use for Ollama — to make the + // models selectable. + // + // A "$VAR" reference is written whenever the provider has a key env var, + // because Pi resolves it at run time: the key need not be set when the + // Outfit is applied, and exporting it later is enough. The exception is an + // apiKeyOptional provider, pointed at a local endpoint, whose var is unset: + // one provider covers both a keyless local server and an authenticated + // remote one, and for the local server a reference to a variable set + // nowhere would hide the models. That exception is deliberately conditioned + // on the endpoint rather than only on the key, because a placeholder + // written for a remote endpoint could not be repaired by exporting the key + // afterwards — Pi would keep sending the placeholder. + switch { + case p.APIKeyEnv != "" && p.APIKeyOptional && resolve(p.APIKeyEnv) == "" && IsLocalEndpoint(prov.BaseURL): + prov.APIKey = piPlaceholderAPIKey + case p.APIKeyEnv != "": prov.APIKey = "$" + p.APIKeyEnv - } else { - // Pi only surfaces a provider's models in /model once auth is configured; - // with no apiKey at all the models load but stay unavailable. Keyless local - // servers (llama.cpp, Ollama, …) ignore the key, so write a dummy literal — - // the same placeholder pattern Pi's own docs use for Ollama — to make the - // models selectable. + default: prov.APIKey = piPlaceholderAPIKey } @@ -344,3 +370,27 @@ func BuildPiProvider(id string, p *Provider, familyName, modelOverride, baseURLO return prov, defaultModelKey, nil } + +// EnvRef renders an environment-variable reference in the form opencode +// substitutes when it reads its config, so a secret never lands on disk. +func EnvRef(name string) string { + return "{env:" + name + "}" +} + +// IsLocalEndpoint reports whether a base URL points at this machine, where a +// server commonly needs no API key. An empty or unparseable URL counts as +// local, so callers only treat an endpoint as remote on real evidence. +func IsLocalEndpoint(baseURL string) bool { + if baseURL == "" { + return true + } + u, err := url.Parse(baseURL) + if err != nil { + return true + } + switch u.Hostname() { + case "", "localhost", "127.0.0.1", "0.0.0.0", "::1": + return true + } + return false +} diff --git a/internal/catalog/catalog_test.go b/internal/catalog/catalog_test.go index 5aabbd90..83f9f969 100644 --- a/internal/catalog/catalog_test.go +++ b/internal/catalog/catalog_test.go @@ -188,7 +188,7 @@ func TestBuildPiProvider_BaseURLFallbackAndOverride(t *testing.T) { func TestBuildPiProvider_ModelOverrideNoKey(t *testing.T) { cat, _ := Load() - p := cat.Providers["llamacpp"] // no apiKeyEnv + p := cat.Providers["llamacpp"] // apiKeyOptional, and the var is unset here prov, model, err := BuildPiProvider("llamacpp", p, "", "my-local", "", noEnv) if err != nil { @@ -266,7 +266,7 @@ func TestLoadCatalogFrom(t *testing.T) { } } -func TestBuildProviderBlock_OpenRouterKeyInjected(t *testing.T) { +func TestBuildProviderBlock_OpenRouterKeyReferenced(t *testing.T) { cat, _ := Load() p := cat.Providers["openrouter"] @@ -280,8 +280,13 @@ func TestBuildProviderBlock_OpenRouterKeyInjected(t *testing.T) { t.Errorf("default model = %q, want %q", model, want) } opts := block["options"].(map[string]any) - if opts["apiKey"] != "sk-or-v1-abc" { - t.Errorf("apiKey = %v, want injected key", opts["apiKey"]) + // The secret is never written to the config; opencode substitutes the + // reference when it reads it. + if opts["apiKey"] != "{env:DEEPSEEK_API_KEY}" { + t.Errorf("apiKey = %v, want the env reference", opts["apiKey"]) + } + if opts["apiKey"] == "sk-or-v1-abc" { + t.Error("the resolved secret must not be written to the config") } if models := block["models"].(map[string]any); len(models) != 2 { t.Errorf("got %d models, want 2", len(models)) @@ -483,3 +488,128 @@ func TestEmbeddedYAML(t *testing.T) { t.Error("EmbeddedYAML should return a fresh copy, not the underlying bytes") } } + +// llamacpp names both a keyless local server and the authenticated remote +// deployment, so its key is optional: written through when set, and replaced by +// Pi's placeholder when not, rather than a $VAR that resolves to nothing. +func TestBuildPiProvider_OptionalKey(t *testing.T) { + cat, _ := Load() + p := cat.Providers["llamacpp"] + + withKey := func(name string) string { + if name == "OPENAI_API_KEY" { + return "sk-remote" + } + return "" + } + prov, _, err := BuildPiProvider("llamacpp", p, "local", "", "", withKey) + if err != nil { + t.Fatalf("BuildPiProvider: %v", err) + } + if prov.APIKey != "$OPENAI_API_KEY" { + t.Errorf("apiKey = %q, want the $VAR reference when the key is set", prov.APIKey) + } + + prov, _, err = BuildPiProvider("llamacpp", p, "local", "", "", noEnv) + if err != nil { + t.Fatalf("BuildPiProvider: %v", err) + } + if prov.APIKey != piPlaceholderAPIKey { + t.Errorf("apiKey = %q, want the placeholder when the key is unset", prov.APIKey) + } +} + +// A provider whose key is NOT optional keeps the reference even when unset, so +// the key can be exported after the Outfit is applied. +func TestBuildPiProvider_RequiredKeyKeepsReference(t *testing.T) { + cat, _ := Load() + prov, _, err := BuildPiProvider("openai-compatible", cat.Providers["openai-compatible"], "gpt", "", "", noEnv) + if err != nil { + t.Fatalf("BuildPiProvider: %v", err) + } + if prov.APIKey != "$OPENAI_API_KEY" { + t.Errorf("apiKey = %q, want the $VAR reference to survive an unset key", prov.APIKey) + } +} + +// The opencode block injects the resolved key, so a remote llama.cpp endpoint +// is authenticated rather than 401ing. +func TestBuildProviderBlock_LlamacppReferencesKeyForRemote(t *testing.T) { + cat, _ := Load() + resolve := func(name string) string { + if name == "OPENAI_API_KEY" { + return "sk-remote" + } + return "" + } + block, _, err := BuildProviderBlock("llamacpp", cat.Providers["llamacpp"], "local", "", "http://198.51.100.1:8000/v1", resolve) + if err != nil { + t.Fatalf("BuildProviderBlock: %v", err) + } + options := block["options"].(map[string]any) + if options["apiKey"] != "{env:OPENAI_API_KEY}" { + t.Errorf("apiKey = %v, want the env reference", options["apiKey"]) + } + + block, _, err = BuildProviderBlock("llamacpp", cat.Providers["llamacpp"], "local", "", "", noEnv) + if err != nil { + t.Fatalf("BuildProviderBlock: %v", err) + } + options = block["options"].(map[string]any) + if _, ok := options["apiKey"]; ok { + t.Error("a local keyless server should get no apiKey at all") + } + + // A remote endpoint keeps the reference even with the key unset, so setting + // it before the agent runs is enough. + block, _, err = BuildProviderBlock("llamacpp", cat.Providers["llamacpp"], "local", "", "http://198.51.100.1:8000/v1", noEnv) + if err != nil { + t.Fatalf("BuildProviderBlock: %v", err) + } + if got := block["options"].(map[string]any)["apiKey"]; got != "{env:OPENAI_API_KEY}" { + t.Errorf("apiKey = %v, want the env reference for a remote endpoint", got) + } +} + +func TestIsLocalEndpoint(t *testing.T) { + for _, u := range []string{ + "", "http://localhost:8080/v1", "http://127.0.0.1:8080/v1", + "http://0.0.0.0:8000/v1", "http://[::1]:8080/v1", "::not a url::", + } { + if !IsLocalEndpoint(u) { + t.Errorf("IsLocalEndpoint(%q) = false, want true", u) + } + } + for _, u := range []string{"http://198.51.100.1:8000/v1", "https://api.example.com/v1"} { + if IsLocalEndpoint(u) { + t.Errorf("IsLocalEndpoint(%q) = true, want false", u) + } + } +} + +// Pi resolves its $VAR reference when it runs, so a placeholder written for a +// remote endpoint can never be repaired by exporting the key afterwards — Pi +// would keep sending the placeholder. The placeholder is therefore only right +// for a local server. +func TestBuildPiProvider_RemoteOptionalKeyKeepsReference(t *testing.T) { + cat, _ := Load() + p := cat.Providers["llamacpp"] // apiKeyOptional + + prov, _, err := BuildPiProvider("llamacpp", p, "local", "", "http://198.51.100.1:8000/v1", noEnv) + if err != nil { + t.Fatalf("BuildPiProvider: %v", err) + } + if prov.APIKey != "$OPENAI_API_KEY" { + t.Errorf("apiKey = %q, want the reference so exporting the key later works", prov.APIKey) + } + + // The local server keeps the placeholder: it needs no key, and a reference + // to a variable set nowhere would hide its models in /model. + prov, _, err = BuildPiProvider("llamacpp", p, "local", "", "http://127.0.0.1:8080/v1", noEnv) + if err != nil { + t.Fatalf("BuildPiProvider: %v", err) + } + if prov.APIKey != piPlaceholderAPIKey { + t.Errorf("apiKey = %q, want the placeholder for a local server", prov.APIKey) + } +} diff --git a/internal/catalog/providers.yaml b/internal/catalog/providers.yaml index 32c05f08..8ae1609c 100644 --- a/internal/catalog/providers.yaml +++ b/internal/catalog/providers.yaml @@ -11,6 +11,9 @@ # apiKeyEnv env var (looked up in .env, then the environment) whose # value is injected as options.apiKey # apiKeyRequired fail if apiKeyEnv resolves to nothing +# apiKeyOptional the provider also works with no key (a local server). When +# apiKeyEnv resolves to nothing, the Pi harness writes its +# keyless placeholder instead of an unresolvable $VAR # apiKeyPrefix sanity-check that the key starts with this # options static options merged into the provider block # optionsFromEnv option -> env var; injected into options when the var is set @@ -81,9 +84,15 @@ providers: name: Llama 3.1 llamacpp: - description: Local llama.cpp server (OpenAI-compatible API) - name: llama.cpp (local) + description: llama.cpp server (OpenAI-compatible API) + name: llama.cpp npm: "@ai-sdk/openai-compatible" + # A local llama-server usually needs no key, and none is injected when the + # var is unset. But the same provider also names a remote deployment (see + # `outfit remote deploy`), which is started with --api-key, so the key is + # picked up when there is one. + apiKeyEnv: OPENAI_API_KEY + apiKeyOptional: true options: baseURL: http://localhost:8080/v1 optionsFromEnv: @@ -98,6 +107,27 @@ providers: local-model: name: llama.cpp model + vllm: + description: vLLM server (OpenAI-compatible API) + name: vLLM + npm: "@ai-sdk/openai-compatible" + # vLLM is started with an API key when exposed beyond localhost (as the + # cloud-vm-llm deployment does), so the key is injected when the var is set. + apiKeyEnv: OPENAI_API_KEY + options: + baseURL: http://localhost:8000/v1 + optionsFromEnv: + baseURL: VLLM_BASE_URL + pi: + api: openai-completions + families: + served: + description: Model loaded by your vLLM server + defaultModel: served-model + models: + served-model: + name: vLLM model + openai-compatible: description: Generic OpenAI-compatible endpoint (set base URL and key via env) name: OpenAI-compatible diff --git a/internal/harness/adapters.go b/internal/harness/adapters.go index ef525b51..b2d3f8ed 100644 --- a/internal/harness/adapters.go +++ b/internal/harness/adapters.go @@ -30,8 +30,8 @@ func (opencodeHarness) Command() string { return "opencode" } func (opencodeHarness) ConfigPath() (string, error) { return opencode.ResolveConfigFile() } -func (opencodeHarness) Apply(p *catalog.Provider, sel outfit.Selection, contextWindow, outputTokens int) (Summary, error) { - block, defaultModel, err := catalog.BuildProviderBlock(sel.Provider, p, sel.Family, modelKey(sel), sel.BaseURL, opencode.ResolveEnv) +func (opencodeHarness) Apply(p *catalog.Provider, sel outfit.Selection, contextWindow, outputTokens int, resolve func(string) string) (Summary, error) { + block, defaultModel, err := catalog.BuildProviderBlock(sel.Provider, p, sel.Family, modelKey(sel), sel.BaseURL, resolve) if err != nil { return Summary{}, err } @@ -51,8 +51,12 @@ func (opencodeHarness) Apply(p *catalog.Provider, sel outfit.Selection, contextW var notes []string if opts, ok := block["options"].(map[string]any); ok { - if _, ok := opts["apiKey"]; ok { - notes = append(notes, fmt.Sprintf("API key injected from %s.", p.APIKeyEnv)) + baseURL, _ := opts["baseURL"].(string) + if warning := missingKeyWarning(p, baseURL, resolve); warning != "" { + notes = append(notes, warning) + } else if _, ok := opts["apiKey"]; ok { + notes = append(notes, fmt.Sprintf( + "API key read from %s when opencode runs (never written to the config).", p.APIKeyEnv)) } if b, ok := opts["baseURL"]; ok { notes = append(notes, fmt.Sprintf("Base URL: %v", b)) @@ -95,8 +99,8 @@ func (piHarness) Command() string { return "pi" } func (piHarness) ConfigPath() (string, error) { return pi.ConfigPath() } -func (piHarness) Apply(p *catalog.Provider, sel outfit.Selection, contextWindow, outputTokens int) (Summary, error) { - prov, defaultModel, err := catalog.BuildPiProvider(sel.Provider, p, sel.Family, modelKey(sel), sel.BaseURL, opencode.ResolveEnv) +func (piHarness) Apply(p *catalog.Provider, sel outfit.Selection, contextWindow, outputTokens int, resolve func(string) string) (Summary, error) { + prov, defaultModel, err := catalog.BuildPiProvider(sel.Provider, p, sel.Family, modelKey(sel), sel.BaseURL, resolve) if err != nil { return Summary{}, err } @@ -109,7 +113,9 @@ func (piHarness) Apply(p *catalog.Provider, sel outfit.Selection, contextWindow, } var notes []string - if prov.APIKey != "" { + if warning := missingKeyWarning(p, prov.BaseURL, resolve); warning != "" { + notes = append(notes, warning) + } else if prov.APIKey != "" { notes = append(notes, fmt.Sprintf("API key referenced as %s (set it in your environment or Pi auth.json).", prov.APIKey)) } if prov.BaseURL != "" { @@ -139,3 +145,19 @@ func (piHarness) State() (map[string]ProviderState, string, error) { } return out, "", nil } + +// missingKeyWarning returns a warning when a provider's config is being written +// with no API key while its endpoint is not a local address — a combination +// that writes successfully and then fails on the first request. It happens most +// easily with a provider whose key is optional (the same engine run locally, +// where no key is needed, and remotely, where one is), so nothing errors and +// the config simply cannot authenticate. baseURL is empty when the provider +// declares none. +func missingKeyWarning(p *catalog.Provider, baseURL string, resolve func(string) string) string { + if p.APIKeyEnv == "" || resolve(p.APIKeyEnv) != "" || catalog.IsLocalEndpoint(baseURL) { + return "" + } + return fmt.Sprintf( + "Warning: no API key was set, but %s is not a local address, so requests will "+ + "probably be rejected. Set %s and apply again.", baseURL, p.APIKeyEnv) +} diff --git a/internal/harness/harness.go b/internal/harness/harness.go index c3b3aa25..57404ef0 100644 --- a/internal/harness/harness.go +++ b/internal/harness/harness.go @@ -59,7 +59,9 @@ type Harness interface { ConfigPath() (string, error) // Apply writes a single provider selection into the harness config. // contextWindow and outputTokens, when > 0, are the resolved limits to set. - Apply(p *catalog.Provider, sel outfit.Selection, contextWindow, outputTokens int) (Summary, error) + // resolve looks up an API key variable — see opencode.EnvResolver, which + // builds one from the Outfit's directory. + Apply(p *catalog.Provider, sel outfit.Selection, contextWindow, outputTokens int, resolve func(string) string) (Summary, error) // Remove removes a provider, or specific model keys within it. With no // modelKeys the whole provider is removed. Returns the number of removals. Remove(providerID string, modelKeys []string) (int, error) diff --git a/internal/harness/harness_test.go b/internal/harness/harness_test.go index 9f53b7ac..56c6f163 100644 --- a/internal/harness/harness_test.go +++ b/internal/harness/harness_test.go @@ -1,8 +1,10 @@ package harness import ( + "github.com/lucinate-ai/outfit/internal/catalog" "os" "path/filepath" + "strings" "testing" "github.com/lucinate-ai/outfit/internal/config" @@ -136,3 +138,27 @@ func TestNamesAndLookup(t *testing.T) { t.Error("Lookup(nope) should fail") } } + +// A config written with no key for a remote endpoint succeeds and then fails on +// the first request, so it has to be called out at the time. +func TestMissingKeyWarning(t *testing.T) { + keyed := &catalog.Provider{APIKeyEnv: "OPENAI_API_KEY"} + unset := func(string) string { return "" } + set := func(string) string { return "sk-test" } + + if w := missingKeyWarning(keyed, "http://198.51.100.1:8000/v1", unset); w == "" { + t.Error("a remote endpoint with no key should warn") + } else if !strings.Contains(w, "OPENAI_API_KEY") { + t.Errorf("the warning should name the variable to set, got %q", w) + } + + if w := missingKeyWarning(keyed, "http://198.51.100.1:8000/v1", set); w != "" { + t.Errorf("a key that is set should not warn, got %q", w) + } + if w := missingKeyWarning(keyed, "http://127.0.0.1:8080/v1", unset); w != "" { + t.Errorf("a local server needs no key, got %q", w) + } + if w := missingKeyWarning(&catalog.Provider{}, "https://api.example.com/v1", unset); w != "" { + t.Errorf("a provider with no key variable should not warn, got %q", w) + } +} diff --git a/internal/opencode/opencode.go b/internal/opencode/opencode.go index 51a8ffcf..6aac3b0a 100644 --- a/internal/opencode/opencode.go +++ b/internal/opencode/opencode.go @@ -8,7 +8,6 @@ import ( "fmt" "os" "path/filepath" - "runtime" "sort" "strings" @@ -17,20 +16,27 @@ import ( // envFilePath returns the path to the .env file alongside this program's // source, mirroring the original "the .env next to the tool" behaviour. -func envFilePath() string { - if _, file, _, ok := runtime.Caller(0); ok { - return filepath.Join(filepath.Dir(file), ".env") - } - return ".env" -} -// ResolveEnv looks up an environment variable, preferring the .env file next -// to the tool and falling back to the process environment. -func ResolveEnv(name string) string { - if v := readEnvFileVar(envFilePath(), name); v != "" { - return v +// EnvResolver returns a lookup that prefers a `.env` beside the Outfit being +// applied, falling back to the process environment. dir is that Outfit's +// directory; when empty — a selection made entirely from flags, with no Outfit +// to sit beside — the working directory is used, so `outfit add` still finds a +// project's own `.env`. +// +// The file sits beside the Outfit rather than beside the binary because that is +// where it belongs to a project — the same rule PRESET and REMOTE follow — so an +// Outfit and the key it needs travel together, and an installed binary can find +// one at all. +func EnvResolver(dir string) func(string) string { + if dir == "" { + dir = "." + } + return func(name string) string { + if v := readEnvFileVar(filepath.Join(dir, ".env"), name); v != "" { + return v + } + return os.Getenv(name) } - return os.Getenv(name) } // readEnvFileVar returns the value of name from a dotenv-style file, or "". diff --git a/internal/opencode/opencode_test.go b/internal/opencode/opencode_test.go index bf105c77..df6c10b9 100644 --- a/internal/opencode/opencode_test.go +++ b/internal/opencode/opencode_test.go @@ -285,3 +285,58 @@ func TestResolveConfigFile(t *testing.T) { t.Errorf("got %q, want %q", got, want) } } + +// The .env belongs beside the Outfit, the same rule PRESET and REMOTE follow — +// not beside the binary, and emphatically not beside the source file, which is +// a path from the build machine that an installed binary can never find. +func TestEnvResolver_ReadsDotEnvBesideTheOutfit(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, ".env"), []byte("OPENAI_API_KEY=sk-beside\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("OPENAI_API_KEY", "") + + if got := EnvResolver(dir)("OPENAI_API_KEY"); got != "sk-beside" { + t.Errorf("EnvResolver(dir) = %q, want the value from the Outfit's .env", got) + } + // Another directory's .env is none of this Outfit's business. + if got := EnvResolver(t.TempDir())("OPENAI_API_KEY"); got != "" { + t.Errorf("resolved %q from an unrelated directory", got) + } +} + +func TestEnvResolver_FallsBackToTheEnvironment(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("OPENAI_API_KEY", "sk-exported") + if got := EnvResolver("")("OPENAI_API_KEY"); got != "sk-exported" { + t.Errorf("with no Outfit directory, want the environment's value, got %q", got) + } + + // The file wins when it has the variable, so a project can override. + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, ".env"), []byte("OPENAI_API_KEY=sk-beside\n"), 0o600); err != nil { + t.Fatal(err) + } + if got := EnvResolver(dir)("OPENAI_API_KEY"); got != "sk-beside" { + t.Errorf("the Outfit's .env should win, got %q", got) + } + // …and the environment still answers for anything the file omits. + if got := EnvResolver(dir)("SOMETHING_ELSE"); got != "" { + t.Errorf("unset variable resolved to %q", got) + } +} + +// A command with no Outfit — `outfit add -p openrouter` and friends — still has +// a project around it, so its .env is worth reading. +func TestEnvResolver_NoOutfitReadsTheWorkingDirectory(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + if err := os.WriteFile(filepath.Join(dir, ".env"), []byte("DEEPSEEK_API_KEY=sk-or-cwd\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("DEEPSEEK_API_KEY", "") + + if got := EnvResolver("")("DEEPSEEK_API_KEY"); got != "sk-or-cwd" { + t.Errorf("EnvResolver(\"\") = %q, want the working directory's .env", got) + } +} diff --git a/internal/outfit/outfit.go b/internal/outfit/outfit.go index fe40bb0c..bc00b6a3 100644 --- a/internal/outfit/outfit.go +++ b/internal/outfit/outfit.go @@ -14,6 +14,7 @@ // OUTPUT 32k # optional; max output tokens // BASEURL https://gateway/v1 # optional; API base URL override // PRESET ./preset.ini # optional; llama.cpp preset for `serve` +// REMOTE ./remote.json # optional; remote-instance config for `remote` // // MODEL is the reference the provider itself understands: an OpenRouter/Bedrock // model id, an Ollama name, or — for llamacpp — a Hugging Face repo @@ -45,6 +46,7 @@ type Selection struct { Providers string BaseURL string Preset string + Remote string } // Outfit keywords, in their canonical (lower-cased) form for matching. @@ -57,6 +59,7 @@ const ( kwOutput = "output" kwBaseURL = "baseurl" kwPreset = "preset" + kwRemote = "remote" ) // canonicalKeyword resolves an Outfit keyword (already lower-cased) to its @@ -64,7 +67,7 @@ const ( // "" for an unrecognised keyword. func canonicalKeyword(kw string) string { switch kw { - case kwProvider, kwFamily, kwModel, kwAlias, kwContext, kwOutput, kwPreset: + case kwProvider, kwFamily, kwModel, kwAlias, kwContext, kwOutput, kwPreset, kwRemote: return kw case kwBaseURL, "base-url", "base_url", "url": return kwBaseURL @@ -92,7 +95,7 @@ func Parse(data []byte) (Selection, error) { fields := strings.Fields(text) canon := canonicalKeyword(strings.ToLower(fields[0])) if canon == "" { - return Selection{}, fmt.Errorf("line %d: unknown keyword %q (expected PROVIDER, FAMILY, MODEL, ALIAS, CONTEXT, OUTPUT, BASEURL, or PRESET)", line, fields[0]) + return Selection{}, fmt.Errorf("line %d: unknown keyword %q (expected PROVIDER, FAMILY, MODEL, ALIAS, CONTEXT, OUTPUT, BASEURL, PRESET, or REMOTE)", line, fields[0]) } switch { case len(fields) < 2: @@ -124,6 +127,8 @@ func Parse(data []byte) (Selection, error) { sel.BaseURL = value case kwPreset: sel.Preset = value + case kwRemote: + sel.Remote = value } } if err := scanner.Err(); err != nil { @@ -169,5 +174,6 @@ func Format(sel Selection) string { line("OUTPUT", sel.Output) line("BASEURL", sel.BaseURL) line("PRESET", sel.Preset) + line("REMOTE", sel.Remote) return b.String() } diff --git a/internal/outfit/outfit_test.go b/internal/outfit/outfit_test.go index c6fdddc9..7f325865 100644 --- a/internal/outfit/outfit_test.go +++ b/internal/outfit/outfit_test.go @@ -106,3 +106,19 @@ func TestFormatRoundTrip(t *testing.T) { t.Errorf("round-trip changed selection: %+v -> %+v", sel, got) } } + +func TestParse_Remote(t *testing.T) { + sel, err := Parse([]byte("PROVIDER openai-compatible\nREMOTE ./remote.json\n")) + if err != nil { + t.Fatal(err) + } + if sel.Remote != "./remote.json" { + t.Errorf("Remote = %q, want ./remote.json", sel.Remote) + } + if out := Format(sel); !strings.Contains(out, "REMOTE ./remote.json") { + t.Errorf("Format should emit REMOTE, got:\n%s", out) + } + if _, err := Parse([]byte("PROVIDER x\nREMOTE a\nREMOTE b\n")); err == nil { + t.Error("duplicate REMOTE should error") + } +} diff --git a/internal/preset/preset.go b/internal/preset/preset.go index ea9809f4..407de260 100644 --- a/internal/preset/preset.go +++ b/internal/preset/preset.go @@ -236,6 +236,14 @@ func canonicalName(key string) string { return name } +// CanonicalKey resolves a preset key to the canonical long-form flag name that +// Flags would emit, so the same setting written different ways (`ngl` and +// `n-gpu-layers`, `c` and `ctx-size`) compares equal. Callers that need to +// match specific flags rather than render them should compare canonical names. +func CanonicalKey(key string) string { + return canonicalName(key) +} + // flagFor turns one preset key/value into its `llama-server` flag tokens. func flagFor(key, value string) []string { name := canonicalName(key) diff --git a/internal/remote/remote.go b/internal/remote/remote.go new file mode 100644 index 00000000..3270c485 --- /dev/null +++ b/internal/remote/remote.go @@ -0,0 +1,312 @@ +// Package remote controls the scale-to-zero GPU inference instance defined by +// this repository's remote/ subproject, by calling its Lambdas through their +// Function URLs. The URLs use IAM auth, so every request is SigV4-signed +// (service "lambda") with the caller's AWS credentials. +package remote + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + awsconfig "github.com/aws/aws-sdk-go-v2/config" +) + +// httpClient is a package variable so tests can substitute it. The long +// timeout matters: a start call blocks while the instance boots and loads the +// model into VRAM, which takes minutes. +var httpClient = &http.Client{Timeout: 10 * time.Minute} + +// Config holds the connection details for the remote instance's control +// Lambdas: deploying remote/ prints it as the OutfitRemoteConfig output, ready +// to paste into the config file. +type Config struct { + StartURL string `json:"start_url"` + StopURL string `json:"stop_url"` + // DeployURL is the deploy Lambda's Function URL. Unlike the other two it is + // optional: configs written before `remote deploy` existed do not have it, + // and start/stop work without it. Deploy reports its absence itself. + DeployURL string `json:"deploy_url"` + Region string `json:"region"` +} + +// ConfigPath returns the path of the remote config file, alongside outfit's +// own config in the same directory. +func ConfigPath() string { + dir := os.Getenv("XDG_CONFIG_HOME") + if dir == "" { + home, _ := os.UserHomeDir() + dir = filepath.Join(home, ".config") + } + return filepath.Join(dir, "outfit", "remote.json") +} + +// LoadConfig reads the per-user config file and applies environment overrides +// (OUTFIT_REMOTE_START_URL, OUTFIT_REMOTE_STOP_URL, OUTFIT_REMOTE_REGION; the +// region also falls back to AWS_REGION and then to the region embedded in the +// Function URL host). A missing file is fine — env vars alone can carry the +// config. getenv is injectable for tests. +func LoadConfig(getenv func(string) string) (Config, error) { + var cfg Config + data, err := os.ReadFile(ConfigPath()) + if err == nil { + if err := json.Unmarshal(data, &cfg); err != nil { + return Config{}, fmt.Errorf("parsing %s: %w", ConfigPath(), err) + } + } else if !os.IsNotExist(err) { + return Config{}, err + } + return finishConfig(cfg, getenv, ConfigPath()) +} + +// LoadConfigFile reads the remote config from an explicit file — typically +// one named by an Outfit's REMOTE instruction — then applies the same +// environment overrides as LoadConfig. Unlike LoadConfig, the file must +// exist: it was asked for by name. +func LoadConfigFile(path string, getenv func(string) string) (Config, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return Config{}, fmt.Errorf( + "remote config %s does not exist: paste the OutfitRemoteConfig output of the remote/ deployment there", + path) + } + return Config{}, err + } + var cfg Config + if err := json.Unmarshal(data, &cfg); err != nil { + return Config{}, fmt.Errorf("parsing %s: %w", path, err) + } + return finishConfig(cfg, getenv, path) +} + +// finishConfig applies env overrides and validates. source names the config +// file for error messages. +func finishConfig(cfg Config, getenv func(string) string, source string) (Config, error) { + if v := getenv("OUTFIT_REMOTE_START_URL"); v != "" { + cfg.StartURL = v + } + if v := getenv("OUTFIT_REMOTE_STOP_URL"); v != "" { + cfg.StopURL = v + } + if v := getenv("OUTFIT_REMOTE_DEPLOY_URL"); v != "" { + cfg.DeployURL = v + } + if v := getenv("OUTFIT_REMOTE_REGION"); v != "" { + cfg.Region = v + } + if cfg.StartURL == "" || cfg.StopURL == "" { + return Config{}, fmt.Errorf( + "remote is not configured: paste the OutfitRemoteConfig output of the remote/ deployment into %s", + source) + } + if cfg.Region == "" { + cfg.Region = getenv("AWS_REGION") + } + if cfg.Region == "" { + cfg.Region = regionFromURL(cfg.StartURL) + } + if cfg.Region == "" { + return Config{}, fmt.Errorf( + "cannot determine the AWS region: set \"region\" in %s or OUTFIT_REMOTE_REGION", + source) + } + return cfg, nil +} + +// regionFromURL extracts the region from a Lambda Function URL host +// (.lambda-url..on.aws). +func regionFromURL(rawURL string) string { + u, err := url.Parse(rawURL) + if err != nil { + return "" + } + parts := strings.Split(u.Hostname(), ".") + for i, part := range parts { + if part == "lambda-url" && i+1 < len(parts) { + return parts[i+1] + } + } + return "" +} + +// Response is the control Lambdas' JSON reply. +type Response struct { + StatusCode int `json:"-"` + State string `json:"state"` + Healthy *bool `json:"healthy"` + BaseURL string `json:"base_url"` + APIKey string `json:"api_key"` + Message string `json:"message"` + RetryAfterSeconds int `json:"retry_after_seconds"` + // Deploy-specific fields. + Deployed bool `json:"deployed"` + Seeding bool `json:"seeding"` + SeedInstanceID string `json:"seedInstanceId"` + Runner string `json:"runner"` + ModelID string `json:"modelId"` + ContextSize int `json:"contextSize"` + WeightsPrefix string `json:"weightsPrefix"` + Error string `json:"error"` +} + +// DeployConfig is what the deploy Lambda accepts: the runner-neutral +// description of WHAT to serve, derived from an Outfit. Deliberately no +// weights prefix — the Lambda derives the S3 layout itself, and seeds the +// weights when they are not there yet, so this stays a statement of intent. +type DeployConfig struct { + Runner string `json:"runner"` + ModelID string `json:"modelId"` + Quant string `json:"quant"` + ContextSize int `json:"contextSize"` + ServedModelName string `json:"servedModelName"` + ServeArgs []string `json:"serveArgs"` +} + +// Deploy sets what the next wake will serve. The Lambda validates the config, +// seeds the weights into S3 if they are absent, and stores it; deploying does +// not start the instance. +func Deploy(ctx context.Context, cfg Config, dc DeployConfig) (*Response, error) { + if cfg.DeployURL == "" { + return nil, fmt.Errorf( + "no deploy_url configured: add the remote/ deployment's DeployUrl output to the remote config (or set OUTFIT_REMOTE_DEPLOY_URL)") + } + body, err := json.Marshal(dc) + if err != nil { + return nil, err + } + resp, err := call(ctx, cfg, http.MethodPost, cfg.DeployURL, body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + detail := resp.Error + if detail == "" { + detail = resp.Message + } + return nil, fmt.Errorf("deploy failed (HTTP %d): %s", resp.StatusCode, detail) + } + return resp, nil +} + +// Start boots the instance and blocks until vLLM is serving, retrying while +// the endpoint reports it is still starting. progress is called with a status +// line before each wait. +func Start(ctx context.Context, cfg Config, progress func(string)) (*Response, error) { + for { + resp, err := call(ctx, cfg, http.MethodPost, cfg.StartURL, nil) + if err != nil { + return nil, err + } + switch { + case resp.StatusCode == http.StatusOK && resp.State == "ready": + return resp, nil + case resp.StatusCode == http.StatusServiceUnavailable: + wait := resp.RetryAfterSeconds + if wait <= 0 { + wait = 1 + } + progress(fmt.Sprintf("instance %s; retrying in %ds", resp.State, wait)) + select { + case <-ctx.Done(): + return nil, fmt.Errorf("gave up waiting for the endpoint: %w", ctx.Err()) + case <-time.After(time.Duration(wait) * time.Second): + } + default: + return nil, fmt.Errorf("start failed (HTTP %d, state %q): %s", + resp.StatusCode, resp.State, resp.Message) + } + } +} + +// Status reports the instance state and endpoint health without side effects. +func Status(ctx context.Context, cfg Config) (*Response, error) { + return call(ctx, cfg, http.MethodGet, cfg.StartURL, nil) +} + +// Stop stops the instance immediately rather than waiting for the idle timer. +func Stop(ctx context.Context, cfg Config) (*Response, error) { + return call(ctx, cfg, http.MethodPost, cfg.StopURL, nil) +} + +// call signs and sends one request. body is nil for the bodyless calls +// (start/stop/status); deploy passes JSON, which must be hashed into the +// signature rather than sent unsigned. +func call(ctx context.Context, cfg Config, method, rawURL string, body []byte) (*Response, error) { + var reader io.Reader + if body != nil { + reader = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, rawURL, reader) + if err != nil { + return nil, err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + // Set explicitly: with a bytes.Reader net/http would infer it, but the + // signature covers Content-Length, so leaving it to chance risks a + // mismatch between what is signed and what is sent. + req.ContentLength = int64(len(body)) + } + if err := sign(ctx, req, cfg.Region, body); err != nil { + return nil, err + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + respBody, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return nil, err + } + out := &Response{StatusCode: resp.StatusCode} + if err := json.Unmarshal(respBody, out); err != nil { + hint := "" + if resp.StatusCode == http.StatusForbidden { + hint = " (do your AWS credentials grant lambda:InvokeFunctionUrl?)" + } + return nil, fmt.Errorf("%s returned HTTP %d%s: %s", + method, resp.StatusCode, hint, truncate(string(respBody), 200)) + } + return out, nil +} + +// sign SigV4-signs the request with the default AWS credential chain +// (environment, shared config/credentials, SSO). Function URL IAM auth +// requires the payload hash to be sent and signed via X-Amz-Content-Sha256. +func sign(ctx context.Context, req *http.Request, region string, body []byte) error { + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) + if err != nil { + return err + } + creds, err := awsCfg.Credentials.Retrieve(ctx) + if err != nil { + return fmt.Errorf( + "resolving AWS credentials: %w (configure env credentials, a profile or an SSO session)", err) + } + // sha256 of the exact bytes sent (of the empty string for a bodyless call). + hash := sha256.Sum256(body) + payloadHash := hex.EncodeToString(hash[:]) + req.Header.Set("X-Amz-Content-Sha256", payloadHash) + return v4.NewSigner().SignHTTP(ctx, creds, req, payloadHash, "lambda", region, time.Now()) +} + +func truncate(s string, n int) string { + s = strings.TrimSpace(s) + if len(s) <= n { + return s + } + return s[:n] + "..." +} diff --git a/internal/remote/remote_test.go b/internal/remote/remote_test.go new file mode 100644 index 00000000..cd24aeae --- /dev/null +++ b/internal/remote/remote_test.go @@ -0,0 +1,295 @@ +package remote + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// isolateConfig sandboxes the config file location. +func isolateConfig(t *testing.T) string { + t.Helper() + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + return home +} + +// stubAWSEnv pins the default credential chain to static env credentials so +// signing works offline and no real profile, SSO session or IMDS is consulted. +func stubAWSEnv(t *testing.T) { + t.Helper() + t.Setenv("AWS_ACCESS_KEY_ID", "AKIATESTTESTTESTTEST") + t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret") + t.Setenv("AWS_SESSION_TOKEN", "") + t.Setenv("AWS_PROFILE", "") + t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "no-such-file")) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "no-such-file")) + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") +} + +func writeConfig(t *testing.T, cfg Config) { + t.Helper() + path := ConfigPath() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + data, err := json.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } +} + +func noEnv(string) string { return "" } + +func envMap(m map[string]string) func(string) string { + return func(k string) string { return m[k] } +} + +func TestConfigPath(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", "/tmp/xdg") + if got, want := ConfigPath(), "/tmp/xdg/outfit/remote.json"; got != want { + t.Errorf("ConfigPath() = %q, want %q", got, want) + } +} + +func TestLoadConfig_FromFile(t *testing.T) { + isolateConfig(t) + writeConfig(t, Config{ + StartURL: "https://start.example/", + StopURL: "https://stop.example/", + Region: "eu-west-1", + }) + cfg, err := LoadConfig(noEnv) + if err != nil { + t.Fatal(err) + } + if cfg.StartURL != "https://start.example/" || cfg.Region != "eu-west-1" { + t.Errorf("unexpected config: %+v", cfg) + } +} + +func TestLoadConfig_EnvOverrides(t *testing.T) { + isolateConfig(t) + writeConfig(t, Config{StartURL: "https://old/", StopURL: "https://old-stop/", Region: "us-east-1"}) + cfg, err := LoadConfig(envMap(map[string]string{ + "OUTFIT_REMOTE_START_URL": "https://new/", + "OUTFIT_REMOTE_REGION": "eu-west-2", + })) + if err != nil { + t.Fatal(err) + } + if cfg.StartURL != "https://new/" { + t.Errorf("env should override start URL, got %q", cfg.StartURL) + } + if cfg.StopURL != "https://old-stop/" { + t.Errorf("stored stop URL should survive, got %q", cfg.StopURL) + } + if cfg.Region != "eu-west-2" { + t.Errorf("env should override region, got %q", cfg.Region) + } +} + +func TestLoadConfig_RegionDerivedFromURL(t *testing.T) { + isolateConfig(t) + writeConfig(t, Config{ + StartURL: "https://abc123.lambda-url.eu-west-1.on.aws/", + StopURL: "https://def456.lambda-url.eu-west-1.on.aws/", + }) + cfg, err := LoadConfig(noEnv) + if err != nil { + t.Fatal(err) + } + if cfg.Region != "eu-west-1" { + t.Errorf("region should be derived from the URL, got %q", cfg.Region) + } +} + +func TestLoadConfig_Unconfigured(t *testing.T) { + isolateConfig(t) + _, err := LoadConfig(noEnv) + if err == nil || !strings.Contains(err.Error(), "not configured") { + t.Errorf("expected a not-configured error, got %v", err) + } +} + +func TestLoadConfig_NoRegion(t *testing.T) { + isolateConfig(t) + writeConfig(t, Config{StartURL: "https://start.example/", StopURL: "https://stop.example/"}) + _, err := LoadConfig(noEnv) + if err == nil || !strings.Contains(err.Error(), "region") { + t.Errorf("expected a region error, got %v", err) + } +} + +func TestRegionFromURL(t *testing.T) { + cases := map[string]string{ + "https://abc.lambda-url.eu-west-1.on.aws/": "eu-west-1", + "https://abc.lambda-url.us-east-2.on.aws": "us-east-2", + "https://example.com/": "", + "://bad": "", + } + for in, want := range cases { + if got := regionFromURL(in); got != want { + t.Errorf("regionFromURL(%q) = %q, want %q", in, got, want) + } + } +} + +func TestStart_RetriesUntilReady(t *testing.T) { + stubAWSEnv(t) + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("start should POST, got %s", r.Method) + } + if auth := r.Header.Get("Authorization"); !strings.HasPrefix(auth, "AWS4-HMAC-SHA256") { + t.Errorf("request is not SigV4-signed: %q", auth) + } + if r.Header.Get("X-Amz-Content-Sha256") == "" { + t.Error("X-Amz-Content-Sha256 header missing") + } + calls++ + w.Header().Set("Content-Type", "application/json") + if calls == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte(`{"state":"starting","retry_after_seconds":0}`)) + return + } + w.Write([]byte(`{"state":"ready","base_url":"http://198.51.100.1:8000/v1","api_key":"sk-test"}`)) + })) + defer server.Close() + + cfg := Config{StartURL: server.URL, StopURL: server.URL, Region: "eu-west-1"} + var progress []string + resp, err := Start(context.Background(), cfg, func(msg string) { progress = append(progress, msg) }) + if err != nil { + t.Fatal(err) + } + if resp.State != "ready" || resp.BaseURL != "http://198.51.100.1:8000/v1" || resp.APIKey != "sk-test" { + t.Errorf("unexpected response: %+v", resp) + } + if calls != 2 { + t.Errorf("expected 2 calls, got %d", calls) + } + if len(progress) != 1 || !strings.Contains(progress[0], "starting") { + t.Errorf("unexpected progress lines: %v", progress) + } +} + +func TestStart_Failure(t *testing.T) { + stubAWSEnv(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(`{"state":"terminated","message":"cannot start"}`)) + })) + defer server.Close() + + cfg := Config{StartURL: server.URL, StopURL: server.URL, Region: "eu-west-1"} + _, err := Start(context.Background(), cfg, func(string) {}) + if err == nil || !strings.Contains(err.Error(), "cannot start") { + t.Errorf("expected the server's message in the error, got %v", err) + } +} + +func TestStart_ContextDeadline(t *testing.T) { + stubAWSEnv(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte(`{"state":"starting","retry_after_seconds":60}`)) + })) + defer server.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + cfg := Config{StartURL: server.URL, StopURL: server.URL, Region: "eu-west-1"} + _, err := Start(ctx, cfg, func(string) {}) + if err == nil || !strings.Contains(err.Error(), "gave up") { + t.Errorf("expected a gave-up error, got %v", err) + } +} + +func TestStatusAndStop(t *testing.T) { + stubAWSEnv(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodGet { + w.Write([]byte(`{"state":"running","healthy":true,"base_url":"http://198.51.100.1:8000/v1"}`)) + return + } + w.Write([]byte(`{"state":"stopping"}`)) + })) + defer server.Close() + + cfg := Config{StartURL: server.URL, StopURL: server.URL, Region: "eu-west-1"} + status, err := Status(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + if status.State != "running" || status.Healthy == nil || !*status.Healthy { + t.Errorf("unexpected status: %+v", status) + } + + stop, err := Stop(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + if stop.State != "stopping" { + t.Errorf("unexpected stop response: %+v", stop) + } +} + +func TestCall_NonJSONError(t *testing.T) { + stubAWSEnv(t) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("Forbidden")) + })) + defer server.Close() + + cfg := Config{StartURL: server.URL, StopURL: server.URL, Region: "eu-west-1"} + _, err := Status(context.Background(), cfg) + if err == nil || !strings.Contains(err.Error(), "403") || + !strings.Contains(err.Error(), "lambda:InvokeFunctionUrl") { + t.Errorf("expected a 403 error with the IAM hint, got %v", err) + } +} + +func TestLoadConfigFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "remote.json") + content := `{"start_url":"https://start.example/","stop_url":"https://stop.example/","region":"eu-west-1"}` + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := LoadConfigFile(path, noEnv) + if err != nil { + t.Fatal(err) + } + if cfg.StartURL != "https://start.example/" || cfg.Region != "eu-west-1" { + t.Errorf("unexpected config: %+v", cfg) + } + + cfg, err = LoadConfigFile(path, envMap(map[string]string{"OUTFIT_REMOTE_REGION": "eu-west-2"})) + if err != nil { + t.Fatal(err) + } + if cfg.Region != "eu-west-2" { + t.Errorf("env should override the file's region, got %q", cfg.Region) + } +} + +func TestLoadConfigFile_Missing(t *testing.T) { + _, err := LoadConfigFile(filepath.Join(t.TempDir(), "remote.json"), noEnv) + if err == nil || !strings.Contains(err.Error(), "does not exist") { + t.Errorf("expected a does-not-exist error, got %v", err) + } +} diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/.openspec.yaml b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/.openspec.yaml new file mode 100644 index 00000000..e8209ffa --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-28 diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/design.md b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/design.md new file mode 100644 index 00000000..454a9aad --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/design.md @@ -0,0 +1,143 @@ +## Context + +Until now every command has been offline and local: read a catalogue, write a +harness config, or launch a server on this machine. Controlling a remote +endpoint breaks both properties — it makes network calls and it authenticates — +so the main design pressure is keeping that blast radius small enough that the +rest of the tool is unaffected. + +The endpoint itself is this repository's `remote/` subproject: a scale-to-zero +GPU instance fronted by control Lambdas. It is scale-to-zero precisely because +a GPU costs money by the hour, which is why starting and stopping are +first-class commands rather than something the user is trusted to remember. + +## Goals / Non-Goals + +**Goals:** + +- One declaration. The Outfit that describes a model locally describes the same + model remotely, so the name a coding agent asks for and the name the endpoint + serves cannot drift apart. +- Keep the AWS dependency confined to one package and one command group. +- Require no credentials of outfit's own: use the caller's AWS chain, and the + narrowest permission that works. +- Fail loudly and early. A misconfigured deployment should be a clear error + before anything is launched, not a server that boots and then cannot serve. + +**Non-Goals:** + +- Provisioning. This creates no infrastructure; the endpoint is deployed by its + own project, and outfit only tells it what to serve. +- Being a general AWS or cloud client. It calls three URLs. +- Modelling the endpoint's storage. Where weights live and how they get there + is the endpoint's business. +- Supporting arbitrary providers remotely. Only a self-hostable engine can be + deployed, because only those have something to deploy. + +## Decisions + +**`PROVIDER` names the engine; there is no `RUNNER` keyword.** An Outfit +already says `llamacpp`, and `outfit serve` already runs llama.cpp from it. The +remote case is the same statement pointed elsewhere, so the engine is read from +`PROVIDER` and the command decides local or remote. The alternative — a new +keyword — would let an Outfit say `PROVIDER llamacpp` with `RUNNER vllm`, a +contradiction worth making unrepresentable. The cost is that a provider which +is not a self-hosted engine cannot be deployed; that is an error, not a gap. + +**The deployment describes intent, not storage.** The request carries the +engine, model, quantisation, context, served name and engine flags — and +deliberately no storage location. The endpoint derives its own layout and +fetches the weights when they are missing. This keeps a storage-layout change +on the endpoint's side from becoming an outfit release, and means a caller +cannot point a deployment at the wrong place. + +**The preset is the single source of serving detail.** Rather than a second +list of remote flags, the existing preset is reused and the settings the +endpoint owns (host, port, model location, API key, context, alias, metrics) +are dropped from it. So one file serves locally and deploys unchanged. The +subtlety is that flags must be matched after alias resolution — `ngl` and +`n-gpu-layers` are the same setting — and that a preset's `[*]` defaults are a +separate layer that must be merged, or settings written once for every model +would silently vanish. + +**Signing covers the body.** The control URLs use IAM auth, so requests are +SigV4-signed. `deploy` is the first request with a payload, and the signature +covers the payload hash and `Content-Length`; both are therefore computed from +the exact bytes sent rather than assumed empty. + +**`deploy_url` is optional.** A configuration written before `deploy` existed +still works for `start`/`stop`/`status`; only `deploy` complains, and it says +what to add. Validating it up front would break working setups for a command +they do not use. + +**The command and the deployment ship together.** They are one feature — a +change to what the deploy request carries touches both sides — so they live in +one repository and are reviewed as one unit. The cost is a polyglot repository, +a TypeScript CDK project beside a Go CLI, handled by giving the deployment its +own workflow rather than bending the Go one around it. + +**Terminate, do not stop.** A stopped instance still bills for its disk, and +the endpoint is stateless — the weights are fetched at start — so there is +nothing worth keeping between sessions. Terminating means the resting cost is +storage and an address, and a forgotten session costs minutes rather than a +night. The price is a slower start, which the lifecycle bounds are designed +around rather than against. + +**Nothing identifying a deployment may be committed.** This repository is +public, so it is enforced by a check rather than a convention: the build fails +on an account id, ARN, endpoint hostname, public address, bucket name or +resource id. Documentation and tests use the reserved ranges instead. Logical +stack names and tag keys are deliberately *not* covered — they are chosen in +source and identify the running stack, so changing them would orphan it. + +**Progress belongs on stderr, results on stdout.** `start` blocks for minutes +while the model loads, so it has to report; but its useful output is two shell +exports. Splitting them means `eval "$(outfit remote start)"` works while the +person watching still sees progress. The alternative — progress on stdout, as it +was — forced a `grep` in the documented idiom and threw the progress away. + +**An optional API key is a catalogue fact, not a special case in code.** The +same provider covers a keyless local server and an authenticated remote one, so +the catalogue marks the key optional and the harness builders read that. The +alternative — inferring it from a localhost base URL, or always falling back to +a placeholder — would either be a guess or would break providers whose key is +genuinely resolved later. + +## Risks / Trade-offs + +- **A network- and credential-bearing dependency in an otherwise offline tool.** + → It is confined to `internal/remote` and reached only by `outfit remote`; + every other command still runs with no credentials and no network. +- **Deploying is not starting, so a deployment can be "live" while its weights + are still being fetched.** → The response says whether a fetch was started, + and the command tells the user to wait before starting. +- **The Outfit's context wins over the preset's, and human-friendly sizes are + decimal — `128k` is 128000, not 131072.** → Consistent with `outfit serve`, + where the same precedence already applies, so the two cannot disagree; the + exact value can always be written out in full. +- **Only one endpoint per Outfit.** → Deliberate: an Outfit describes one + selection, and a second endpoint is a second Outfit. +- **The catalogue is embedded at build time**, so a stale binary applies an old + catalogue and appears to ignore a fix. → Documented as a trap in AGENTS.md; + the runtime override reads a file instead. +- **A public repository now holds the deployment's source**, so a careless + example could disclose a real account. → The identifier check runs first in + CI, and its patterns were verified against real samples of each class. +- **The keys the launched agent receives are wider than one provider's.** → + Only variables outfit can already resolve are passed, never invented, and + anything already in the environment is left as it is. + +## Migration Plan + +Additive throughout: an Outfit with no `REMOTE` behaves exactly as before, and +no existing command changes behaviour. Rollback is dropping the command group; +nothing persists on the user's machine beyond the configuration file they +wrote. + +## Open Questions + +- Whether `status` should also report the deployed configuration (the endpoint + can return it) or stay a liveness check. +- Whether a future harness-facing command should read the served model name + back from the endpoint rather than trusting `ALIAS`, once more than one + Outfit can target the same endpoint. diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/proposal.md b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/proposal.md new file mode 100644 index 00000000..3a90c96c --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/proposal.md @@ -0,0 +1,131 @@ +## Why + +A model too large for a laptop has to run somewhere else, but the moment it +does, the Outfit stops being the whole story: the endpoint has to be started +before use and stopped after, and something has to tell it which model to load. +Doing that by hand — cloud console, then a config edit, then remembering to +shut it down — is exactly the fiddling `outfit` exists to remove, and it drifts: +the model the endpoint serves and the model the harness asks for are maintained +in two places and quietly disagree. + +An Outfit already describes a model precisely enough to serve it, which +`outfit serve` proves locally. The same description can drive a remote engine, +making local and cloud the same declaration pointed at a different machine — so +this change delivers both halves: the command, and the deployment it commands. + +## What Changes + +- A new `outfit remote` command group — the CLI's first nested subcommand — + with `start`, `stop`, `status` and `deploy`. It targets the scale-to-zero + endpoint defined by this repository's `remote/` subproject, calling its + control Lambdas over SigV4-signed Function URLs. +- A new `REMOTE` Outfit instruction naming the file that holds those URLs, + resolved relative to the Outfit like `PRESET`, so an Outfit and the endpoint + it belongs to travel together. Without it, a per-user config is used, so + `outfit remote` still works outside a project. +- `outfit remote deploy` derives what to serve from the Outfit and its preset: + `PROVIDER` selects the engine, `MODEL` or the preset's `hf` the weights, + `CONTEXT` or the preset's `ctx-size` the window, `ALIAS` the served name, and + the preset's remaining flags the engine's own arguments. Settings the remote + owns (host, port, model path, API key, context, alias, metrics) are dropped, + so one preset serves locally and deploys unchanged. +- A `vllm` provider in the catalogue, so both self-hostable engines can be named + by `PROVIDER` and swapping between them is an edit to one line. +- An `apiKeyOptional` flag on a catalogue provider, for one that works both + unauthenticated (a local server) and authenticated (the same engine deployed + remotely). `llamacpp` becomes such a provider. +- Tab completion for a nested command: `outfit remote ` offers the + subcommands, and the argument after one completes as an Outfit. +- `start` reports progress. The endpoint blocks until the model is serving, so a + cold start previously sat silent for minutes; it now says what it is doing and + repeats with the elapsed time. Progress goes to stderr and only the exports to + stdout, so the output stays pipeable. +- Applying a config that cannot authenticate is called out. A remote endpoint + with no resolvable key succeeded silently and failed later as a rejected + request; it now warns, naming the variable to set. +- **A deployment that runs the model**, under `remote/`: an endpoint that holds + no instance until asked for one, launches into whichever zone has GPU + capacity, and terminates itself once unused — so it costs nothing at rest and + cannot quietly run all night. A retention marker, a maximum runtime and a + grace period bound it in that order. +- **A choice of inference engine**, llama.cpp or vLLM, made per deployment + rather than baked in. What to serve — engine, model, quantisation, context, + engine flags — is a single stored configuration the endpoint reads when it + starts, so changing model is a deploy, not a rebuild. +- **Weights kept outside the machine image**, in object storage, at a location + derived from the engine, model and quantisation. Deploying a model whose + weights are absent fetches them first, so nothing has to be staged by hand. +- Nothing identifying a deployment may be committed: account ids, ARNs, + endpoint hostnames, allocated addresses, bucket names, resource ids. This + repository is public, so `scripts/check-no-cloud-identifiers.sh` fails the + build on any of them, and a deployment's own state stays in files that are + never committed. +- **BREAKING** for opencode users who relied on the config carrying the key: + the key is now referenced as `{env:VAR}` and resolved when opencode runs, so + the variable must be set — `outfit harness` passes on whatever outfit can + resolve, and an explicit export always wins. + +Everything else is additive: every new instruction and command is new surface, +and an Outfit without `REMOTE` behaves exactly as before. The one behavioural +change is the opencode key above — a deliberate reversal of the previous choice +to embed the secret, which was justified by a global config being unable to rely +on a project-local `.env`. Passing resolved keys to the launched agent removes +that justification, and stops writing a secret to disk. + +## Capabilities + +### New Capabilities + +- `remote-endpoint`: controlling a remote inference endpoint from an Outfit — + discovering its configuration, starting and stopping it, reporting its state, + and deploying what it serves. +- `endpoint-lifecycle`: an endpoint that exists only while it is used — + starting on demand, judging whether it is still wanted, and the bounds that + decide when it is torn down. +- `inference-runners`: which engine serves a deployment and how it is + configured, including the stored description of what to serve. +- `model-weights`: where a model's weights live, how their location is derived, + and how they get there. + +### Modified Capabilities + +- `outfit-files`: the instruction set gains `REMOTE`, and Outfit path + resolution now also covers the `remote` subcommands. +- `provider-catalog`: a provider entry may declare that its API key is + optional. +- `pi-integration`: the keyless-provider placeholder is now conditioned on the + endpoint being local, so a remote endpoint keeps the reference Pi resolves at + run time. +- `opencode-integration`: the API key is referenced as `{env:VAR}` rather than + embedded, so no secret is written to disk. +- `harness-management`: the launched agent inherits the API keys outfit can + resolve, since neither harness stores the secret itself. +- `provider-selection`: a key's `.env` is resolved beside the Outfit being + applied rather than relative to the tool. +- `shell-completion`: completion covers a command's subcommands, not only its + flags and positionals. + +## Impact + +- **Code**: new `internal/remote` (the only package making network calls or + using the AWS SDK) and `cmd/outfit/remote.go`; the Outfit parser, the + catalogue schema, both harness adapters, the key resolver, and the completion + table. The deployment under `remote/` — a CDK application, and the + repository's only non-Go code. +- **CI**: a `No cloud identifiers` step ahead of everything else, and a separate + `Remote deployment` workflow that typechecks, tests and synthesizes `remote/` + when it changes, so the deployment cannot rot untested. +- **Dependencies**: `aws-sdk-go-v2` (config + SigV4 signer) — the project's + first non-stdlib runtime dependency outside YAML parsing. It is reached only + by `outfit remote`; every other command stays offline. +- **Credentials**: `outfit remote` uses the caller's own AWS credential chain + and needs only `lambda:InvokeFunctionUrl`. No AWS permissions are needed by + any other command, and none are stored by outfit. +- **Internal contract**: the deploy payload is consumed by the deployment's own + deploy function, which owns the storage layout and fetching the weights. The + CLI states intent; where anything is stored is not its business, which is why + the payload names no location. +- **Cost**: the deployment bills for GPU time only while an instance is up, so + the lifecycle bounds are the difference between minutes and hours of it. +- The implementation for this change already exists on the `feat/remote` + branch; this change records the specification it introduced. diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/endpoint-lifecycle/spec.md b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/endpoint-lifecycle/spec.md new file mode 100644 index 00000000..dbe38c13 --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/endpoint-lifecycle/spec.md @@ -0,0 +1,98 @@ +## ADDED Requirements + +### Requirement: Starting on demand + +The endpoint SHALL hold no running instance when idle. A start request SHALL +launch one, and SHALL try each configured availability zone in turn until one +has capacity, since GPU capacity is not guaranteed in any single zone. The +instance SHALL be given a stable address so the endpoint's URL does not change +between launches, and the request SHALL NOT report success until the model is +answering — the caller receives one "ready", never a URL that is not yet +serving. When no capacity can be found anywhere, the response SHALL say so and +SHALL be retryable rather than fatal. + +#### Scenario: A zone without capacity is not the end of it + +- **WHEN** the first availability zone cannot provide the instance type +- **THEN** the remaining zones are tried before reporting failure + +#### Scenario: Ready means serving + +- **WHEN** a start request returns success +- **THEN** the model is answering requests at the reported address + +#### Scenario: No capacity anywhere + +- **WHEN** every configured zone is out of capacity +- **THEN** the response says so and indicates the caller may retry shortly + +#### Scenario: Nothing has been deployed + +- **WHEN** a start is requested before any configuration has been deployed +- **THEN** it fails saying what to deploy, rather than launching an instance + with nothing to serve + +### Requirement: Stopping when unused + +A running instance SHALL be **terminated**, not stopped, once unused, so that +no storage is billed while the endpoint is idle. Activity SHALL be judged from +the inference server's own counters, read on the instance, and SHALL account +for both requests in flight and work that started and finished between two +checks. Because the metric names differ per inference engine, the check SHALL +read the names belonging to the engine that is deployed. + +A failed reading SHALL be treated as no activity rather than as activity, so a +wedged server is terminated rather than left running indefinitely. + +#### Scenario: Idle for longer than the threshold + +- **WHEN** no activity is observed for the configured idle period +- **THEN** the instance is terminated + +#### Scenario: A long generation is not mistaken for idleness + +- **WHEN** a single request runs across two checks without any request being in + flight at the moment either is taken +- **THEN** the moved token counters count as activity and the instance is kept + +#### Scenario: The server has stopped responding + +- **WHEN** the activity reading fails +- **THEN** the instance is still terminated once the idle period passes + +### Requirement: Bounds on a running instance + +The following SHALL take precedence over one another in this order, so that the +stronger guarantee always wins: + +1. A **retention override** — an instance marked to be retained until a stated + time SHALL NOT be terminated automatically before it, for any reason. +2. A **maximum runtime** — an instance SHALL be terminated once it has run + longer than the configured maximum, **even while requests are in flight**, + as a backstop against a session nobody is watching. +3. A **grace period** — an instance SHALL NOT be terminated for idleness within + the configured period after launch, which covers loading the model. + +A manual stop SHALL take effect immediately regardless of all three. + +#### Scenario: Retention beats the maximum runtime + +- **WHEN** an instance is marked retained until a future time and has also + exceeded the maximum runtime +- **THEN** it is kept, and the reason given is the retention + +#### Scenario: The maximum runtime beats activity + +- **WHEN** an instance has run longer than the maximum and requests are still + in flight +- **THEN** it is terminated + +#### Scenario: Loading the model is not idleness + +- **WHEN** an instance is still within the grace period and reports no activity +- **THEN** it is kept + +#### Scenario: A manual stop is not delayed + +- **WHEN** a stop is requested for a retained instance +- **THEN** it is terminated diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/harness-management/spec.md b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/harness-management/spec.md new file mode 100644 index 00000000..19db50c7 --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/harness-management/spec.md @@ -0,0 +1,28 @@ +## ADDED Requirements + +### Requirement: Keys reach the launched agent + +When outfit launches a harness, the agent's environment SHALL carry the API key +variables outfit can resolve for the catalogue's providers, so a key kept where +only outfit reads it still reaches the agent. Neither harness stores the secret +itself — each resolves a reference when it runs — so without this the user would +have to set the variable by hand. A variable already present in outfit's own +environment SHALL be passed through unchanged, so an explicit setting always +wins. Failure to read the catalogue SHALL NOT prevent the launch. + +#### Scenario: A key only outfit can see still reaches the agent + +- **WHEN** outfit can resolve a provider's key variable but it is absent from + the environment, and the harness is launched +- **THEN** the launched agent's environment carries that variable + +#### Scenario: An explicit setting is not overridden + +- **WHEN** the variable is already set in the environment and outfit can also + resolve a different value +- **THEN** the launched agent sees the environment's value + +#### Scenario: An unreadable catalogue still launches the agent + +- **WHEN** the provider catalogue cannot be loaded +- **THEN** the harness is launched anyway, with the environment unchanged diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/inference-runners/spec.md b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/inference-runners/spec.md new file mode 100644 index 00000000..de95925e --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/inference-runners/spec.md @@ -0,0 +1,82 @@ +## ADDED Requirements + +### Requirement: Choosing an inference engine + +A deployment SHALL serve through one of a fixed set of inference engines. There +SHALL be no default: an engine is a deliberate choice, and a deployment whose +engine is unset or unrecognised SHALL fail loudly rather than assume one, at +the point the configuration is accepted and again when an instance is started. + +Each engine SHALL have its own machine image, identified by the engine it was +built for, so that starting an instance uses an image carrying that engine. +Both images SHALL remain available, so moving a deployment from one engine to +the other is a matter of what is deployed, not of rebuilding anything. + +#### Scenario: No engine chosen + +- **WHEN** a configuration naming no engine is deployed +- **THEN** it is rejected saying which engines are accepted + +#### Scenario: The image matches the engine + +- **WHEN** an instance is started for a deployment +- **THEN** it is launched from the most recent image built for that + deployment's engine + +#### Scenario: Changing engine + +- **WHEN** a deployment naming the other engine is applied +- **THEN** the next start serves through that engine, with no image rebuilt + +### Requirement: What to serve is stored, not built in + +What a deployment serves — its engine, model, quantisation, context window, the +name it serves under, and the engine's own arguments — SHALL be held as a +single stored configuration, read when an instance starts. Changing any of it +SHALL therefore take effect on the next start without redeploying the +infrastructure. + +That configuration SHALL be owned by whoever deploys it, and SHALL NOT be +overwritten by deploying the infrastructure itself, so a routine deployment +cannot silently replace what is being served. An instance started before +anything has been configured SHALL fail saying so. + +#### Scenario: Changing model does not rebuild anything + +- **WHEN** a configuration naming a different model is deployed +- **THEN** the next start serves that model, with no image or infrastructure + change + +#### Scenario: Deploying the infrastructure preserves what is served + +- **WHEN** the infrastructure is deployed again after a configuration has been + set +- **THEN** the stored configuration is left exactly as it was + +#### Scenario: Started before being configured + +- **WHEN** an instance is started while no configuration has been stored +- **THEN** it fails saying that something must be deployed first + +### Requirement: Building the engine's command + +The engine's command line SHALL be derived from the stored configuration. The +settings that belong to the deployment rather than the model — the address and +port to listen on, where the weights are on disk, the API key, the context +window, the served name, and metrics — SHALL be set by the deployment itself, +and the configuration's remaining arguments SHALL be passed through unchanged. + +The API key SHALL be given to the engine by reference to a file readable only +by the owner, never as a command-line argument, so it does not appear in the +machine's process list. + +#### Scenario: The deployment's own settings are not taken from the request + +- **WHEN** a configuration's arguments include a listen address or a context + window +- **THEN** the deployment's values are used for them + +#### Scenario: The key is not visible in the process list + +- **WHEN** the engine is started with an API key +- **THEN** the key is passed by reference to an owner-only file diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/model-weights/spec.md b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/model-weights/spec.md new file mode 100644 index 00000000..ab2a1f6d --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/model-weights/spec.md @@ -0,0 +1,75 @@ +## ADDED Requirements + +### Requirement: Weights live outside the machine image + +A model's weights SHALL be held in object storage rather than baked into the +machine image, so that changing model requires no new image and an image stays +useful for any model. An instance SHALL copy them onto its own disk when it +starts, and SHALL therefore depend on no third party at start time. + +Because they are copied at start, the storage SHALL be in the same region as +the instance, so a start transfers nothing billable. + +#### Scenario: Changing model does not rebuild the image + +- **WHEN** a deployment names a model the current image was never built for +- **THEN** the instance serves it, having copied the weights at start + +### Requirement: Where weights live is derived, never supplied + +The location of a model's weights SHALL be derived from the engine, the model +and the quantisation, so the same model always resolves to the same place and +two engines' copies of one model never collide. A location supplied by a +caller SHALL be ignored: whoever deploys states what to serve, and the +deployment decides where that is kept. + +#### Scenario: The same model resolves to the same place + +- **WHEN** the same engine, model and quantisation are deployed twice +- **THEN** both resolve to the same location + +#### Scenario: Two engines do not collide + +- **WHEN** the same model is deployed for each of two engines +- **THEN** each has its own location + +#### Scenario: A supplied location is ignored + +- **WHEN** a deployment request names a storage location +- **THEN** it has no effect, and the derived location is used + +### Requirement: Fetching weights that are absent + +Deploying a model whose weights are not in storage SHALL fetch them, without +anyone staging them by hand. The fetch SHALL run within the deployment rather +than from the caller's machine, and SHALL dispose of whatever ran it once the +weights are stored. + +Presence SHALL be judged by a marker written only once a fetch has completed, +so a fetch that failed or is still running is not mistaken for weights that are +ready. + +A deployment SHALL NOT be stored if its weights are absent and a fetch cannot +be started, so a configuration that could never serve does not replace one that +can. The reply SHALL say whether a fetch was started, because an instance +started before it finishes would copy an incomplete model. + +#### Scenario: Weights already present + +- **WHEN** a deployment names a model whose weights are stored +- **THEN** nothing is fetched and the deployment is stored + +#### Scenario: Weights absent + +- **WHEN** a deployment names a model whose weights are absent +- **THEN** a fetch is started, and the reply says so + +#### Scenario: A half-finished fetch is not mistaken for success + +- **WHEN** an earlier fetch failed part way, leaving some files behind +- **THEN** the weights are treated as absent and fetched again + +#### Scenario: A deployment that could never serve is refused + +- **WHEN** the weights are absent and a fetch cannot be started +- **THEN** the deployment is rejected and the stored configuration is unchanged diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/opencode-integration/spec.md b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/opencode-integration/spec.md new file mode 100644 index 00000000..35f702ed --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/opencode-integration/spec.md @@ -0,0 +1,41 @@ +## MODIFIED Requirements + +### Requirement: Secrets and file permissions + +The API key SHALL be written to the provider block's options as an +environment-variable reference in opencode's `{env:VAR}` form, naming the +provider's key variable — never the resolved secret, so no secret is written to +disk. The reference SHALL be written even when the variable is currently unset, +because opencode substitutes it when it reads the config, so the key may be set +after the Outfit is applied. The option SHALL be omitted entirely when the +provider declares no key variable, or when its key variable is declared +optional and the provider's endpoint is a local address — a local server needs +no key, and naming a variable nobody will set would only mislead. + +The config SHALL still be written with owner-only (`0600`) permissions — +enforced even when the file already existed with looser ones — since it names +the user's providers and endpoints. + +#### Scenario: The key is referenced, not embedded + +- **WHEN** a provider with an API key variable is applied +- **THEN** the config's `apiKey` is that variable's `{env:VAR}` reference, and + the resolved secret appears nowhere in the file + +#### Scenario: A key set after applying still works + +- **WHEN** an Outfit is applied with the key variable unset, and the variable is + set before opencode runs +- **THEN** opencode resolves the reference to that value + +#### Scenario: A local keyless server gets no key option + +- **WHEN** a provider whose key is optional is applied against a local base URL + with its key variable unset +- **THEN** the provider block carries no `apiKey` option + +#### Scenario: Permissions enforced on existing file + +- **WHEN** the config file exists with permissive mode and `outfit add` writes + it +- **THEN** the file's mode is owner-only afterwards diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/outfit-files/spec.md b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/outfit-files/spec.md new file mode 100644 index 00000000..f384d7e9 --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/outfit-files/spec.md @@ -0,0 +1,64 @@ +## MODIFIED Requirements + +### Requirement: Outfit file format + +An Outfit SHALL be a flat, line-oriented text file of `KEYWORD value` +instructions. The keywords are `PROVIDER`, `FAMILY`, `MODEL`, `ALIAS`, +`CONTEXT`, `OUTPUT`, `BASEURL` (also accepted as `BASE-URL`, `BASE_URL`, or +`URL`), `PRESET`, and `REMOTE`. Keywords SHALL match case-insensitively, with +UPPERCASE as the canonical form. Blank lines, full-line `#` comments, and +trailing comments introduced by whitespace-then-`#` SHALL be ignored. Each +instruction SHALL take exactly one value and SHALL appear at most once. +`PROVIDER` is required. Parse errors SHALL name the offending line. + +#### Scenario: A minimal Outfit + +- **WHEN** a file containing only `PROVIDER openrouter` and + `FAMILY deepseek-v4` is parsed +- **THEN** it yields a selection of that provider and family + +#### Scenario: Duplicate instruction + +- **WHEN** an Outfit sets `MODEL` on two lines +- **THEN** parsing fails, citing both line numbers + +#### Scenario: Unknown keyword + +- **WHEN** an Outfit contains `HARNESS pi` +- **THEN** parsing fails listing the accepted keywords + +#### Scenario: Missing provider + +- **WHEN** an Outfit has no `PROVIDER` instruction +- **THEN** parsing fails saying the PROVIDER instruction is missing + +#### Scenario: Naming a remote endpoint + +- **WHEN** an Outfit contains `REMOTE ./remote.json` +- **THEN** it parses, and the value is available to the `remote` command group + +### Requirement: Outfit path resolution + +Commands that take an Outfit path (`apply`, `unapply`, `serve`, `alias`, +`harness --outfit`, and the `remote` subcommands) SHALL default to `./Outfit` +when no path is given, SHALL accept a directory and use the `Outfit` file +inside it, and SHALL accept a registered alias name in place of a path. When +the default `./Outfit` is missing, the error SHALL suggest passing a path or an +alias. + +#### Scenario: Bare command in a project directory + +- **WHEN** the user runs `outfit apply` in a directory holding an `Outfit` +- **THEN** that file is applied + +#### Scenario: Directory argument + +- **WHEN** the user runs `outfit apply path/to/dir` and the directory holds an + `Outfit` +- **THEN** `path/to/dir/Outfit` is applied + +#### Scenario: A remote subcommand resolves the same way + +- **WHEN** the user runs `outfit remote status` in a directory holding an + `Outfit` +- **THEN** that Outfit is read to find the endpoint's configuration diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/pi-integration/spec.md b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/pi-integration/spec.md new file mode 100644 index 00000000..228c6cbd --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/pi-integration/spec.md @@ -0,0 +1,51 @@ +## MODIFIED Requirements + +### Requirement: API key idiom + +The managed provider's `apiKey` SHALL be written as a `$ENV_VAR` interpolation +referencing the provider's key variable — never the resolved secret. The +reference SHALL be written even when that variable is currently unset, because +Pi resolves it when it runs, so the key may be exported after the Outfit is +applied. + +A dummy literal placeholder SHALL be written instead when the provider has no +key variable at all, or when its key variable is declared optional, resolves to +nothing, AND the provider's endpoint is a local address — because Pi hides a +provider's models from `/model` until some auth is configured, and for a local +server, which ignores the value, a reference to a variable set nowhere would +leave them hidden. The placeholder SHALL NOT be written for a non-local +endpoint, since Pi resolves the reference when it runs and a placeholder could +not then be repaired by setting the variable. + +#### Scenario: Keyed provider references the variable + +- **WHEN** an OpenRouter selection is applied to Pi +- **THEN** the entry's `apiKey` is the literal string `$DEEPSEEK_API_KEY`-style + reference, not the key's value + +#### Scenario: Keyless provider gets a placeholder + +- **WHEN** an Ollama or llama.cpp selection is applied to Pi +- **THEN** the entry's `apiKey` is a dummy literal so the models are selectable + in `/model` + +#### Scenario: An optional key that is set is referenced + +- **WHEN** a provider whose key is optional is applied to Pi with its key + variable set +- **THEN** the entry's `apiKey` is the `$ENV_VAR` reference, so the remote + endpoint is authenticated + +#### Scenario: An optional key at a remote endpoint keeps its reference + +- **WHEN** a provider whose key is optional is applied to Pi against a non-local + base URL, with its key variable unset +- **THEN** the entry's `apiKey` is the `$ENV_VAR` reference, so setting the + variable before running Pi is enough + +#### Scenario: A required key survives being unset at apply time + +- **WHEN** a provider whose key is not optional is applied to Pi with its key + variable unset +- **THEN** the entry's `apiKey` is still the `$ENV_VAR` reference, so exporting + the key later is enough diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/provider-catalog/spec.md b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/provider-catalog/spec.md new file mode 100644 index 00000000..c3edfc3c --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/provider-catalog/spec.md @@ -0,0 +1,39 @@ +## MODIFIED Requirements + +### Requirement: Embedded provider catalogue + +The system SHALL ship with a built-in catalogue of providers and model +families, defined in a YAML file (`providers.yaml`) embedded into the binary at +build time. Each provider entry declares a description and MAY declare a +display name, an npm package, an API key environment variable (with optional +required flag, optional flag, and expected key prefix), static options (such as +`baseURL`), options resolved from environment variables, a `pi` block marking +the provider as usable by the Pi harness, and named model families. Each family +declares a description, a default model, and a set of models. + +A provider whose API key is declared optional is one that also works +unauthenticated — the same engine run as a local server and as an +authenticated remote endpoint — so an unset key variable SHALL mean "no key", +not "a key that is missing". + +#### Scenario: Catalogue loads without external files + +- **WHEN** any command that needs the catalogue runs with no `--providers` flag + and no `OUTFIT_PROVIDERS` environment variable +- **THEN** the embedded catalogue is used, with no file read from disk + +#### Scenario: Family default model is always a member of the family + +- **WHEN** the catalogue defines a model family +- **THEN** that family's `defaultModel` is one of the family's `models` keys + +#### Scenario: An optional key is injected only when set + +- **WHEN** a provider whose API key is optional is applied, with its key + variable set +- **THEN** the key is injected into the harness config + +#### Scenario: An optional key that is unset is not an error + +- **WHEN** the same provider is applied with the key variable unset +- **THEN** the configuration is written with no key, and the command succeeds diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/provider-selection/spec.md b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/provider-selection/spec.md new file mode 100644 index 00000000..3bb08fc8 --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/provider-selection/spec.md @@ -0,0 +1,45 @@ +## MODIFIED Requirements + +### Requirement: API key resolution + +When a provider declares an API key environment variable, the system SHALL +resolve its value from a `.env` file **beside the Outfit being applied** first, +then from the process environment. When no Outfit is involved — a selection made +entirely from flags — the working directory SHALL be used in its place, so a +project's own `.env` is still found. A missing key SHALL be an error when the +provider marks it required. A resolved key that does not start with the +provider's declared prefix SHALL be rejected. +Secrets SHALL never be written into an Outfit file. + +The file sits beside the Outfit for the same reason `PRESET` and `REMOTE` do: +an Outfit and the key it needs belong to the same project and travel together, +and a location relative to the tool cannot be resolved by an installed binary. + +#### Scenario: The key travels with the Outfit + +- **WHEN** an Outfit is applied and a `.env` beside it sets the provider's key + variable, which is unset in the environment +- **THEN** that value is used + +#### Scenario: A command with no Outfit reads the working directory + +- **WHEN** a selection made entirely from flags is applied and a `.env` in the + working directory sets the provider's key variable +- **THEN** that value is used + +#### Scenario: Another project's .env is not consulted + +- **WHEN** an Outfit is applied and the key variable is set only in a `.env` + belonging to a different directory +- **THEN** the key does not resolve from that file + +#### Scenario: Required key missing + +- **WHEN** the user adds a provider whose key is required and the variable is + set in neither `.env` nor the environment +- **THEN** the command fails naming the variable to set + +#### Scenario: Malformed key + +- **WHEN** the resolved key does not start with the provider's declared prefix +- **THEN** the command fails saying the key does not look right diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/remote-endpoint/spec.md b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/remote-endpoint/spec.md new file mode 100644 index 00000000..da2fc891 --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/remote-endpoint/spec.md @@ -0,0 +1,139 @@ +## ADDED Requirements + +### Requirement: Remote command group + +The system SHALL provide a `remote` command group with the subcommands `start`, +`stop`, `status` and `deploy`, each taking an optional Outfit path. `start` +SHALL boot the endpoint and block until it is serving, printing the base URL +and API key as shell exports; `stop` SHALL stop it immediately rather than +waiting for its idle timer; `status` SHALL report instance state and endpoint +health without side effects; `deploy` SHALL set what the endpoint serves. An +unrecognised subcommand SHALL fail naming the accepted ones. + +#### Scenario: Starting the endpoint + +- **WHEN** the user runs `outfit remote start` and the endpoint reports ready +- **THEN** the base URL and API key are printed as `export` lines + +#### Scenario: Waiting through a cold start + +- **WHEN** the endpoint reports that it is still starting +- **THEN** the command waits and retries until it is ready or the timeout + passes, rather than failing on the first attempt + +#### Scenario: Unknown subcommand + +- **WHEN** the user runs `outfit remote frobnicate` +- **THEN** the command fails listing the accepted subcommands + +### Requirement: Reporting a start in progress + +Because the endpoint blocks until the model is serving, `start` SHALL report +that it is waiting rather than appear to hang: it SHALL say what is happening +before the first attempt and repeat at intervals with the elapsed time, and +SHALL report how long it took once ready. Progress SHALL be written to standard +error and the resulting exports to standard output, so the command's output can +be evaluated directly while a person watching still sees progress. + +#### Scenario: A cold start is not silent + +- **WHEN** the endpoint takes minutes to become ready +- **THEN** the command explains what it is waiting for and continues to report + the elapsed time until it succeeds + +#### Scenario: Only the result is on standard output + +- **WHEN** a start succeeds and its output is captured +- **THEN** standard output holds exactly the environment exports, with every + progress line on standard error + +### Requirement: Remote configuration discovery + +The endpoint's control URLs SHALL come from a JSON configuration naming a start +URL, a stop URL, an optional deploy URL, and a region. An Outfit's `REMOTE` +instruction SHALL select that file, resolved relative to the Outfit when the +value is not absolute. When no Outfit names one, the per-user configuration +SHALL be used, so the command works outside any project. Environment variables +SHALL override individual values, and the region SHALL fall back to the +standard AWS region variable and then to the region named in the URL. A missing +or incomplete configuration SHALL fail saying where to put it. + +#### Scenario: Outfit names the configuration + +- **WHEN** an Outfit sets `REMOTE ./remote.json` and a `remote` subcommand runs + with that Outfit +- **THEN** the URLs come from that file, resolved beside the Outfit + +#### Scenario: Explicit Outfit without a REMOTE instruction + +- **WHEN** a `remote` subcommand is given an Outfit that has no `REMOTE` +- **THEN** it fails saying that Outfit has no `REMOTE` instruction, rather than + silently using the per-user configuration + +#### Scenario: No Outfit in play + +- **WHEN** a `remote` subcommand runs outside a project +- **THEN** the per-user configuration is used + +### Requirement: Authenticated control requests + +Requests to the control URLs SHALL be signed with the caller's own AWS +credentials, resolved from the standard credential chain, and SHALL carry the +hash of the request body so that a request with a payload is signed over that +payload. Outfit SHALL NOT store AWS credentials of its own. A rejected request +SHALL report that the credentials may lack permission to invoke the endpoint. + +#### Scenario: A request carrying a body is signed over it + +- **WHEN** `outfit remote deploy` sends a configuration +- **THEN** the request is signed including the body's hash, not as an empty + payload + +#### Scenario: Credentials are missing + +- **WHEN** no AWS credentials can be resolved +- **THEN** the command fails saying how to configure them + +### Requirement: Deploying what the endpoint serves + +`outfit remote deploy` SHALL derive the deployment from the Outfit and its +preset: `PROVIDER` SHALL select the inference engine, `MODEL` or the preset's +Hugging Face reference SHALL name the weights as a repository and optional +quantisation, `CONTEXT` or the preset's context size SHALL set the window, +`ALIAS` SHALL set the name the endpoint serves under (defaulting to the +repository), and the preset's remaining settings SHALL become the engine's +arguments. Settings the endpoint owns — host, port, model location, API key, +context size, alias, and metrics — SHALL be dropped, so one preset can both +serve locally and deploy unchanged. The request SHALL describe only what to +serve, never where the weights are stored. A `--dry-run` SHALL print the +derived deployment without sending it. + +#### Scenario: A preset drives both serving and deploying + +- **WHEN** an Outfit with a preset is deployed +- **THEN** the engine's arguments are the preset's, minus the settings the + endpoint sets itself + +#### Scenario: The Outfit overrides its preset + +- **WHEN** the Outfit states a `MODEL` and `CONTEXT` that differ from the + preset's +- **THEN** the Outfit's values are deployed + +#### Scenario: A provider that is not a self-hosted engine + +- **WHEN** an Outfit naming a hosted provider is deployed +- **THEN** the command fails saying that only a self-hosted engine can be + deployed + +#### Scenario: A local model file + +- **WHEN** an Outfit naming a local model file is deployed +- **THEN** the command fails saying to name a repository instead, because the + endpoint fetches its own weights + +#### Scenario: Deploying is not starting + +- **WHEN** a deployment succeeds +- **THEN** the endpoint is configured but not started, and the report says + whether the weights still have to be fetched before it can serve diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/shell-completion/spec.md b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/shell-completion/spec.md new file mode 100644 index 00000000..02341dc6 --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/specs/shell-completion/spec.md @@ -0,0 +1,40 @@ +## MODIFIED Requirements + +### Requirement: Completion coverage + +Completion SHALL cover the full visible command surface: command names (the +hidden `__complete` excluded), each command's flags, its subcommands where it +has them, and context-aware values — provider names from the resolved catalogue +(honouring a `--providers` override already on the line), family and model +names scoped to the `--provider` (and `--model-family`) already typed, harness +names, registered alias names where an Outfit path is accepted, and the +supported shells for `completion`. For a command with subcommands, the first +positional slot SHALL offer those subcommands and any later slot SHALL fall +through to what the command otherwise accepts. Positional slots beyond a +command's arity SHALL offer nothing. + +#### Scenario: Families scoped to the typed provider + +- **WHEN** the user completes `outfit add -p openrouter -f ` +- **THEN** only the families of `openrouter` are offered + +#### Scenario: Unalias offers exactly the registered names + +- **WHEN** the user completes `outfit unalias ` +- **THEN** the registered alias names are offered with no file paths + +#### Scenario: New commands cannot be forgotten + +- **WHEN** a new subcommand is added to the CLI's dispatch +- **THEN** the completion surface must list it too (enforced by a test that + scans the dispatch) + +#### Scenario: A nested command offers its subcommands + +- **WHEN** the user completes `outfit remote ` +- **THEN** its subcommands are offered, with no file paths + +#### Scenario: After a subcommand, the Outfit slot completes + +- **WHEN** the user completes `outfit remote deploy ` +- **THEN** registered alias names and paths are offered diff --git a/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/tasks.md b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/tasks.md new file mode 100644 index 00000000..07028322 --- /dev/null +++ b/openspec/changes/archive/2026-07-29-add-remote-endpoint-control/tasks.md @@ -0,0 +1,64 @@ +All tasks are already implemented on the `feat/remote` branch; this change +records the specification delta that work introduced, so the boxes are ticked. + +## 1. Outfit format + +- [x] 1.1 Add the `REMOTE` keyword to the Outfit parser and `Selection`, keeping it out of `apply`'s business +- [x] 1.2 Render `REMOTE` in `Format` so `outfit export` round-trips it +- [x] 1.3 Cover the new keyword in the parser tests, including the duplicate and unknown-keyword paths + +## 2. Remote transport + +- [x] 2.1 Add `internal/remote` with the configuration type, its file and environment resolution, and region fallback +- [x] 2.2 SigV4-sign requests from the caller's credential chain, hashing the exact bytes sent so a request with a body signs over it +- [x] 2.3 Implement `Start` (retrying while the endpoint reports it is still starting), `Stop` and `Status` +- [x] 2.4 Implement `Deploy`, with `deploy_url` optional so existing configurations keep working for the other subcommands +- [x] 2.5 Test the flows against an `httptest` server with a pinned credential chain + +## 3. Command group + +- [x] 3.1 Add the `remote` dispatch with `start`/`stop`/`status`, resolving the configuration from an Outfit's `REMOTE` or the per-user file +- [x] 3.2 Add `deploy`, mapping the Outfit and its preset to a deployment: provider to engine, model or preset `hf` to weights, context, alias, and the remaining preset flags +- [x] 3.3 Drop the settings the endpoint owns, matching on canonical flag names, and layer the preset's `[*]` defaults under the chosen section +- [x] 3.4 Add `--dry-run`, and report whether the endpoint has to fetch weights before it can serve +- [x] 3.5 Test the mapping, its rejections, and that a deployment is posted and signed + +## 4. Catalogue and harnesses + +- [x] 4.1 Add a `vllm` provider entry +- [x] 4.2 Add `apiKeyOptional` to the provider schema and mark `llamacpp` with it, so the remote endpoint's key is injected when set +- [x] 4.3 Write Pi's keyless placeholder when an optional key resolves to nothing, keeping the `$VAR` reference for every other provider +- [x] 4.4 Test both directions for an optional key, and that a non-optional key keeps its reference when unset + +## 5. Completion and documentation + +- [x] 5.1 Teach the completion table about subcommands and add `remote`, satisfying the dispatch-coverage test +- [x] 5.2 Document the `REMOTE` instruction in the user documentation +- [x] 5.3 Give `outfit remote` a page in the user manual and list it in the index +- [x] 5.4 Update AGENTS.md with the new packages, and record the traps found along the way + +## 6. Usable output + +- [x] 6.1 Report progress while `start` waits, with the elapsed time, since the endpoint blocks until the model is serving +- [x] 6.2 Put progress on stderr and only the exports on stdout, so the result stays pipeable +- [x] 6.3 Explain `remote` in the usage text, as every other command is + +## 7. Keys that actually work + +- [x] 7.1 Warn when a config is written with no resolvable key for a non-local endpoint +- [x] 7.2 Write opencode's key as an `{env:VAR}` reference so no secret reaches disk +- [x] 7.3 Keep Pi's reference for a remote endpoint, since a placeholder cannot be repaired later +- [x] 7.4 Pass the keys outfit can resolve to the agent it launches +- [x] 7.5 Resolve `.env` beside the Outfit, falling back to the working directory when there is none + +## 8. The deployment + +- [x] 8.1 Start on demand: try each zone for capacity, take a stable address, and report ready only once the model answers +- [x] 8.2 Terminate when unused, judging activity from the engine's own counters and reading the names its engine exposes +- [x] 8.3 Order the bounds — retention, then maximum runtime, then the grace period — and let a manual stop through regardless +- [x] 8.4 Serve through a chosen engine, with an image per engine and no default +- [x] 8.5 Hold what to serve as one stored configuration, owned by whoever deploys and not overwritten by deploying the infrastructure +- [x] 8.6 Derive where weights live from engine, model and quantisation, ignoring any location a caller supplies +- [x] 8.7 Fetch absent weights within the deployment, judging presence by a marker written only on completion +- [x] 8.8 Add `scripts/check-no-cloud-identifiers.sh` and run it first in CI +- [x] 8.9 Add a `Remote deployment` workflow: install, typecheck, test, and synthesize diff --git a/openspec/specs/endpoint-lifecycle/spec.md b/openspec/specs/endpoint-lifecycle/spec.md new file mode 100644 index 00000000..58c461f6 --- /dev/null +++ b/openspec/specs/endpoint-lifecycle/spec.md @@ -0,0 +1,106 @@ +# Endpoint Lifecycle Specification + +## Purpose + +Define when the remote endpoint's instance exists — how it is started on +demand, how it is judged to be still wanted, and the bounds that decide +when it is torn down. + +## Requirements + +### Requirement: Starting on demand + +The endpoint SHALL hold no running instance when idle. A start request SHALL +launch one, and SHALL try each configured availability zone in turn until one +has capacity, since GPU capacity is not guaranteed in any single zone. The +instance SHALL be given a stable address so the endpoint's URL does not change +between launches, and the request SHALL NOT report success until the model is +answering — the caller receives one "ready", never a URL that is not yet +serving. When no capacity can be found anywhere, the response SHALL say so and +SHALL be retryable rather than fatal. + +#### Scenario: A zone without capacity is not the end of it + +- **WHEN** the first availability zone cannot provide the instance type +- **THEN** the remaining zones are tried before reporting failure + +#### Scenario: Ready means serving + +- **WHEN** a start request returns success +- **THEN** the model is answering requests at the reported address + +#### Scenario: No capacity anywhere + +- **WHEN** every configured zone is out of capacity +- **THEN** the response says so and indicates the caller may retry shortly + +#### Scenario: Nothing has been deployed + +- **WHEN** a start is requested before any configuration has been deployed +- **THEN** it fails saying what to deploy, rather than launching an instance + with nothing to serve + +### Requirement: Stopping when unused + +A running instance SHALL be **terminated**, not stopped, once unused, so that +no storage is billed while the endpoint is idle. Activity SHALL be judged from +the inference server's own counters, read on the instance, and SHALL account +for both requests in flight and work that started and finished between two +checks. Because the metric names differ per inference engine, the check SHALL +read the names belonging to the engine that is deployed. + +A failed reading SHALL be treated as no activity rather than as activity, so a +wedged server is terminated rather than left running indefinitely. + +#### Scenario: Idle for longer than the threshold + +- **WHEN** no activity is observed for the configured idle period +- **THEN** the instance is terminated + +#### Scenario: A long generation is not mistaken for idleness + +- **WHEN** a single request runs across two checks without any request being in + flight at the moment either is taken +- **THEN** the moved token counters count as activity and the instance is kept + +#### Scenario: The server has stopped responding + +- **WHEN** the activity reading fails +- **THEN** the instance is still terminated once the idle period passes + +### Requirement: Bounds on a running instance + +The following SHALL take precedence over one another in this order, so that the +stronger guarantee always wins: + +1. A **retention override** — an instance marked to be retained until a stated + time SHALL NOT be terminated automatically before it, for any reason. +2. A **maximum runtime** — an instance SHALL be terminated once it has run + longer than the configured maximum, **even while requests are in flight**, + as a backstop against a session nobody is watching. +3. A **grace period** — an instance SHALL NOT be terminated for idleness within + the configured period after launch, which covers loading the model. + +A manual stop SHALL take effect immediately regardless of all three. + +#### Scenario: Retention beats the maximum runtime + +- **WHEN** an instance is marked retained until a future time and has also + exceeded the maximum runtime +- **THEN** it is kept, and the reason given is the retention + +#### Scenario: The maximum runtime beats activity + +- **WHEN** an instance has run longer than the maximum and requests are still + in flight +- **THEN** it is terminated + +#### Scenario: Loading the model is not idleness + +- **WHEN** an instance is still within the grace period and reports no activity +- **THEN** it is kept + +#### Scenario: A manual stop is not delayed + +- **WHEN** a stop is requested for a retained instance +- **THEN** it is terminated diff --git a/openspec/specs/harness-management/spec.md b/openspec/specs/harness-management/spec.md index e45e7569..5b4f8595 100644 --- a/openspec/specs/harness-management/spec.md +++ b/openspec/specs/harness-management/spec.md @@ -139,3 +139,30 @@ to `outfit add`, not an error. - **WHEN** the harness config holds no providers - **THEN** show prints the harness, config path, and a hint to run `outfit add` + +### Requirement: Keys reach the launched agent + +When outfit launches a harness, the agent's environment SHALL carry the API key +variables outfit can resolve for the catalogue's providers, so a key kept where +only outfit reads it still reaches the agent. Neither harness stores the secret +itself — each resolves a reference when it runs — so without this the user would +have to set the variable by hand. A variable already present in outfit's own +environment SHALL be passed through unchanged, so an explicit setting always +wins. Failure to read the catalogue SHALL NOT prevent the launch. + +#### Scenario: A key only outfit can see still reaches the agent + +- **WHEN** outfit can resolve a provider's key variable but it is absent from + the environment, and the harness is launched +- **THEN** the launched agent's environment carries that variable + +#### Scenario: An explicit setting is not overridden + +- **WHEN** the variable is already set in the environment and outfit can also + resolve a different value +- **THEN** the launched agent sees the environment's value + +#### Scenario: An unreadable catalogue still launches the agent + +- **WHEN** the provider catalogue cannot be loaded +- **THEN** the harness is launched anyway, with the environment unchanged diff --git a/openspec/specs/inference-runners/spec.md b/openspec/specs/inference-runners/spec.md new file mode 100644 index 00000000..fbd00326 --- /dev/null +++ b/openspec/specs/inference-runners/spec.md @@ -0,0 +1,89 @@ +# Inference Runners Specification + +## Purpose + +Define which inference engine serves a deployment, how it is chosen and +configured, and how the engine's command is built. + +## Requirements + +### Requirement: Choosing an inference engine + +A deployment SHALL serve through one of a fixed set of inference engines. There +SHALL be no default: an engine is a deliberate choice, and a deployment whose +engine is unset or unrecognised SHALL fail loudly rather than assume one, at +the point the configuration is accepted and again when an instance is started. + +Each engine SHALL have its own machine image, identified by the engine it was +built for, so that starting an instance uses an image carrying that engine. +Both images SHALL remain available, so moving a deployment from one engine to +the other is a matter of what is deployed, not of rebuilding anything. + +#### Scenario: No engine chosen + +- **WHEN** a configuration naming no engine is deployed +- **THEN** it is rejected saying which engines are accepted + +#### Scenario: The image matches the engine + +- **WHEN** an instance is started for a deployment +- **THEN** it is launched from the most recent image built for that + deployment's engine + +#### Scenario: Changing engine + +- **WHEN** a deployment naming the other engine is applied +- **THEN** the next start serves through that engine, with no image rebuilt + +### Requirement: What to serve is stored, not built in + +What a deployment serves — its engine, model, quantisation, context window, the +name it serves under, and the engine's own arguments — SHALL be held as a +single stored configuration, read when an instance starts. Changing any of it +SHALL therefore take effect on the next start without redeploying the +infrastructure. + +That configuration SHALL be owned by whoever deploys it, and SHALL NOT be +overwritten by deploying the infrastructure itself, so a routine deployment +cannot silently replace what is being served. An instance started before +anything has been configured SHALL fail saying so. + +#### Scenario: Changing model does not rebuild anything + +- **WHEN** a configuration naming a different model is deployed +- **THEN** the next start serves that model, with no image or infrastructure + change + +#### Scenario: Deploying the infrastructure preserves what is served + +- **WHEN** the infrastructure is deployed again after a configuration has been + set +- **THEN** the stored configuration is left exactly as it was + +#### Scenario: Started before being configured + +- **WHEN** an instance is started while no configuration has been stored +- **THEN** it fails saying that something must be deployed first + +### Requirement: Building the engine's command + +The engine's command line SHALL be derived from the stored configuration. The +settings that belong to the deployment rather than the model — the address and +port to listen on, where the weights are on disk, the API key, the context +window, the served name, and metrics — SHALL be set by the deployment itself, +and the configuration's remaining arguments SHALL be passed through unchanged. + +The API key SHALL be given to the engine by reference to a file readable only +by the owner, never as a command-line argument, so it does not appear in the +machine's process list. + +#### Scenario: The deployment's own settings are not taken from the request + +- **WHEN** a configuration's arguments include a listen address or a context + window +- **THEN** the deployment's values are used for them + +#### Scenario: The key is not visible in the process list + +- **WHEN** the engine is started with an API key +- **THEN** the key is passed by reference to an owner-only file diff --git a/openspec/specs/model-weights/spec.md b/openspec/specs/model-weights/spec.md new file mode 100644 index 00000000..da5a6da3 --- /dev/null +++ b/openspec/specs/model-weights/spec.md @@ -0,0 +1,82 @@ +# Model Weights Specification + +## Purpose + +Define where a model's weights live, how their location is derived, and +how they come to be there. + +## Requirements + +### Requirement: Weights live outside the machine image + +A model's weights SHALL be held in object storage rather than baked into the +machine image, so that changing model requires no new image and an image stays +useful for any model. An instance SHALL copy them onto its own disk when it +starts, and SHALL therefore depend on no third party at start time. + +Because they are copied at start, the storage SHALL be in the same region as +the instance, so a start transfers nothing billable. + +#### Scenario: Changing model does not rebuild the image + +- **WHEN** a deployment names a model the current image was never built for +- **THEN** the instance serves it, having copied the weights at start + +### Requirement: Where weights live is derived, never supplied + +The location of a model's weights SHALL be derived from the engine, the model +and the quantisation, so the same model always resolves to the same place and +two engines' copies of one model never collide. A location supplied by a +caller SHALL be ignored: whoever deploys states what to serve, and the +deployment decides where that is kept. + +#### Scenario: The same model resolves to the same place + +- **WHEN** the same engine, model and quantisation are deployed twice +- **THEN** both resolve to the same location + +#### Scenario: Two engines do not collide + +- **WHEN** the same model is deployed for each of two engines +- **THEN** each has its own location + +#### Scenario: A supplied location is ignored + +- **WHEN** a deployment request names a storage location +- **THEN** it has no effect, and the derived location is used + +### Requirement: Fetching weights that are absent + +Deploying a model whose weights are not in storage SHALL fetch them, without +anyone staging them by hand. The fetch SHALL run within the deployment rather +than from the caller's machine, and SHALL dispose of whatever ran it once the +weights are stored. + +Presence SHALL be judged by a marker written only once a fetch has completed, +so a fetch that failed or is still running is not mistaken for weights that are +ready. + +A deployment SHALL NOT be stored if its weights are absent and a fetch cannot +be started, so a configuration that could never serve does not replace one that +can. The reply SHALL say whether a fetch was started, because an instance +started before it finishes would copy an incomplete model. + +#### Scenario: Weights already present + +- **WHEN** a deployment names a model whose weights are stored +- **THEN** nothing is fetched and the deployment is stored + +#### Scenario: Weights absent + +- **WHEN** a deployment names a model whose weights are absent +- **THEN** a fetch is started, and the reply says so + +#### Scenario: A half-finished fetch is not mistaken for success + +- **WHEN** an earlier fetch failed part way, leaving some files behind +- **THEN** the weights are treated as absent and fetched again + +#### Scenario: A deployment that could never serve is refused + +- **WHEN** the weights are absent and a fetch cannot be started +- **THEN** the deployment is rejected and the stored configuration is unchanged diff --git a/openspec/specs/opencode-integration/spec.md b/openspec/specs/opencode-integration/spec.md index 433d2734..fd9ba4ed 100644 --- a/openspec/specs/opencode-integration/spec.md +++ b/openspec/specs/opencode-integration/spec.md @@ -68,10 +68,37 @@ it alone otherwise. ### Requirement: Secrets and file permissions -The resolved API key SHALL be embedded in the provider block's options (a -global config cannot rely on a project-local `.env`), and the config SHALL be -written with owner-only (`0600`) permissions — enforced even when the file -already existed with looser ones. +The API key SHALL be written to the provider block's options as an +environment-variable reference in opencode's `{env:VAR}` form, naming the +provider's key variable — never the resolved secret, so no secret is written to +disk. The reference SHALL be written even when the variable is currently unset, +because opencode substitutes it when it reads the config, so the key may be set +after the Outfit is applied. The option SHALL be omitted entirely when the +provider declares no key variable, or when its key variable is declared +optional and the provider's endpoint is a local address — a local server needs +no key, and naming a variable nobody will set would only mislead. + +The config SHALL still be written with owner-only (`0600`) permissions — +enforced even when the file already existed with looser ones — since it names +the user's providers and endpoints. + +#### Scenario: The key is referenced, not embedded + +- **WHEN** a provider with an API key variable is applied +- **THEN** the config's `apiKey` is that variable's `{env:VAR}` reference, and + the resolved secret appears nowhere in the file + +#### Scenario: A key set after applying still works + +- **WHEN** an Outfit is applied with the key variable unset, and the variable is + set before opencode runs +- **THEN** opencode resolves the reference to that value + +#### Scenario: A local keyless server gets no key option + +- **WHEN** a provider whose key is optional is applied against a local base URL + with its key variable unset +- **THEN** the provider block carries no `apiKey` option #### Scenario: Permissions enforced on existing file diff --git a/openspec/specs/outfit-files/spec.md b/openspec/specs/outfit-files/spec.md index 8950b7f1..c66c6b52 100644 --- a/openspec/specs/outfit-files/spec.md +++ b/openspec/specs/outfit-files/spec.md @@ -13,11 +13,11 @@ provider selection — and the commands that consume and produce it: An Outfit SHALL be a flat, line-oriented text file of `KEYWORD value` instructions. The keywords are `PROVIDER`, `FAMILY`, `MODEL`, `ALIAS`, `CONTEXT`, `OUTPUT`, `BASEURL` (also accepted as `BASE-URL`, `BASE_URL`, or -`URL`), and `PRESET`. Keywords SHALL match case-insensitively, with UPPERCASE -as the canonical form. Blank lines, full-line `#` comments, and trailing -comments introduced by whitespace-then-`#` SHALL be ignored. Each instruction -SHALL take exactly one value and SHALL appear at most once. `PROVIDER` is -required. Parse errors SHALL name the offending line. +`URL`), `PRESET`, and `REMOTE`. Keywords SHALL match case-insensitively, with +UPPERCASE as the canonical form. Blank lines, full-line `#` comments, and +trailing comments introduced by whitespace-then-`#` SHALL be ignored. Each +instruction SHALL take exactly one value and SHALL appear at most once. +`PROVIDER` is required. Parse errors SHALL name the offending line. #### Scenario: A minimal Outfit @@ -40,6 +40,11 @@ required. Parse errors SHALL name the offending line. - **WHEN** an Outfit has no `PROVIDER` instruction - **THEN** parsing fails saying the PROVIDER instruction is missing +#### Scenario: Naming a remote endpoint + +- **WHEN** an Outfit contains `REMOTE ./remote.json` +- **THEN** it parses, and the value is available to the `remote` command group + ### Requirement: Harness neutrality An Outfit SHALL NOT name a harness, and SHALL NOT name an alias-registry entry: @@ -56,10 +61,11 @@ applicable to any supported harness. ### Requirement: Outfit path resolution Commands that take an Outfit path (`apply`, `unapply`, `serve`, `alias`, -`harness --outfit`) SHALL default to `./Outfit` when no path is given, SHALL -accept a directory and use the `Outfit` file inside it, and SHALL accept a -registered alias name in place of a path. When the default `./Outfit` is -missing, the error SHALL suggest passing a path or an alias. +`harness --outfit`, and the `remote` subcommands) SHALL default to `./Outfit` +when no path is given, SHALL accept a directory and use the `Outfit` file +inside it, and SHALL accept a registered alias name in place of a path. When +the default `./Outfit` is missing, the error SHALL suggest passing a path or an +alias. #### Scenario: Bare command in a project directory @@ -72,6 +78,12 @@ missing, the error SHALL suggest passing a path or an alias. `Outfit` - **THEN** `path/to/dir/Outfit` is applied +#### Scenario: A remote subcommand resolves the same way + +- **WHEN** the user runs `outfit remote status` in a directory holding an + `Outfit` +- **THEN** that Outfit is read to find the endpoint's configuration + ### Requirement: Applying and unapplying an Outfit `outfit apply` SHALL apply the Outfit's selection exactly as the equivalent diff --git a/openspec/specs/pi-integration/spec.md b/openspec/specs/pi-integration/spec.md index 0ad5a747..a3e6877e 100644 --- a/openspec/specs/pi-integration/spec.md +++ b/openspec/specs/pi-integration/spec.md @@ -46,10 +46,19 @@ sorted by id so the file is deterministic. ### Requirement: API key idiom The managed provider's `apiKey` SHALL be written as a `$ENV_VAR` interpolation -referencing the provider's key variable — never the resolved secret. A keyless -local provider SHALL get a dummy literal placeholder instead, because Pi hides -a provider's models from `/model` until some auth is configured; local servers -ignore the value. +referencing the provider's key variable — never the resolved secret. The +reference SHALL be written even when that variable is currently unset, because +Pi resolves it when it runs, so the key may be exported after the Outfit is +applied. + +A dummy literal placeholder SHALL be written instead when the provider has no +key variable at all, or when its key variable is declared optional, resolves to +nothing, AND the provider's endpoint is a local address — because Pi hides a +provider's models from `/model` until some auth is configured, and for a local +server, which ignores the value, a reference to a variable set nowhere would +leave them hidden. The placeholder SHALL NOT be written for a non-local +endpoint, since Pi resolves the reference when it runs and a placeholder could +not then be repaired by setting the variable. #### Scenario: Keyed provider references the variable @@ -63,6 +72,27 @@ ignore the value. - **THEN** the entry's `apiKey` is a dummy literal so the models are selectable in `/model` +#### Scenario: An optional key that is set is referenced + +- **WHEN** a provider whose key is optional is applied to Pi with its key + variable set +- **THEN** the entry's `apiKey` is the `$ENV_VAR` reference, so the remote + endpoint is authenticated + +#### Scenario: An optional key at a remote endpoint keeps its reference + +- **WHEN** a provider whose key is optional is applied to Pi against a non-local + base URL, with its key variable unset +- **THEN** the entry's `apiKey` is the `$ENV_VAR` reference, so setting the + variable before running Pi is enough + +#### Scenario: A required key survives being unset at apply time + +- **WHEN** a provider whose key is not optional is applied to Pi with its key + variable unset +- **THEN** the entry's `apiKey` is still the `$ENV_VAR` reference, so exporting + the key later is enough + ### Requirement: Pi capability gating A provider SHALL be usable with the Pi harness only when the catalogue gives it diff --git a/openspec/specs/provider-catalog/spec.md b/openspec/specs/provider-catalog/spec.md index 5b74105b..5e214dc5 100644 --- a/openspec/specs/provider-catalog/spec.md +++ b/openspec/specs/provider-catalog/spec.md @@ -15,10 +15,15 @@ The system SHALL ship with a built-in catalogue of providers and model families, defined in a YAML file (`providers.yaml`) embedded into the binary at build time. Each provider entry declares a description and MAY declare a display name, an npm package, an API key environment variable (with optional -required flag and expected key prefix), static options (such as `baseURL`), -options resolved from environment variables, a `pi` block marking the provider -as usable by the Pi harness, and named model families. Each family declares a -description, a default model, and a set of models. +required flag, optional flag, and expected key prefix), static options (such as +`baseURL`), options resolved from environment variables, a `pi` block marking +the provider as usable by the Pi harness, and named model families. Each family +declares a description, a default model, and a set of models. + +A provider whose API key is declared optional is one that also works +unauthenticated — the same engine run as a local server and as an +authenticated remote endpoint — so an unset key variable SHALL mean "no key", +not "a key that is missing". #### Scenario: Catalogue loads without external files @@ -31,6 +36,17 @@ description, a default model, and a set of models. - **WHEN** the catalogue defines a model family - **THEN** that family's `defaultModel` is one of the family's `models` keys +#### Scenario: An optional key is injected only when set + +- **WHEN** a provider whose API key is optional is applied, with its key + variable set +- **THEN** the key is injected into the harness config + +#### Scenario: An optional key that is unset is not an error + +- **WHEN** the same provider is applied with the key variable unset +- **THEN** the configuration is written with no key, and the command succeeds + ### Requirement: Catalogue listing `outfit list` SHALL print every provider in the catalogue in stable diff --git a/openspec/specs/provider-selection/spec.md b/openspec/specs/provider-selection/spec.md index 1240b391..912b17c7 100644 --- a/openspec/specs/provider-selection/spec.md +++ b/openspec/specs/provider-selection/spec.md @@ -91,10 +91,35 @@ configures. ### Requirement: API key resolution When a provider declares an API key environment variable, the system SHALL -resolve its value from a `.env` file next to the tool first, then from the -process environment. A missing key SHALL be an error when the provider marks it -required. A resolved key that does not start with the provider's declared -prefix SHALL be rejected. Secrets SHALL never be written into an Outfit file. +resolve its value from a `.env` file **beside the Outfit being applied** first, +then from the process environment. When no Outfit is involved — a selection made +entirely from flags — the working directory SHALL be used in its place, so a +project's own `.env` is still found. A missing key SHALL be an error when the +provider marks it required. A resolved key that does not start with the +provider's declared prefix SHALL be rejected. +Secrets SHALL never be written into an Outfit file. + +The file sits beside the Outfit for the same reason `PRESET` and `REMOTE` do: +an Outfit and the key it needs belong to the same project and travel together, +and a location relative to the tool cannot be resolved by an installed binary. + +#### Scenario: The key travels with the Outfit + +- **WHEN** an Outfit is applied and a `.env` beside it sets the provider's key + variable, which is unset in the environment +- **THEN** that value is used + +#### Scenario: A command with no Outfit reads the working directory + +- **WHEN** a selection made entirely from flags is applied and a `.env` in the + working directory sets the provider's key variable +- **THEN** that value is used + +#### Scenario: Another project's .env is not consulted + +- **WHEN** an Outfit is applied and the key variable is set only in a `.env` + belonging to a different directory +- **THEN** the key does not resolve from that file #### Scenario: Required key missing diff --git a/openspec/specs/remote-endpoint/spec.md b/openspec/specs/remote-endpoint/spec.md new file mode 100644 index 00000000..46cc9614 --- /dev/null +++ b/openspec/specs/remote-endpoint/spec.md @@ -0,0 +1,146 @@ +# Remote Endpoint Specification + +## Purpose + +Define how a remote inference endpoint is discovered, controlled and told +what to serve from an Outfit: the `outfit remote` command group. + +## Requirements + +### Requirement: Remote command group + +The system SHALL provide a `remote` command group with the subcommands `start`, +`stop`, `status` and `deploy`, each taking an optional Outfit path. `start` +SHALL boot the endpoint and block until it is serving, printing the base URL +and API key as shell exports; `stop` SHALL stop it immediately rather than +waiting for its idle timer; `status` SHALL report instance state and endpoint +health without side effects; `deploy` SHALL set what the endpoint serves. An +unrecognised subcommand SHALL fail naming the accepted ones. + +#### Scenario: Starting the endpoint + +- **WHEN** the user runs `outfit remote start` and the endpoint reports ready +- **THEN** the base URL and API key are printed as `export` lines + +#### Scenario: Waiting through a cold start + +- **WHEN** the endpoint reports that it is still starting +- **THEN** the command waits and retries until it is ready or the timeout + passes, rather than failing on the first attempt + +#### Scenario: Unknown subcommand + +- **WHEN** the user runs `outfit remote frobnicate` +- **THEN** the command fails listing the accepted subcommands + +### Requirement: Reporting a start in progress + +Because the endpoint blocks until the model is serving, `start` SHALL report +that it is waiting rather than appear to hang: it SHALL say what is happening +before the first attempt and repeat at intervals with the elapsed time, and +SHALL report how long it took once ready. Progress SHALL be written to standard +error and the resulting exports to standard output, so the command's output can +be evaluated directly while a person watching still sees progress. + +#### Scenario: A cold start is not silent + +- **WHEN** the endpoint takes minutes to become ready +- **THEN** the command explains what it is waiting for and continues to report + the elapsed time until it succeeds + +#### Scenario: Only the result is on standard output + +- **WHEN** a start succeeds and its output is captured +- **THEN** standard output holds exactly the environment exports, with every + progress line on standard error + +### Requirement: Remote configuration discovery + +The endpoint's control URLs SHALL come from a JSON configuration naming a start +URL, a stop URL, an optional deploy URL, and a region. An Outfit's `REMOTE` +instruction SHALL select that file, resolved relative to the Outfit when the +value is not absolute. When no Outfit names one, the per-user configuration +SHALL be used, so the command works outside any project. Environment variables +SHALL override individual values, and the region SHALL fall back to the +standard AWS region variable and then to the region named in the URL. A missing +or incomplete configuration SHALL fail saying where to put it. + +#### Scenario: Outfit names the configuration + +- **WHEN** an Outfit sets `REMOTE ./remote.json` and a `remote` subcommand runs + with that Outfit +- **THEN** the URLs come from that file, resolved beside the Outfit + +#### Scenario: Explicit Outfit without a REMOTE instruction + +- **WHEN** a `remote` subcommand is given an Outfit that has no `REMOTE` +- **THEN** it fails saying that Outfit has no `REMOTE` instruction, rather than + silently using the per-user configuration + +#### Scenario: No Outfit in play + +- **WHEN** a `remote` subcommand runs outside a project +- **THEN** the per-user configuration is used + +### Requirement: Authenticated control requests + +Requests to the control URLs SHALL be signed with the caller's own AWS +credentials, resolved from the standard credential chain, and SHALL carry the +hash of the request body so that a request with a payload is signed over that +payload. Outfit SHALL NOT store AWS credentials of its own. A rejected request +SHALL report that the credentials may lack permission to invoke the endpoint. + +#### Scenario: A request carrying a body is signed over it + +- **WHEN** `outfit remote deploy` sends a configuration +- **THEN** the request is signed including the body's hash, not as an empty + payload + +#### Scenario: Credentials are missing + +- **WHEN** no AWS credentials can be resolved +- **THEN** the command fails saying how to configure them + +### Requirement: Deploying what the endpoint serves + +`outfit remote deploy` SHALL derive the deployment from the Outfit and its +preset: `PROVIDER` SHALL select the inference engine, `MODEL` or the preset's +Hugging Face reference SHALL name the weights as a repository and optional +quantisation, `CONTEXT` or the preset's context size SHALL set the window, +`ALIAS` SHALL set the name the endpoint serves under (defaulting to the +repository), and the preset's remaining settings SHALL become the engine's +arguments. Settings the endpoint owns — host, port, model location, API key, +context size, alias, and metrics — SHALL be dropped, so one preset can both +serve locally and deploy unchanged. The request SHALL describe only what to +serve, never where the weights are stored. A `--dry-run` SHALL print the +derived deployment without sending it. + +#### Scenario: A preset drives both serving and deploying + +- **WHEN** an Outfit with a preset is deployed +- **THEN** the engine's arguments are the preset's, minus the settings the + endpoint sets itself + +#### Scenario: The Outfit overrides its preset + +- **WHEN** the Outfit states a `MODEL` and `CONTEXT` that differ from the + preset's +- **THEN** the Outfit's values are deployed + +#### Scenario: A provider that is not a self-hosted engine + +- **WHEN** an Outfit naming a hosted provider is deployed +- **THEN** the command fails saying that only a self-hosted engine can be + deployed + +#### Scenario: A local model file + +- **WHEN** an Outfit naming a local model file is deployed +- **THEN** the command fails saying to name a repository instead, because the + endpoint fetches its own weights + +#### Scenario: Deploying is not starting + +- **WHEN** a deployment succeeds +- **THEN** the endpoint is configured but not started, and the report says + whether the weights still have to be fetched before it can serve diff --git a/openspec/specs/shell-completion/spec.md b/openspec/specs/shell-completion/spec.md index b4f4f453..f731d99a 100644 --- a/openspec/specs/shell-completion/spec.md +++ b/openspec/specs/shell-completion/spec.md @@ -39,12 +39,15 @@ can never spew over the user's prompt. ### Requirement: Completion coverage Completion SHALL cover the full visible command surface: command names (the -hidden `__complete` excluded), each command's flags, and context-aware values — -provider names from the resolved catalogue (honouring a `--providers` override -already on the line), family and model names scoped to the `--provider` (and -`--model-family`) already typed, harness names, registered alias names where an -Outfit path is accepted, and the supported shells for `completion`. Positional -slots beyond a command's arity SHALL offer nothing. +hidden `__complete` excluded), each command's flags, its subcommands where it +has them, and context-aware values — provider names from the resolved catalogue +(honouring a `--providers` override already on the line), family and model +names scoped to the `--provider` (and `--model-family`) already typed, harness +names, registered alias names where an Outfit path is accepted, and the +supported shells for `completion`. For a command with subcommands, the first +positional slot SHALL offer those subcommands and any later slot SHALL fall +through to what the command otherwise accepts. Positional slots beyond a +command's arity SHALL offer nothing. #### Scenario: Families scoped to the typed provider @@ -61,3 +64,13 @@ slots beyond a command's arity SHALL offer nothing. - **WHEN** a new subcommand is added to the CLI's dispatch - **THEN** the completion surface must list it too (enforced by a test that scans the dispatch) + +#### Scenario: A nested command offers its subcommands + +- **WHEN** the user completes `outfit remote ` +- **THEN** its subcommands are offered, with no file paths + +#### Scenario: After a subcommand, the Outfit slot completes + +- **WHEN** the user completes `outfit remote deploy ` +- **THEN** registered alias names and paths are offered diff --git a/remote/.gitignore b/remote/.gitignore new file mode 100644 index 00000000..cf14b12e --- /dev/null +++ b/remote/.gitignore @@ -0,0 +1,9 @@ +node_modules/ +cdk.out/ +cdk.context.json +coverage/ +.env +remote.json +Outfit +cdk-outputs.json +*.tsbuildinfo diff --git a/remote/Outfit.example b/remote/Outfit.example new file mode 100644 index 00000000..22279f0f --- /dev/null +++ b/remote/Outfit.example @@ -0,0 +1,28 @@ +# Remote Qwen endpoint deployed by this directory. +# +# The concrete `Outfit` and `remote.json` are generated from the CDK stack +# outputs by `pnpm write-config` (run for you by `pnpm deploy`) — edit this +# template, not the generated files. Then, from this directory: +# outfit remote deploy # tell the endpoint what to serve (this file + preset) +# outfit remote start # boot the instance; prints the OPENAI_API_KEY export +# outfit apply # point your coding agent at the endpoint (uses BASEURL) +# outfit remote stop # stop now instead of waiting for the idle timer +# +# PROVIDER names the inference engine, so this file is what chooses llama.cpp +# over vLLM: `outfit remote deploy` asks the cloud for that engine, and the same +# file runs it locally under `outfit serve`. To cut back to vLLM, set PROVIDER +# to vllm, drop PRESET, and set MODEL to the FP8 repo (Qwen/Qwen3.6-27B-FP8). +# Either way the deploy Lambda seeds the weights into S3 if they are not there. +# +# ALIAS is both the name your coding agent asks for and the name the server is +# started under (llama-server's --alias), so the two cannot drift apart. +# +# The endpoint is a stable Elastic IP, so BASEURL is static config here rather +# than something fetched at runtime. remote.json carries the Lambda URLs that +# `outfit remote` drives. +PROVIDER llamacpp +ALIAS qwen3.6-27b +CONTEXT 131072 +BASEURL __BASEURL__ +PRESET ./preset.ini +REMOTE remote.json diff --git a/remote/README.md b/remote/README.md new file mode 100644 index 00000000..2408b560 --- /dev/null +++ b/remote/README.md @@ -0,0 +1,389 @@ +# remote — the cloud GPU deployment + +The deployment [`outfit remote`](../docs/commands/remote.md) drives: a +scale-to-zero, self-hosted **Qwen3.6-27B** endpoint on AWS, exposing an +OpenAI-compatible API for use as a coding-agent backend. It runs on a GPU +instance that exists only while you are actually using it: a start Lambda +launches it on demand, and a stop Lambda terminates it after a period of +idleness. + +The inference engine is **pluggable** — [llama.cpp](https://github.com/ggml-org/llama.cpp) +or [vLLM](https://docs.vllm.ai). The deployed default is llama.cpp, serving +Unsloth's `Qwen3.6-27B-MTP-GGUF` at **128k context** with a q8 KV cache and +**multi-token prediction** (~0.8 draft acceptance, so decode is roughly twice +what it would be without). Which engine you get is decided by one line in an +`Outfit` file, not by a redeploy. + +The instance is **stateless**, and responsibilities are split cleanly: + +- A slim **AMI per engine** (baked by the image stack via EC2 Image Builder) + carries only the NVIDIA driver and that engine — a `uv` venv for vLLM, a + prebuilt CUDA `llama-server` for llama.cpp. No Docker. Both are + model-agnostic and rarely change. +- The **model weights** live in an **S3 bucket**, put there by a disposable + seed job that downloads from Hugging Face entirely within AWS. You do not run + it by hand: deploying a model whose weights are missing starts it for you. +- At boot the instance **syncs the weights from S3** onto its disk (~2–4 min) + and starts the engine pointed at them. + +Because the AMI is a regional artifact, the start Lambda can launch in **any** +availability zone — it tries each g6e zone in turn until one has capacity. + +The image stack defines an Image Builder **pipeline**, not a build, so +deploying it never runs (or fails on) a bake. You trigger bakes out-of-band +with `pnpm bake `; each successful bake **tags** its AMI with its +engine, and the start Lambda launches the **newest AMI matching the engine it +was told to run**. A failed bake produces no new AMI and changes nothing. + +``` + pnpm bake llamacpp ─▶ Image Builder pipeline ─(async)─▶ AMI (driver + engine), tagged +outfit remote deploy ─▶ deploy Lambda ─▶ deploy-config (what to serve) + └─ seeds weights ─▶ S3 weights bucket + newest AMI by tag + weights ◀─┐ (at launch) +outfit remote start ─SigV4▶ start Lambda ─ RunInstances (try each AZ) ─▶ EC2 g6e.xlarge +outfit remote status ─────▶ (Function URL, + EIP, SSM health) │ L40S 48GB +outfit remote stop ───────▶ stop Lambda AWS_IAM auth │ s3 sync weights + ▲ │ engine on :8000 +EventBridge rate(5 min) ─────────┘ (idle check → TerminateInstances) ▼ +coding agent ── OPENAI_BASE_URL=http://:8000/v1 + api key ──▶ direct HTTP +``` + +Inference traffic goes directly from your machine to the instance; the Lambdas +only orchestrate. They live outside the VPC (no NAT gateway cost) and observe +the engine by running `curl localhost` on the instance via SSM Run Command, so +nothing is exposed beyond the API port itself, and that only to your IP. A +stable Elastic IP is re-associated with each launch so the endpoint URL never +changes. + +For how the pieces fit together — the stacks, the deploy-config control plane, +and the wake/idle lifecycle — see [docs/architecture.md](docs/architecture.md). + +## Prerequisites + +- An AWS account with admin (or equivalent) credentials configured locally +- Node.js 22+ and [pnpm](https://pnpm.io) +- The [`outfit`](https://github.com/lucinate-ai/outfit) CLI, which drives the + endpoint +- AWS CLI v2, plus the [Session Manager plugin](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html) + for shell access (there is no SSH) + +> **⚠️ GPU quota**: new AWS accounts have a vCPU quota of **0** for G-series +> instances, which makes the very first `start` fail. Check before deploying: +> +> ```sh +> aws service-quotas get-service-quota --service-code ec2 \ +> --quota-code L-DB2E81BA --region us-east-1 \ +> --query 'Quota.Value' +> ``` +> +> A `g6e.xlarge` needs 4 vCPUs. If the value is below 4, request an increase +> for "Running On-Demand G and VT instances" and expect approval to take up to +> a day or two. (The quota is per region — a grant in one region does not carry +> to another.) + +**Changing region?** g6e coverage is patchy: the region must offer the type at +all (none of eu-west-1/eu-west-2 do), and within a region it exists in only +some AZs (in us-east-1, not us-east-1a). Set `availabilityZones` to the g6e +zones — the start Lambda tries them in order. List what your account sees: + +```sh +aws ec2 describe-instance-type-offerings --location-type availability-zone \ + --region --filters Name=instance-type,Values=g6e.xlarge \ + --query 'InstanceTypeOfferings[].Location' +``` + +## Deploy + +Two stacks. The **image stack** defines the Image Builder pipelines (fast to +deploy); `pnpm bake` triggers an actual build. The **runtime stack** is the +endpoint itself. + +```sh +pnpm install +pnpm set-ip # writes this machine's public IP to .env +pnpm cdk bootstrap # once per account/region +pnpm deploy:image # creates the bake pipelines — instant, no build yet +pnpm bake llamacpp # bakes that engine's AMI — ~15-25 min, in the background +pnpm run deploy # deploys the runtime + S3 bucket, generates Outfit/remote.json +outfit remote deploy # says what to serve; seeds the weights if they're missing +``` + +- `pnpm deploy:image` only creates the pipelines, so it deploys in seconds and + can never fail because of a bake. +- `pnpm bake ` builds that engine's AMI and returns immediately; + the build runs asynchronously — check it with the `aws imagebuilder get-image` + command the script prints. Re-bake only when the engine version or the driver + changes; the model is **not** baked in. +- `pnpm run deploy` deploys the runtime stack (VPC, Lambdas, EIP, **S3 weights + bucket**) and generates the two gitignored files outfit reads: `Outfit` (with + the endpoint in its `BASEURL`) and `remote.json` (the Lambda URLs), both from + [`Outfit.example`](Outfit.example) — nothing to copy by hand. +- `outfit remote deploy` reads the `Outfit` and its [`preset.ini`](preset.ini) + and tells the endpoint what to serve. If those weights are not in S3 it starts + the seed job itself (~15–20 min, all within AWS) and says so; wait for it + before the first `start`. + +> Use `pnpm run deploy`, not `pnpm deploy` — the latter is pnpm's own built-in +> `deploy` command. And don't pass `-c` flags through it: pnpm appends extra +> arguments to the **last** command in the script chain, not to `cdk deploy`. +> Put context in `cdk.json` instead. + +The bake and the weight seed are independent and can run in parallel. + +Two settings are **required, with no default**: `allowedCidr` (the IP allowed to +reach the endpoint) and `runner` (which engine's AMI to launch). `pnpm set-ip` +stores the CIDR in the gitignored `.env` (`ALLOWED_CIDR=/32`); put `runner` +in `cdk.json` context. `.env` can also hold `HF_TOKEN` for gated model repos +(used only when seeding). Everything else has defaults, overridable in +`cdk.json`: + +| Context key | Default | Notes | +|---|---|---| +| `allowedCidr` | *(required)* | Your IP as a /32; `pnpm set-ip` writes it to `.env` | +| `runner` | *(required)* | `llamacpp` or `vllm`. No default; one must be chosen | +| `region` | `us-east-1` | g6e is not offered everywhere (absent from all of eu-west-1/2) | +| `availabilityZones` | `us-east-1b,c,d,e` | g6e zones the start Lambda tries, in order | +| `hfToken` | *(empty)* | Only for gated repos; used only when seeding | +| `instanceType` | `g6e.xlarge` | Runtime GPU type, 1× L40S 48 GB, ~$1.86/hr | +| `builderInstanceType` | `m5.xlarge` | Cheap non-GPU type used to bake the AMI and to seed | +| `imageVolumeGb` | `80` | AMI root — fits the OS + engine + the model synced at boot | +| `llamacppRelease` | `b10107` | Pinned ai-dock/llama.cpp-cuda build baked into the llama.cpp AMI | +| `vllmVersion` | `0.26.0` | vLLM version installed into that AMI's venv (`uv pip install`) | +| `nvidiaDriverPackage` | `nvidia-driver-570-server-open` | Driver installed in both AMIs | +| `idleThresholdMinutes` | `15` | Terminate after this long without requests | +| `gracePeriodMinutes` | `30` | Never terminate this soon after boot (covers the cold load) | +| `maxRuntimeMinutes` | `240` | Hard terminate this long after boot, even if busy | + +The **model, quant, context window and engine flags are not in this table** — +they come from the `Outfit` and its preset via `outfit remote deploy`, so +changing model is a command, not a redeploy. (`modelId`, `maxModelLen`, +`vllmExtraArgs`, `toolCallParser` and `reasoningParser` remain as context keys, +but only to seed the very first deploy-config before outfit takes over.) + +What needs what: +- Change **model, quant, context or engine flags** → edit the `Outfit`/preset, + then `outfit remote deploy`. No bake, no redeploy. +- Change **`llamacppRelease`/`vllmVersion`/`nvidiaDriverPackage`** → bump the + recipe `version` in `lib/image-stack.ts`, then `pnpm deploy:image` + + `pnpm bake `. +- Change a runtime-only setting (idle timers, `allowedCidr`) → `pnpm run deploy`. + +**On the model choice**: BF16 weights for a 27B model are ~54 GB and do not fit +the L40S's 48 GB, so a quantised checkpoint is mandatory. The default Q6_K_XL +GGUF is ~22.5 GB, leaving room for a 128k q8 KV cache (~16 GB) inside 48 GB. +For vLLM, FP8 is hardware-native on the L40S (Ada generation). + +### Switching engine, or model + +The `Outfit` is the control surface. `PROVIDER` names the engine, so the same +file that runs a model locally under `outfit serve` deploys it to the cloud +under `outfit remote deploy`: + +```sh +outfit remote deploy # what ./Outfit describes +outfit remote deploy path/to/Outfit # something else +outfit remote deploy --dry-run # print the config without sending it +``` + +Cutting back to vLLM means an Outfit with `PROVIDER vllm` and the FP8 repo as +its `MODEL` — both AMIs stay baked, so it is a deploy, not a rebuild. Note that +the `runner` context key must match the engine you deploy, since it selects +which AMI the start Lambda launches. + +### First boot + +A wake is an instance launch, an **S3 sync of the weights** (~2–4 min, +EBS-write-bound), then loading them into VRAM and warm-up — so roughly +**8–10 minutes**, every time, with no Hugging Face dependency. The first +request after a cold start also pays a one-off warm-up (~30 s); steady-state +decode is around 28 tokens/s. Watch a wake: + +```sh +pnpm console # SSM shell onto the running instance +sudo journalctl -u llama-server -f # engine logs (or -u vllm for vLLM) +tail -f /var/log/cloud-init-output.log # boot: s3 sync progress +``` + +## Daily use + +The endpoint is driven by the `outfit` CLI, using the generated `Outfit` and +`remote.json`. Run these from this directory (`outfit` reads `./Outfit`): + +```sh +outfit remote start # boots the instance, blocks until it is serving, + # prints OPENAI_BASE_URL + OPENAI_API_KEY exports +outfit apply # points your coding agent at the endpoint +outfit remote status # instance state + endpoint health +outfit remote stop # stop immediately instead of waiting for the idle timer +``` + +`outfit apply` writes the endpoint's `BASEURL` and API key into your harness +config, so export the key that `outfit remote start` prints first. The model +name to request is the Outfit's `ALIAS` (`qwen3.6-27b`) — the same value the +server is started under, so the two cannot drift: + +```sh +eval "$(outfit remote start)" # sets OPENAI_BASE_URL + OPENAI_API_KEY +curl "$OPENAI_BASE_URL/models" -H "Authorization: Bearer $OPENAI_API_KEY" +curl "$OPENAI_BASE_URL/chat/completions" \ + -H "Authorization: Bearer $OPENAI_API_KEY" -H 'Content-Type: application/json' \ + -d '{"model":"qwen3.6-27b","messages":[{"role":"user","content":"Say hi"}]}' +``` + +> Qwen3.6 is a reasoning model and will happily spend a small token budget +> entirely on thinking, returning empty `content` with the text in +> `reasoning_content`. Give requests generous `max_tokens`. + +### Idle behaviour + +Every 5 minutes the stop Lambda scrapes the engine's request/token counters +(via SSM, on the instance — it reads whichever metric names the deployed engine +exposes). If nothing has moved for `idleThresholdMinutes`, the instance is +**terminated**. With defaults, expect that **15–20 minutes** after the last +request. Terminated means $0 compute and no volume left behind — the next +`outfit remote start` launches a fresh one from the AMI. + +If the metrics scrape fails (e.g. the engine crashed), the instance is still +terminated at the threshold — deliberately, so a wedged box does not run up +GPU-hours unnoticed. + +There is also a hard cap: `maxRuntimeMinutes` (default 4 hours) terminates the +instance that long after it started **even if requests are still flowing**, as +a backstop against a runaway session. Each launch resets the clock, so it caps +a session, not anything cumulative — if you hit it mid-work, `outfit remote +start` brings the endpoint back for another 4 hours. Like the idle stop, it +lands on the next 5-minute tick. + +**Pinning an instance up**: tag it `Retain-Until` with a UTC ISO-8601 time and +neither the idle timer nor the hard cap will touch it until then — handy while +debugging on the box. A manual `outfit remote stop` still works. + +```sh +aws ec2 create-tags --resources \ + --tags Key=Retain-Until,Value=2026-07-25T22:45:00Z +``` + +## Costs + +At rest (nothing running) the endpoint costs only the **Elastic IP, the S3 +weights (~$0.60/mo for the 26 GB GGUF), the AMI snapshots, and Secrets +Manager — roughly $6–7/month** — because the instance is terminated, not stopped, so there +is no idle EBS volume. While running it is **$1.86/hour in us-east-1**; ~2 h of +coding a day lands around $90/month. Full breakdown in +[docs/costs.md](docs/costs.md). + +## Operations + +- **Logs**: `pnpm console` (an SSM shell onto the running instance) then + `sudo journalctl -u llama-server -f` or `tail -f /var/log/cloud-init-output.log` + (boot / S3 sync). Lambda decisions (launch AZ, idle/terminate, deploys) are in + the three Lambdas' CloudWatch log groups. +- **Changing the model**: edit the `Outfit`/preset and run `outfit remote + deploy`. It seeds the new weights if needed. No bake, no redeploy. +- **Changing the engine version or the driver**: update `llamacppRelease` / + `vllmVersion` / `nvidiaDriverPackage`, **bump the recipe (and component) + `version` in `lib/image-stack.ts`** (Image Builder versions are immutable), + then `pnpm deploy:image` + `pnpm bake `. +- **Your home IP changed**: `pnpm set-ip && pnpm run deploy` (runtime only). +- **Force a fresh AMI** (same config): just `pnpm bake ` — the runtime + launches the newest tagged AMI. +- **Force a re-seed** of weights already in S3: `pnpm seed-model` (the deploy + Lambda only seeds what is missing). + +### Diagnostics + +`pnpm console` drops you onto the running instance over SSM (needs +`session-manager-plugin`). Once there: + +| Want to know | Command | +|---|---| +| Follow the engine's logs | `sudo journalctl -u llama-server -f` | +| Why it won't start | `sudo journalctl -u llama-server --no-pager \| tail -50` | +| Is it up? | `systemctl is-active llama-server` · `ss -ltn \| grep :8000` | +| Is MTP actually working | `journalctl -u llama-server \| grep 'draft acceptance'` | +| Boot / S3-sync progress | `tail -f /var/log/cloud-init-output.log` | +| Weights pulled so far | `du -sh /opt/llm/model` | +| GPU + driver | `nvidia-smi` | +| RAM + swap | `free -h` | + +(For a vLLM deployment the unit is `vllm` rather than `llama-server`.) + +From your own machine, no shell needed (the EIP is ``): + +```sh +curl -s -o /dev/null -w '%{http_code}\n' http://:8000/health # 200 once serving +eval "$(outfit remote start)" # base URL + key +curl "$OPENAI_BASE_URL/models" -H "Authorization: Bearer $OPENAI_API_KEY" +``` + +The Lambdas log every decision to CloudWatch. In the **stop** Lambda's log +group, each 5-minute tick prints a JSON line — grep `"mode":"idle"` to see why +it kept or terminated the instance (e.g. `"decision":"stop","reason":"idle for +32.9 min"`, or `"reason":"retained until …"` when a `Retain-Until` tag is set). +The **start** Lambda logs the launch AZ and each wake phase. + +## Security notes + +- The API port is only reachable from `allowedCidr`, and the engine itself + requires the generated API key (stored in Secrets Manager) as a second layer. +- Traffic is plain HTTP, so the API key is visible in transit; the /32 + restriction is what makes this acceptable for solo use. The planned fix — + which also ends the allowed-CIDR juggling when your home IP changes — is + joining the instance to a tailnet: see + [docs/tailscale-plan.md](docs/tailscale-plan.md). +- No SSH ingress; shell access is via SSM Session Manager. IMDSv2 is enforced. +- The Function URLs require SigV4-signed requests (`lambda:InvokeFunctionUrl`), + so `outfit` needs no AWS permissions beyond invoking them. + +## Teardown + +```sh +pnpm cdk destroy cloud-vm-llm cloud-vm-llm-image +``` + +Removes both stacks. If an instance is currently running, terminate it first +(`outfit remote stop`) — it is not owned by CloudFormation. The **S3 weights +bucket is retained** on destroy (so you don't lose the seeded weights); baked +AMIs and their snapshots are not owned by the stacks either. Delete the bucket, +deregister the AMIs, and delete their snapshots by hand to reclaim that storage. + +## Troubleshooting + +- **`start` returns `unconfigured`**: nothing has been deployed yet. Run + `outfit remote deploy`. +- **`start` returns `no-ami`**: no AMI is tagged for the engine you asked for. + Run `pnpm bake ` and wait for it to reach `AVAILABLE`. +- **`start` returns `no-capacity`**: every configured AZ was out of g6e + capacity at that moment. The start Lambda already tried them all; wait a few + minutes and retry, or widen/adjust `availabilityZones`. +- **A bake fails**: it does **not** touch the stack or the previous AMI. Check + it with `aws imagebuilder get-image --image-build-version-arn ` (the arn + `pnpm bake` printed) and the CloudWatch log group for the recipe, fix, and + `pnpm bake` again. The driver install is the most likely failure — adjust + `nvidiaDriverPackage`. +- **`deploy:image` fails with "recipe/component version already exists"**: you + changed a baked-in setting without bumping the `version` on the recipe (or + component) in `lib/image-stack.ts`. Bump and redeploy. +- **`start` reaches `running` but never `ready`, or the model is empty**: the + weights aren't in S3 yet, or a seed is still running. Check + `aws s3 ls s3:///models///`. +- **Quota errors on launch**: see the GPU quota warning above. +- **`start` times out repeatedly**: `pnpm console` onto the instance and read + `sudo journalctl -u llama-server --no-pager | tail -50`. Known startup + crashes, all handled for the defaults but reachable after a bump: + - `libcudart.so.12: cannot open shared object` — the prebuilt llama.cpp + tarball bundles only its own libraries, not the CUDA runtime; the AMI + installs `cuda-cudart-12-8`, `libcublas-12-8` and `libnccl2` for it. + - `Python.h: No such file or directory` (vLLM) — a model needs Triton's + runtime compile; the AMI installs `python3.12-dev` for it. + - `Could not find nvcc` (vLLM) — the FlashInfer sampler wants the CUDA + toolkit, which the slim AMI omits. The user-data sets + `VLLM_USE_FLASHINFER_SAMPLER=0` (native sampler) to avoid it. + - Engine start OOM — lower `CONTEXT` in the Outfit, or use a smaller quant; + the driver failing to load shows up as a bad `nvidia-smi`. +- **The coding agent reports `the model ... does not exist`**: the `model` id it + sends must equal what the server serves. Under llama.cpp that is the Outfit's + `ALIAS`, which is also what the server is started with, so keep the two the + same — and if you deploy with a different `ALIAS`, re-run `outfit apply`. + Under vLLM there is no alias, so the id is the Hugging Face repo. Either way, + `curl "$OPENAI_BASE_URL/models"` shows the truth. diff --git a/remote/bin/app.ts b/remote/bin/app.ts new file mode 100644 index 00000000..b8c83353 --- /dev/null +++ b/remote/bin/app.ts @@ -0,0 +1,25 @@ +import * as cdk from 'aws-cdk-lib'; +import { loadConfig } from '../lib/config'; +import { ImageStack } from '../lib/image-stack'; +import { LlmStack } from '../lib/llm-stack'; + +const app = new cdk.App(); +const config = loadConfig(app); +const env = { region: config.region }; + +// Baked AMI (vLLM image + weights). Deploy rarely — only on a model/image +// change — and expect a ~20-40 min build. Decoupled from the runtime stack +// via the AMI-id SSM parameter, so there is no CloudFormation dependency. +new ImageStack(app, 'cloud-vm-llm-image', { + config, + env, + description: 'Bakes the vLLM + weights AMI for cloud-vm-llm', +}); + +// Scale-to-zero runtime. Its start Lambda launches the baked AMI on demand, +// trying each g6e AZ for capacity; the idle Lambda terminates it. +new LlmStack(app, 'cloud-vm-llm', { + config, + env, + description: 'Scale-to-zero self-hosted Qwen3.6-27B endpoint (vLLM on EC2)', +}); diff --git a/remote/cdk.json b/remote/cdk.json new file mode 100644 index 00000000..6944bd9e --- /dev/null +++ b/remote/cdk.json @@ -0,0 +1,26 @@ +{ + "app": "npx tsx bin/app.ts", + "context": { + "//": "Set allowedCidr (required) and any overrides here or via -c on the CLI.", + "//examples": { + "allowedCidr": "203.0.113.7/32", + "runner": "vllm", + "region": "us-east-1", + "availabilityZones": "us-east-1b,us-east-1c,us-east-1d,us-east-1e", + "modelId": "Qwen/Qwen3.6-27B-FP8", + "hfToken": "", + "instanceType": "g6e.xlarge", + "builderInstanceType": "m5.xlarge", + "imageVolumeGb": 80, + "vllmVersion": "0.26.0", + "nvidiaDriverPackage": "nvidia-driver-570-server-open", + "maxModelLen": 32768, + "vllmExtraArgs": "--enforce-eager", + "toolCallParser": "qwen3_coder", + "reasoningParser": "qwen3", + "idleThresholdMinutes": 15, + "gracePeriodMinutes": 30, + "maxRuntimeMinutes": 240 + } + } +} diff --git a/remote/docs/architecture.md b/remote/docs/architecture.md new file mode 100644 index 00000000..9d89e972 --- /dev/null +++ b/remote/docs/architecture.md @@ -0,0 +1,202 @@ +# Architecture + +Maintainer-facing notes on how this deployment is put together. For *using* it, see +the [README](../README.md); for the pound-and-pence, [costs.md](costs.md). + +## What it is + +A **scale-to-zero, self-hosted OpenAI-compatible LLM endpoint** on AWS. A GPU +instance exists only while you are actually serving requests: a start Lambda +launches one on demand, and a stop Lambda terminates it after idle. Nothing runs +(and almost nothing is billed) at rest. + +Three ideas hold it together: + +- **The instance is stateless.** No fixed EC2 instance, no persistent EBS. The + start Lambda launches one from a baked AMI, the stop Lambda terminates it. The + model weights live in S3 and are synced onto the instance at boot. +- **The runner is pluggable.** The inference server (currently vLLM; llama.cpp + landing) is chosen per deployment, not hard-wired. A runner-neutral + *deploy-config* says what to serve; the start Lambda builds the right command. +- **Deployment config is data, not a redeploy.** What to serve (runner, model, + context, serve args) lives in an SSM parameter written by a deploy Lambda, so + switching model or runner is a parameter write — no `cdk deploy`. + +## The pieces + +```mermaid +flowchart TB + subgraph client["Your machine"] + agent["coding agent
(OpenAI client)"] + outfit["outfit CLI"] + end + + subgraph image["Image stack (cloud-vm-llm-image)"] + pipeline["EC2 Image Builder
pipeline (per runner)"] + ami["slim AMI(s)
tagged by role + runner"] + pipeline -->|pnpm bake| ami + end + + subgraph runtime["Runtime stack (cloud-vm-llm)"] + start["StartFn"] + stop["StopFn"] + deploy["DeployFn"] + dcfg[("SSM: deploy-config")] + idle[("SSM: idle-state")] + eip["Elastic IP"] + s3[("S3: weights")] + sg["Security group
:port from your /32"] + sched["EventBridge
rate(5 min)"] + end + + inst["EC2 g6e (L40S)
runner + weights"] + + outfit -->|SigV4 start, status| start + outfit -->|SigV4 stop| stop + outfit -->|SigV4 deploy| deploy + deploy -->|write| dcfg + start -->|read at wake| dcfg + start -->|RunInstances
+ associate| eip + start -.->|launch newest AMI by tag| ami + inst -->|sync at boot| s3 + eip --- inst + agent -->|http + api key| inst + sched --> stop + stop -->|SSM scrape, terminate| inst + start -->|SSM health| inst +``` + +The Lambdas live **outside the VPC** (no NAT cost) and reach the instance over +**SSM Run Command** (`curl localhost`), so nothing is exposed beyond the vLLM +port, and that only to your `/32`. A stable **Elastic IP** is re-associated on +each launch so the base URL never changes. + +## The deploy-config control plane + +The seam between "what infra exists" (CDK's job, provisioned once) and "what to +serve" (per deployment). `outfit remote deploy` reads an Outfit file and POSTs a +DeployConfig to the deploy Lambda; the Lambda validates it and writes the +`/cloud-vm-llm/deploy-config` SSM parameter. The next wake reads it. + +```mermaid +flowchart LR + outfitfile["Outfit file
(runner, MODEL, CONTEXT, preset)"] + outfit["outfit remote deploy"] + deploy["DeployFn
(validate)"] + param[("deploy-config
SSM param")] + seed["seed instance
(if model changed)"] + start["StartFn (next wake)"] + + outfitfile --> outfit -->|SigV4 POST| deploy + deploy -->|PutParameter| param + deploy -.->|RunInstances| seed --> s3[("S3 weights")] + start -->|read| param + start -->|build serve command| unit["systemd unit"] +``` + +The DeployConfig contract (`lambda/shared/deploy-config.ts`): + +``` +{ runner: "vllm" | "llamacpp", // required; no default + modelId, quant, contextSize, servedModelName, + serveArgs: string[] } // runner-specific flags, pre-tokenised +``` + +`weightsPrefix` is **not** part of the wire contract — the Lambda derives it as +`models//[/]/` and stores it in the parameter, so callers +never encode the S3 layout (and a prefix sent in the body is ignored). If those +weights are not in the bucket, the Lambda launches the seed instance itself and +replies `{seeding: true, seedInstanceId}`; a wake before it finishes would sync +an incomplete prefix, so wait for it. + +`buildServeCommand()` turns it into the ExecStart, branching by runner — +`vllm serve …` or `llama-server …`. There is **no default runner**: an unset or +invalid config fails the wake loudly rather than guessing. + +The parameter is **outfit/manual-owned**. CDK creates it with a constant +`unconfigured` placeholder — deliberately *not* the cfg-derived config — so a +later `cdk deploy` can never clobber what `outfit remote deploy` (or a manual +edit) put there. `pnpm deploy`'s seed step (`scripts/seed-deploy-config.mjs`) +writes a cfg-derived initial config over the placeholder *only* while it is still +unconfigured, and only when CDK knows the full serve config (vLLM); llama.cpp's +serve args come from an Outfit, so its config is left for `outfit remote deploy` +to set. + +## Wake lifecycle + +`outfit remote start` (or any POST to the start Function URL) blocks until the +server is answering, so the caller gets one "ready" with the base URL + key. + +```mermaid +sequenceDiagram + participant O as outfit remote start + participant S as StartFn + participant P as deploy-config (SSM) + participant E as EC2 + participant I as instance + O->>S: POST (SigV4) + S->>P: read deploy-config + alt unconfigured + S-->>O: 503 "run outfit remote deploy" + end + S->>E: RunInstances (try each g6e AZ until capacity) + S->>E: associate Elastic IP + loop until healthy or deadline + S->>I: SSM curl localhost/health + end + Note over I: boot → swap → S3 sync weights → serve + S-->>O: 200 ready {base_url, api_key} +``` + +Boot user-data (built by the start Lambda from the deploy-config): log +`nvidia-smi`, add a swapfile, `aws s3 sync` the weights, fetch the API key, then +write the runner's env file + systemd unit and start it. The health check hits +`/health` on the port — portable across runners. + +## Idle / stop + +```mermaid +flowchart TB + tick["EventBridge tick
(every 5 min)"] --> stop["StopFn idleCheck"] + stop --> retain{"Retain-Until
in the future?"} + retain -->|yes| wait1["wait"] + retain -->|no| cap{"past max runtime?"} + cap -->|yes| term["TerminateInstances"] + cap -->|no| grace{"within grace
of launch?"} + grace -->|yes| wait2["wait"] + grace -->|no| scrape["SSM: scrape metrics"] + scrape --> active{"activity since
the idle anchor?"} + active -->|yes| upd["record, wait"] + active -->|no| idle{"idle > threshold?"} + idle -->|yes| term + idle -->|no| wait3["wait"] +``` + +A failed scrape counts as "no activity", so a wedged box is still terminated at +the threshold rather than burning GPU-hours. A `Retain-Until` instance tag (UTC +ISO-8601) overrides both the idle timer and the max-runtime cap; a manual +`outfit remote stop` still terminates immediately. + +## Image stack + +`cloud-vm-llm-image` defines an **Image Builder pipeline**, not an image — so +`cdk deploy` is instant and a bad bake can never fail the stack. `pnpm bake` +triggers a build out-of-band; each successful bake tags its AMI (role, and — as +the second runner lands — runner), and the start Lambda launches the **newest +AMI matching the tags**. A slim AMI carries only the driver + the runner +(vLLM as a `uv` venv; llama.cpp as a prebuilt CUDA `llama-server`) — no Docker. + +## Key files + +| Concern | File | +|---|---| +| Config + validation | `lib/config.ts` | +| Runtime stack (lambdas, EIP, S3, params) | `lib/llm-stack.ts` | +| Image Builder pipeline | `lib/image-stack.ts` | +| Deploy-config contract + `buildServeCommand` | `lambda/shared/deploy-config.ts` | +| Wake / launch / user-data | `lambda/start/index.ts` | +| Idle / manual stop | `lambda/stop/index.ts`, `lambda/shared/idle.ts` | +| Set the deploy-config | `lambda/deploy/index.ts` | +| Shared AWS + SSM helpers | `lambda/shared/aws.ts` | +| Seed weights to S3 | `scripts/seed-model.mjs` | +| Seed the initial deploy-config (once, over the placeholder) | `scripts/seed-deploy-config.mjs` | diff --git a/remote/docs/costs.md b/remote/docs/costs.md new file mode 100644 index 00000000..f35f12c6 --- /dev/null +++ b/remote/docs/costs.md @@ -0,0 +1,85 @@ +# What this stack costs + +The short version: **~$7/month at rest, plus $1.86 for every hour the GPU +instance is actually running.** Nothing scales with request volume — only with +hours up. Because the instance is terminated (not stopped) when idle, there is +no idle EBS volume, so the resting cost is low. + +## Fixed costs + +These accrue whether or not the instance is ever launched: + +| Item | Monthly (approx.) | +|---|---| +| Elastic IP (billed while allocated) | $3.70 | +| S3 weights (~$0.023/GB — $0.60 for the 26 GB GGUF, ~$1.30 if both engines' weights are kept) | $0.60–1.30 | +| AMI EBS snapshots (~20 GB used each @ ~$0.05/GB; one per engine baked) | $1–2 | +| Secrets Manager (the endpoint's API key) | $0.40 | +| Lambdas, EventBridge, SSM, Parameter Store, CloudWatch logs | ~$0 | +| VPC, subnets, internet gateway | $0 | +| **Total** | **~$6–7** | + +The orchestration layer is effectively free: the idle check runs every +5 minutes but sits comfortably inside the Lambda and EventBridge free tiers, +and the stack deliberately has no NAT gateway (~$36/month avoided — the +Lambdas live outside the VPC and reach the instance via SSM instead). + +There is **no persistent EBS volume**: the weights live in S3, and the running +instance's root volume is created from the engine's AMI and deleted on +termination. So while running you also pay for that ephemeral ~80 GB gp3 root +(~$0.25/day pro-rated while up), but nothing for it at rest. Moving the weights +out of the AMI into S3 also shrank the snapshot from ~113 GB to ~20 GB. + +## Variable cost: GPU hours + +A g6e.xlarge costs **$1.86/hour on-demand in us-east-1** (for comparison: +$1.97 in Stockholm, $2.33 in Frankfurt — and none at all in Ireland or +London, which is how the stack ended up in us-east-1). Billing is per-second +with a 60-second minimum. + +Worked examples (GPU hours + the ~$6 fixed floor): + +| Usage pattern | GPU hours/month | Total/month | +|---|---|---| +| Never launched | 0 | ~$6 | +| ~2 h/day on weekdays | ~44 | ~$90 | +| 4-hour cap hit every weekday | ~88 | ~$170 | +| Always on, 24/7 (what this design avoids) | 730 | ~$1,370 | + +The bake and the weight seed are one-offs: each runs a cheap `m5.xlarge` for +~20–40 minutes, so a few pennies. Bakes are only needed when the engine version +or driver changes — swapping model is just `outfit remote deploy`, and the seed +it starts is the only cost. + +Need more headroom? The 27B checkpoint sits close to the 32 GB of RAM +on a `g6e.xlarge`. A `g6e.2xlarge` doubles RAM to 64 GB (same single L40S) at +**~$3.72/hour**, but needs the G-series vCPU quota raised to 8 (from the 4 a +`g6e.xlarge` uses). On `g6e.xlarge` a boot-time swapfile covers the shortfall. + +## What bounds the bill + +Two mechanisms keep the variable column honest: + +- **Idle terminate** — the instance is terminated 15–20 minutes after the last + request (`idleThresholdMinutes`, default 15, plus up to one 5-minute check + tick). Forgetting to stop it costs about $0.60, not a day of GPU time. +- **Maximum runtime** — `maxRuntimeMinutes` (default 4 hours) terminates the + instance that long after boot even if requests are still flowing, so a + runaway session is bounded at ~$8. Each launch resets the clock. + +Data transfer is negligible: the weights sync from S3 in-region (free), so a +wake pulls no billable bytes, and streamed completions outbound are text. + +## Reducing the fixed floor + +- **`pnpm cdk destroy cloud-vm-llm cloud-vm-llm-image`** drops everything to + $0. Deregister the AMIs/snapshots too if you want that storage back. The + next setup pays a bake (~30 min) again. +- **Tailscale** (planned, see [tailscale-plan.md](tailscale-plan.md)) removes + the Elastic IP, shaving $3.70/month off the fixed floor and closing the + public endpoint at the same time. +- A smaller `imageVolumeGb` shrinks the snapshot line, but leave headroom for + the OS + engine (~20 GB), the ~16 GB boot-time swapfile, and the weights + synced from S3 at boot (~26 GB for the default GGUF). +- Only bake the engine you actually run; each baked AMI carries its own + snapshot. Keeping both is what buys the one-command cut-back. diff --git a/remote/docs/tailscale-plan.md b/remote/docs/tailscale-plan.md new file mode 100644 index 00000000..9d2e1f39 --- /dev/null +++ b/remote/docs/tailscale-plan.md @@ -0,0 +1,88 @@ +# Plan: move endpoint access onto Tailscale + +**Status: proposed, not implemented.** The stack currently exposes vLLM on a +public Elastic IP, restricted to a single CIDR (`allowedCidr`) with the API +key as a second layer. This works, but the IP allowlist needs updating every +time the home IP changes, and traffic is plain HTTP, so the API key is +visible in transit. Joining the instance to a tailnet fixes both. + +## Target design + +The instance runs `tailscaled` and joins the tailnet at first boot. Clients +reach vLLM at its MagicDNS name (`http://cloud-vm-llm:8000/v1`) instead of a +public IP. Tailscale only makes outbound connections, so: + +- The security group loses its port-8000 ingress rule — nothing is publicly + reachable at all, and `allowedCidr` stops existing as a concept. +- All traffic rides inside WireGuard, so the API key is no longer visible on + the wire. Keep vLLM's `--api-key` anyway as defence in depth. +- The Elastic IP becomes redundant (MagicDNS is the stable name) and can be + dropped, saving ~$3.60/month. + +The scale-to-zero machinery is untouched: the Lambdas reach the instance via +SSM and the EC2 API, not the network path, so wake, idle-stop and status all +work as they do today. The instance is stateless — terminated when idle, a +fresh root from the AMI on each wake — so there is no persisted Tailscale +state: each wake registers as a new node via the reusable, pre-authorised auth +key, and the previous node lingers in the admin console until cleaned up (see +caveats). + +## Changes required + +1. **Tailscale auth key** — a new Secrets Manager secret. Unlike the API key + it cannot be generated by the stack: it comes from the Tailscale admin + console (see one-time setup below) and is passed in as a context value or + pasted into the secret after deploy. Read once by user-data at first boot. +2. **`lib/llm-stack.ts`** + - Add the auth key secret and grant the instance role read on it. + - Remove the SG ingress rule and the `CfnEIP`/association (or gate both + behind a `tailscaleEnabled` config flag during the transition). + - `BaseUrl` output (and the Lambdas' `BASE_URL` env) becomes + `http://:8000/v1`. +3. **User-data** (`buildUserData` in `lambda/start/index.ts`) + - Install tailscaled (`curl -fsSL https://tailscale.com/install.sh | sh` + or the apt repo directly, pinned). + - `tailscale up --authkey "$TS_AUTHKEY" --hostname cloud-vm-llm` before + starting the vLLM service. +4. **`lib/config.ts`** — add `tailscaleHostname` (default `cloud-vm-llm`); + make `allowedCidr` optional (only required if the public path is kept as + an escape hatch). +5. **README** — deploy walkthrough gains the one-time Tailscale setup; the + security notes and cost table are updated (EIP line goes away). +6. **outfit** — no code change. `outfit remote start` prints whatever + `base_url` the start Lambda returns, which now carries the MagicDNS URL. + +## One-time setup (manual, in the Tailscale admin console) + +- Create a tag for the node, e.g. `tag:llm`, owned by your user. Tagged + devices are treated as servers: their node keys don't expire, which is what + a stop/start instance needs. +- Create a **pre-authorised, reusable** auth key with `tag:llm`. (Not + ephemeral — ephemeral nodes are removed when they go offline, and this + instance is offline whenever it's stopped.) +- Add an ACL rule allowing your devices to reach `tag:llm:8000`. + +## Verification + +Deploy, then from a tailnet device: `tailscale status` shows the node; +`curl http://cloud-vm-llm:8000/v1/models -H "Authorization: Bearer $OPENAI_API_KEY"` +works; the same curl against the old public IP fails (port closed). Stop and +wake the instance and confirm the node comes back with the same name. + +## Caveats + +- Home-to-EC2 connections normally go direct over WireGuard; when NAT + traversal fails Tailscale falls back to a DERP relay, which is noticeably + slower for token streaming. Rare, but check `tailscale status` for + `relay` if generation feels sluggish. +- The auth key is only consumed at first boot. If the instance is ever + replaced (user-data or AMI change), the replacement re-registers — the old + node lingers in the admin console until removed, and a reusable key makes + re-registration automatic. Rotate the key like any other credential. +- Free personal Tailscale plan covers this (well under the device limit). + +## Rollback + +Everything is additive: re-enable the SG ingress rule and EIP, point +`BaseUrl` back at the public address, redeploy. The tailnet node can stay — +it does no harm alongside the public path. diff --git a/remote/lambda/deploy/index.ts b/remote/lambda/deploy/index.ts new file mode 100644 index 00000000..d8adbb93 --- /dev/null +++ b/remote/lambda/deploy/index.ts @@ -0,0 +1,97 @@ +import type { LambdaFunctionURLEvent, LambdaFunctionURLResult } from 'aws-lambda'; +import { errorName, readDeployConfig, requireEnv, writeDeployConfig } from '../shared/aws'; +import { parseDeployConfig } from '../shared/deploy-config'; +import { jsonResponse } from '../shared/http'; +import { launchSeedInstance, weightsPresent, type SeedEnv } from '../shared/seed'; + +const DEPLOY_CONFIG_PARAM = requireEnv('DEPLOY_CONFIG_PARAM'); +const WEIGHTS_BUCKET = requireEnv('WEIGHTS_BUCKET'); + +const SEED_ENV: SeedEnv = { + region: requireEnv('AWS_REGION'), + bucket: WEIGHTS_BUCKET, + instanceType: requireEnv('SEED_INSTANCE_TYPE'), + subnetId: requireEnv('SEED_SUBNET_ID'), + securityGroupId: requireEnv('SEED_SECURITY_GROUP_ID'), + instanceProfileArn: requireEnv('SEED_INSTANCE_PROFILE_ARN'), + hfSecretArn: process.env.HF_TOKEN_SECRET_ARN ?? '', + amiRoleTagKey: requireEnv('AMI_ROLE_TAG_KEY'), + amiRoleTagValue: requireEnv('AMI_ROLE_TAG_VALUE'), + amiRunnerTagKey: requireEnv('AMI_RUNNER_TAG_KEY'), +}; + +/** + * The control plane `outfit remote deploy` calls (SigV4 Function URL, like + * start/stop). POST a DeployConfig — derived from an Outfit file — and it is + * validated, the weights are seeded if they are not in S3 yet, and the config + * is written to the deploy-config SSM parameter that the next wake reads. GET + * returns the current config. This keeps outfit thin (Lambda invoke only) with + * all validation, layout decisions and AWS mutation server-side. + */ +export async function handler(event: LambdaFunctionURLEvent): Promise { + const method = event.requestContext?.http?.method ?? 'POST'; + + if (method === 'GET') { + try { + return jsonResponse(200, await readDeployConfig(DEPLOY_CONFIG_PARAM)); + } catch (err) { + // Unset/invalid — nothing has been deployed yet. + return jsonResponse(404, { state: 'unconfigured', message: (err as Error).message }); + } + } + + const body = + event.isBase64Encoded && event.body + ? Buffer.from(event.body, 'base64').toString('utf8') + : (event.body ?? ''); + + let config; + try { + config = parseDeployConfig(body); + } catch (err) { + return jsonResponse(400, { error: (err as Error).message }); + } + + // Seed before writing the config: if the weights are missing and the seed + // cannot even be launched, the current (working) config is left alone rather + // than replaced by one that would fail at wake. + let seeding = false; + let seedInstanceId: string | undefined; + try { + if (!(await weightsPresent(WEIGHTS_BUCKET, config))) { + seedInstanceId = await launchSeedInstance(config, SEED_ENV); + seeding = true; + } + } catch (err) { + console.log(JSON.stringify({ action: 'deploy', error: `seed failed: ${errorName(err)}` })); + return jsonResponse(502, { + error: `weights are not in S3 and the seed could not be started: ${(err as Error).message}`, + }); + } + + await writeDeployConfig(DEPLOY_CONFIG_PARAM, config); + console.log( + JSON.stringify({ + action: 'deploy', + runner: config.runner, + modelId: config.modelId, + quant: config.quant, + weightsPrefix: config.weightsPrefix, + seeding, + seedInstanceId, + }), + ); + return jsonResponse(200, { + deployed: true, + runner: config.runner, + modelId: config.modelId, + contextSize: config.contextSize, + weightsPrefix: config.weightsPrefix, + seeding, + ...(seedInstanceId ? { seedInstanceId } : {}), + // A wake before the seed finishes would sync an incomplete prefix. + ...(seeding + ? { message: 'seeding the weights (~15-20 min); wait for it to finish before starting' } + : {}), + }); +} diff --git a/remote/lambda/shared/aws.ts b/remote/lambda/shared/aws.ts new file mode 100644 index 00000000..f5070175 --- /dev/null +++ b/remote/lambda/shared/aws.ts @@ -0,0 +1,303 @@ +import { + AssociateAddressCommand, + DescribeImagesCommand, + DescribeInstancesCommand, + EC2Client, + type Filter, + RunInstancesCommand, + type RunInstancesCommandInput, + TerminateInstancesCommand, + _InstanceType, +} from '@aws-sdk/client-ec2'; +import { + DescribeInstanceInformationCommand, + GetCommandInvocationCommand, + GetParameterCommand, + PutParameterCommand, + SSMClient, + SendCommandCommand, +} from '@aws-sdk/client-ssm'; +import { randomUUID } from 'node:crypto'; +import { type DeployConfig, parseDeployConfig } from './deploy-config'; +import type { IdleState } from './idle'; + +const ec2 = new EC2Client({}); +const ssm = new SSMClient({}); + +export function requireEnv(name: string): string { + const value = process.env[name]; + if (!value) { + throw new Error(`Missing required environment variable ${name}`); + } + return value; +} + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export function errorName(err: unknown): string { + return err instanceof Error ? err.name : String(err); +} + +/** + * Instance tag holding an ISO-8601 UTC datetime before which automatic + * termination (idle check and the max-runtime cap) must not fire — a manual + * "keep this alive until" override, e.g. while debugging on the box. A manual + * `outfit remote stop` still terminates it: the tag guards against accidental + * death, not deliberate shutdown. + */ +export const RETAIN_UNTIL_TAG = 'Retain-Until'; + +function parseRetainUntil(tags?: { Key?: string; Value?: string }[]): Date | undefined { + const raw = tags?.find((t) => t.Key === RETAIN_UNTIL_TAG)?.Value; + if (!raw) { + return undefined; + } + const ms = Date.parse(raw); + return Number.isNaN(ms) ? undefined : new Date(ms); +} + +export interface InstanceInfo { + instanceId: string; + state: string; + launchTime?: Date; + /** Parsed from the Retain-Until tag, if present and a valid datetime. */ + retainUntil?: Date; +} + +/** Describe a specific instance by id (used to poll a just-launched one). */ +export async function getInstance(instanceId: string): Promise { + const result = await ec2.send(new DescribeInstancesCommand({ InstanceIds: [instanceId] })); + const instance = result.Reservations?.[0]?.Instances?.[0]; + if (!instance) { + throw new Error(`Instance ${instanceId} not found`); + } + return { + instanceId, + state: instance.State?.Name ?? 'unknown', + launchTime: instance.LaunchTime, + retainUntil: parseRetainUntil(instance.Tags), + }; +} + +/** + * Find the current managed instance by tag, if one exists in a live state. + * The runtime holds no fixed instance id — the start Lambda launches one and + * the stop Lambda terminates it — so everything discovers "the instance" this + * way. Returns null when none is up. + */ +export async function findManagedInstance( + tagKey: string, + tagValue: string, +): Promise { + const result = await ec2.send( + new DescribeInstancesCommand({ + Filters: [ + { Name: `tag:${tagKey}`, Values: [tagValue] }, + { Name: 'instance-state-name', Values: ['pending', 'running'] }, + ], + }), + ); + const instance = result.Reservations?.flatMap((r) => r.Instances ?? [])[0]; + if (!instance?.InstanceId) { + return null; + } + return { + instanceId: instance.InstanceId, + state: instance.State?.Name ?? 'unknown', + launchTime: instance.LaunchTime, + retainUntil: parseRetainUntil(instance.Tags), + }; +} + +/** + * Find the newest available AMI matching the given filters (owned by this + * account). This is how the runtime discovers the baked image — the image + * pipeline tags each AMI, and we pick the most recently created. Returns null + * if none match (e.g. no bake has succeeded yet). + */ +export async function findLatestAmi(filters: Filter[]): Promise { + const result = await ec2.send(new DescribeImagesCommand({ Owners: ['self'], Filters: filters })); + const images = (result.Images ?? []).filter((i) => i.State === 'available' && i.ImageId); + if (images.length === 0) { + return null; + } + // CreationDate is ISO 8601, so a lexicographic sort is chronological. + images.sort((a, b) => (b.CreationDate ?? '').localeCompare(a.CreationDate ?? '')); + return images[0].ImageId ?? null; +} + +export interface LaunchSpec { + imageId: string; + instanceType: string; + subnetId: string; + securityGroupId: string; + instanceProfileArn: string; + userData: string; // plain text; encoded here + tags: Record; + /** + * Terminate (rather than stop) when the guest shuts itself down — how the + * seed instance disposes of itself once the weights are in S3. + */ + terminateOnShutdown?: boolean; +} + +/** + * Launch one instance in the given subnet. Throws on failure — the caller + * catches capacity/unsupported errors and retries in another AZ's subnet. + * Returns the new instance id. + */ +export async function runInstance(spec: LaunchSpec): Promise { + const input: RunInstancesCommandInput = { + ImageId: spec.imageId, + InstanceType: spec.instanceType as _InstanceType, + MinCount: 1, + MaxCount: 1, + // Idempotency: RunInstances is not idempotent, and the SDK auto-retries on + // transient errors. Without a token, a retry of a call that actually + // launched (but whose response was lost) launches a *second* instance — or + // hits the vCPU limit the first one just consumed. The token makes a retry + // return the same instance instead. + ClientToken: randomUUID(), + SubnetId: spec.subnetId, + SecurityGroupIds: [spec.securityGroupId], + IamInstanceProfile: { Arn: spec.instanceProfileArn }, + UserData: Buffer.from(spec.userData).toString('base64'), + MetadataOptions: { HttpTokens: 'required' }, + ...(spec.terminateOnShutdown ? { InstanceInitiatedShutdownBehavior: 'terminate' as const } : {}), + TagSpecifications: [ + { + ResourceType: 'instance', + Tags: Object.entries(spec.tags).map(([Key, Value]) => ({ Key, Value })), + }, + ], + }; + const result = await ec2.send(new RunInstancesCommand(input)); + const instanceId = result.Instances?.[0]?.InstanceId; + if (!instanceId) { + throw new Error('RunInstances returned no instance id'); + } + return instanceId; +} + +/** Errors that mean "this AZ can't take the instance" — try the next subnet. */ +export function isCapacityError(err: unknown): boolean { + return ['InsufficientInstanceCapacity', 'Unsupported', 'InstanceLimitExceeded'].includes( + errorName(err), + ); +} + +export async function terminateInstance(instanceId: string): Promise { + await ec2.send(new TerminateInstancesCommand({ InstanceIds: [instanceId] })); +} + +export async function associateEip(allocationId: string, instanceId: string): Promise { + await ec2.send( + new AssociateAddressCommand({ + AllocationId: allocationId, + InstanceId: instanceId, + AllowReassociation: true, + }), + ); +} + +export async function isSsmAgentOnline(instanceId: string): Promise { + const result = await ssm.send( + new DescribeInstanceInformationCommand({ + Filters: [{ Key: 'InstanceIds', Values: [instanceId] }], + }), + ); + return result.InstanceInformationList?.[0]?.PingStatus === 'Online'; +} + +export interface CommandResult { + status: string; + stdout: string; +} + +/** + * Run a shell command on the instance via SSM Run Command and wait for the + * result — how the Lambdas reach vLLM on localhost without living in the VPC. + * GetCommandInvocation truncates stdout at 24 000 chars, so commands must + * filter their own output (e.g. grep /metrics on-instance). + */ +export async function runShellCommand( + instanceId: string, + command: string, + timeoutSeconds = 30, +): Promise { + const sent = await ssm.send( + new SendCommandCommand({ + DocumentName: 'AWS-RunShellScript', + InstanceIds: [instanceId], + Parameters: { commands: [command], executionTimeout: [String(timeoutSeconds)] }, + }), + ); + const commandId = sent.Command?.CommandId; + if (!commandId) { + throw new Error('SendCommand returned no command id'); + } + + const deadline = Date.now() + (timeoutSeconds + 30) * 1000; + while (Date.now() < deadline) { + await sleep(1500); + try { + const invocation = await ssm.send( + new GetCommandInvocationCommand({ CommandId: commandId, InstanceId: instanceId }), + ); + const status = invocation.Status ?? 'Unknown'; + if (status !== 'Pending' && status !== 'InProgress' && status !== 'Delayed') { + return { status, stdout: (invocation.StandardOutputContent ?? '').trim() }; + } + } catch (err) { + // The invocation record is eventually consistent after SendCommand. + if (errorName(err) !== 'InvocationDoesNotExist') { + throw err; + } + } + } + return { status: 'Timeout', stdout: '' }; +} + +export async function readState(paramName: string): Promise { + const result = await ssm.send(new GetParameterCommand({ Name: paramName })); + try { + return JSON.parse(result.Parameter?.Value ?? '{}') as IdleState; + } catch { + return {}; + } +} + +export async function writeState(paramName: string, state: IdleState): Promise { + await ssm.send( + new PutParameterCommand({ + Name: paramName, + Value: JSON.stringify(state), + Type: 'String', + Overwrite: true, + }), + ); +} + +/** + * Read the deploy-config (what to serve) from its SSM parameter. Throws via + * parseDeployConfig if unset/invalid — the start Lambda surfaces that as a + * clear "run outfit remote deploy" rather than launching a mis-configured box. + */ +export async function readDeployConfig(paramName: string): Promise { + const result = await ssm.send(new GetParameterCommand({ Name: paramName })); + return parseDeployConfig(result.Parameter?.Value); +} + +/** Overwrite the deploy-config parameter — the deploy Lambda's one mutation. */ +export async function writeDeployConfig(paramName: string, config: DeployConfig): Promise { + await ssm.send( + new PutParameterCommand({ + Name: paramName, + Value: JSON.stringify(config), + Type: 'String', + Overwrite: true, + }), + ); +} diff --git a/remote/lambda/shared/deploy-config.ts b/remote/lambda/shared/deploy-config.ts new file mode 100644 index 00000000..5b298140 --- /dev/null +++ b/remote/lambda/shared/deploy-config.ts @@ -0,0 +1,167 @@ +/** + * The deploy-config contract — the runner-neutral description of *what* to + * serve, kept separate from the *infra* (bucket, secret, tags) that the CDK + * stack owns. It lives in an SSM parameter that the start Lambda reads at each + * wake, so switching model/runner/context is a parameter write, not a redeploy. + * + * CDK writes the initial value from config; `outfit remote deploy` overwrites + * it later. There is deliberately NO default runner — one of the two must be + * chosen, and an absent/invalid config fails the wake loudly rather than + * silently picking one. + */ + +export const RUNNERS = ['vllm', 'llamacpp'] as const; +export type Runner = (typeof RUNNERS)[number]; + +/** + * The placeholder CDK creates the deploy-config parameter with. It is a + * constant, so a later `cdk deploy` never reasserts (clobbers) a real config + * that `outfit remote deploy` or a manual edit wrote — the parameter is + * outfit/manual-owned. A wake reading this fails loudly; `pnpm deploy` seeds a + * real config over it, but only while it is still this placeholder. + */ +export const UNCONFIGURED_DEPLOY_CONFIG = 'unconfigured'; + +export function isRunner(value: unknown): value is Runner { + return typeof value === 'string' && (RUNNERS as readonly string[]).includes(value); +} + +/** + * Where a model's weights live under the weights bucket. Derived here rather + * than sent on the wire so callers (outfit) never need to know the S3 layout — + * runner + modelId + quant fully determine it, which also means the same model + * always resolves to the same prefix. + */ +export function weightsPrefixFor(runner: Runner, modelId: string, quant: string): string { + const parts = ['models', runner, modelId]; + if (quant) { + parts.push(quant); + } + return `${parts.join('/')}/`; +} + +export interface DeployConfig { + /** Which inference server runs on the instance. Required; no default. */ + runner: Runner; + /** Hugging Face repo id the weights came from (a GGUF repo for llamacpp). */ + modelId: string; + /** GGUF quant tag for llamacpp (e.g. "UD-Q6_K_XL"); empty for vllm. */ + quant: string; + /** + * S3 key prefix the instance syncs from. Always derived by + * `weightsPrefixFor` — never taken from the request body. + */ + weightsPrefix: string; + /** Context window in tokens — vLLM's --max-model-len / llama.cpp's --ctx-size. */ + contextSize: number; + /** The model name the API reports and clients request. */ + servedModelName: string; + /** Runner-specific extra flags appended to the serve command, pre-tokenised. */ + serveArgs: string[]; +} + +/** + * Parse and validate a deploy-config JSON blob. Throws with a clear message on + * anything malformed — the start Lambda surfaces that rather than guessing. + */ +export function parseDeployConfig(raw: string | undefined): DeployConfig { + if (!raw || !raw.trim() || raw.trim() === UNCONFIGURED_DEPLOY_CONFIG) { + throw new Error( + 'deploy-config is not set — run `pnpm deploy` (seeds the initial config) or `outfit remote deploy`', + ); + } + let obj: Record; + try { + obj = JSON.parse(raw) as Record; + } catch (err) { + throw new Error(`deploy-config is not valid JSON: ${(err as Error).message}`); + } + if (!isRunner(obj.runner)) { + throw new Error( + `deploy-config.runner must be one of ${RUNNERS.join('/')}, got ${JSON.stringify(obj.runner)}`, + ); + } + const modelId = requireString(obj, 'modelId'); + const servedModelName = requireString(obj, 'servedModelName'); + const contextSize = Number(obj.contextSize); + if (!Number.isInteger(contextSize) || contextSize <= 0) { + throw new Error(`deploy-config.contextSize must be a positive integer, got ${obj.contextSize}`); + } + const serveArgs = obj.serveArgs ?? []; + if (!Array.isArray(serveArgs) || serveArgs.some((a) => typeof a !== 'string')) { + throw new Error('deploy-config.serveArgs must be an array of strings'); + } + const quant = typeof obj.quant === 'string' ? obj.quant : ''; + return { + runner: obj.runner, + modelId, + quant, + // Derived, so any weightsPrefix in the request body is ignored. + weightsPrefix: weightsPrefixFor(obj.runner, modelId, quant), + contextSize, + servedModelName, + serveArgs: serveArgs as string[], + }; +} + +function requireString(obj: Record, key: string): string { + const value = obj[key]; + if (typeof value !== 'string' || value === '') { + throw new Error(`deploy-config.${key} must be a non-empty string`); + } + return value; +} + +/** + * Build the serve command for a runner. Kept pure (no I/O) so it is unit + * tested; the Lambda drops the returned argv into a systemd ExecStart. The + * weights are already synced to `modelDir`; the API key is passed by reference + * (env file / file) rather than value so it never lands in `ps`. + */ +export function buildServeCommand(cfg: DeployConfig, opts: { modelDir: string; port: number }): string { + const { modelDir, port } = opts; + if (cfg.runner === 'vllm') { + return [ + '/opt/llm/venv/bin/vllm', + 'serve', + modelDir, + '--served-model-name', + quote(cfg.servedModelName), + '--host', + '0.0.0.0', + '--port', + String(port), + '--max-model-len', + String(cfg.contextSize), + '--gpu-memory-utilization', + '0.92', + ...cfg.serveArgs, + ].join(' '); + } + // llamacpp: the GGUF is a single file synced into modelDir. llama-server + // reads the API key from a root-only file (--api-key-file) so it is not + // visible in the process list. --metrics is forced on (not left to the + // Outfit) because the idle scrape depends on the /metrics endpoint — without + // it the scrape 404s and a healthy box reads as idle and self-terminates. + return [ + 'llama-server', + '--model', + `${modelDir}/model.gguf`, + '--host', + '0.0.0.0', + '--port', + String(port), + '--api-key-file', + '/etc/llm/api-key', + '--ctx-size', + String(cfg.contextSize), + '--metrics', + '--alias', + quote(cfg.servedModelName), + ...cfg.serveArgs, + ].join(' '); +} + +function quote(value: string): string { + return `'${value.replace(/'/g, "'\\''")}'`; +} diff --git a/remote/lambda/shared/http.ts b/remote/lambda/shared/http.ts new file mode 100644 index 00000000..b77cf37c --- /dev/null +++ b/remote/lambda/shared/http.ts @@ -0,0 +1,13 @@ +import type { LambdaFunctionURLResult } from 'aws-lambda'; + +export function jsonResponse( + statusCode: number, + body: unknown, + extraHeaders: Record = {}, +): LambdaFunctionURLResult { + return { + statusCode, + headers: { 'content-type': 'application/json', ...extraHeaders }, + body: JSON.stringify(body), + }; +} diff --git a/remote/lambda/shared/idle.ts b/remote/lambda/shared/idle.ts new file mode 100644 index 00000000..58d828f7 --- /dev/null +++ b/remote/lambda/shared/idle.ts @@ -0,0 +1,197 @@ +/** + * Pure idle-detection logic, kept free of AWS calls so it can be unit tested. + */ + +import type { Runner } from './deploy-config'; + +export interface IdleState { + /** Last observed activity counter (sum of vLLM token/request counters). */ + counter?: number; + /** When the counter last changed. */ + last_change_at?: string; + /** When the start Lambda last reported the endpoint ready. */ + last_wake_at?: string; +} + +export type MetricsResult = { ok: true; running: number; counter: number } | { ok: false }; + +// The Prometheus metric names each runner exposes on /metrics that signal +// activity: `running` gauges (in-flight requests) plus cumulative `counters` +// (tokens/requests) that catch a long generation starting and ending between +// two idle ticks. vLLM uses a `vllm:` prefix, llama.cpp a `llamacpp:` one, with +// different names — so the scrape is runner-aware. +interface MetricsSpec { + prefix: string; + running: Set; + counters: Set; +} +const METRICS: Record = { + vllm: { + prefix: 'vllm', + running: new Set(['num_requests_running', 'num_requests_waiting']), + counters: new Set(['prompt_tokens_total', 'generation_tokens_total', 'request_success_total']), + }, + llamacpp: { + prefix: 'llamacpp', + running: new Set(['requests_processing', 'requests_deferred']), + counters: new Set(['prompt_tokens_total', 'tokens_predicted_total', 'n_decode_total']), + }, +}; + +/** The grep -E pattern that pulls just this runner's activity metrics. */ +export function metricsGrepPattern(runner: Runner): string { + const spec = METRICS[runner]; + return `^${spec.prefix}:(${[...spec.running, ...spec.counters].join('|')})`; +} + +/** + * Parse the (pre-grepped) Prometheus metrics scrape for a runner. Returns + * ok: false when the scrape failed or produced nothing recognisable — the + * caller treats that as "no activity observed" so a wedged server still gets + * stopped at the idle threshold instead of burning GPU-hours. + */ +export function parseMetrics(stdout: string, runner: Runner): MetricsResult { + if (!stdout || stdout.includes('SCRAPE_FAILED')) { + return { ok: false }; + } + const spec = METRICS[runner]; + const metricLine = new RegExp(`^${spec.prefix}:([a-z_]+)(?:\\{[^}]*\\})?\\s+([0-9.eE+-]+)$`); + let running = 0; + let counter = 0; + let matched = false; + for (const line of stdout.split('\n')) { + const match = metricLine.exec(line.trim()); + if (!match) { + continue; + } + const [, name, rawValue] = match; + const value = Number(rawValue); + if (!Number.isFinite(value)) { + continue; + } + if (spec.running.has(name)) { + running += value; + matched = true; + } else if (spec.counters.has(name)) { + counter += value; + matched = true; + } + } + return matched ? { ok: true, running, counter } : { ok: false }; +} + +export interface IdleDecisionInput { + now: Date; + launchTime: Date; + metrics: MetricsResult; + state: IdleState; + idleThresholdMinutes: number; + gracePeriodMinutes: number; + /** Hard cap: stop this long after launch even if requests are in flight. */ + maxRuntimeMinutes: number; + /** + * Manual override from the instance's Retain-Until tag: while this is in the + * future, do not terminate for any automatic reason (idle or max-runtime). + */ + retainUntil?: Date; +} + +export type IdleDecision = + | { action: 'wait'; reason: string } + | { action: 'update'; reason: string; newState: IdleState } + | { action: 'stop'; reason: string }; + +export function decideIdle(input: IdleDecisionInput): IdleDecision { + const { + now, + launchTime, + metrics, + state, + idleThresholdMinutes, + gracePeriodMinutes, + maxRuntimeMinutes, + retainUntil, + } = input; + + // A manual Retain-Until override beats every automatic reason to stop, + // including the hard cap — someone has explicitly pinned this instance alive + // (e.g. mid-debug). Only a manual stop overrides it. + if (retainUntil && retainUntil.getTime() > now.getTime()) { + return { + action: 'wait', + reason: `retained until ${retainUntil.toISOString()}`, + }; + } + + // The hard cap beats everything else, activity included — it is the backstop + // against a runaway session quietly burning GPU-hours. EC2 resets + // LaunchTime on every stop/start, so it caps a running session, not the + // instance's lifetime. + const minutesSinceLaunch = minutesBetween(launchTime, now); + if (minutesSinceLaunch > maxRuntimeMinutes) { + return { + action: 'stop', + reason: `running for ${minutesSinceLaunch.toFixed(1)} min, over the maximum runtime (${maxRuntimeMinutes} min)`, + }; + } + + if (minutesSinceLaunch < gracePeriodMinutes) { + return { + action: 'wait', + reason: `in grace period (${minutesSinceLaunch.toFixed(1)} min since launch)`, + }; + } + + if (metrics.ok) { + // "Changed" rather than "increased": a counter that reset to zero after a + // container restart still counts as activity and renews the grace window. + if (metrics.running > 0 || state.counter === undefined || metrics.counter !== state.counter) { + return { + action: 'update', + reason: + metrics.running > 0 + ? `${metrics.running} request(s) in flight` + : `activity counter moved (${state.counter ?? 'unset'} -> ${metrics.counter})`, + newState: { + counter: metrics.counter, + last_change_at: now.toISOString(), + last_wake_at: state.last_wake_at, + }, + }; + } + } + + // The instance is only ever considered idle relative to the most recent + // sign of life. last_wake_at closes the race where the start Lambda reports + // "ready" moments before an idle tick would otherwise stop the instance. + const anchor = Math.max( + parseTime(state.last_change_at), + parseTime(state.last_wake_at), + launchTime.getTime(), + ); + const idleMinutes = (now.getTime() - anchor) / 60_000; + if (idleMinutes > idleThresholdMinutes) { + return { + action: 'stop', + reason: `idle for ${idleMinutes.toFixed(1)} min (threshold ${idleThresholdMinutes})${ + metrics.ok ? '' : '; metrics scrape failed' + }`, + }; + } + return { + action: 'wait', + reason: `idle for ${idleMinutes.toFixed(1)} min (threshold ${idleThresholdMinutes})`, + }; +} + +function minutesBetween(from: Date, to: Date): number { + return (to.getTime() - from.getTime()) / 60_000; +} + +function parseTime(iso: string | undefined): number { + if (!iso) { + return 0; + } + const parsed = Date.parse(iso); + return Number.isNaN(parsed) ? 0 : parsed; +} diff --git a/remote/lambda/shared/seed.ts b/remote/lambda/shared/seed.ts new file mode 100644 index 00000000..a35d2abb --- /dev/null +++ b/remote/lambda/shared/seed.ts @@ -0,0 +1,120 @@ +/** + * Seeding the weights into S3. The deploy Lambda calls this when a config names + * a model whose weights are not in the bucket yet: a disposable instance + * downloads them from Hugging Face, syncs them to S3, then terminates itself. + * + * The seed always runs on the vLLM AMI regardless of the target runner — that + * is the image carrying a Python venv with huggingface_hub, and the job only + * needs that plus the AWS CLI. + */ + +import { HeadObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { errorName, findLatestAmi, runInstance } from './aws'; +import type { DeployConfig, Runner } from './deploy-config'; + +const s3 = new S3Client({}); + +/** + * The file whose presence means "these weights are complete". A bare + * list-under-the-prefix would also match the debris of a failed or in-flight + * seed (Hugging Face writes .cache/**\/*.lock files first), so a per-runner + * sentinel is checked instead — it is written last, by the S3 sync. + */ +function sentinelKey(runner: Runner, weightsPrefix: string): string { + return runner === 'llamacpp' ? `${weightsPrefix}model.gguf` : `${weightsPrefix}config.json`; +} + +/** Whether the weights for this config are already seeded. */ +export async function weightsPresent(bucket: string, cfg: DeployConfig): Promise { + const Key = sentinelKey(cfg.runner, cfg.weightsPrefix); + try { + await s3.send(new HeadObjectCommand({ Bucket: bucket, Key })); + return true; + } catch (err) { + const name = errorName(err); + if (name === 'NotFound' || name === 'NoSuchKey' || name === '404') { + return false; + } + throw err; + } +} + +export interface SeedEnv { + region: string; + bucket: string; + instanceType: string; + subnetId: string; + securityGroupId: string; + instanceProfileArn: string; + /** Empty when no Hugging Face token is configured (public repos only). */ + hfSecretArn: string; + amiRoleTagKey: string; + amiRoleTagValue: string; + amiRunnerTagKey: string; +} + +/** + * The seed instance's user-data. Kept pure so it can be unit tested: shell + * quoting bugs here surface as a silent seed failure 20 minutes later. + */ +export function buildSeedUserData(cfg: DeployConfig, env: SeedEnv): string { + const header = `#!/bin/bash +set -euxo pipefail +HF_TOKEN="" +${ + env.hfSecretArn + ? `HF_TOKEN=$(aws secretsmanager get-secret-value --secret-id '${env.hfSecretArn}' --region '${env.region}' --query SecretString --output text)` + : '' +} +export HF_TOKEN +export MODEL_ID='${cfg.modelId}' +mkdir -p /opt/llm/model +`; + + // llamacpp wants one GGUF (MTP is embedded in it), normalised to model.gguf + // so the runtime need not guess the filename; mmproj/projector files are + // excluded. vLLM wants the whole safetensors checkpoint. + const download = + cfg.runner === 'llamacpp' + ? `export QUANT='${cfg.quant}' +mkdir -p /tmp/dl +/opt/llm/venv/bin/python -c "import os; from huggingface_hub import snapshot_download; snapshot_download(os.environ['MODEL_ID'], allow_patterns=['*'+os.environ['QUANT']+'*'], local_dir='/tmp/dl', token=(os.environ.get('HF_TOKEN') or None))" +mapfile -t GGUFS < <(find /tmp/dl -type f -name '*.gguf' ! -iname '*mmproj*' | sort) +test "\${#GGUFS[@]}" -ge 1 +cp "\${GGUFS[0]}" /opt/llm/model/model.gguf +[ "\${#GGUFS[@]}" -gt 1 ] && echo "WARNING: \${#GGUFS[@]} gguf files for $QUANT; used the first (split quant not handled)" >&2 || true +` + : `/opt/llm/venv/bin/python -c "import os; from huggingface_hub import snapshot_download; snapshot_download(os.environ['MODEL_ID'], local_dir='/opt/llm/model', token=(os.environ.get('HF_TOKEN') or None))" +`; + + // The sync is last, so the sentinel only appears once the download succeeded + // (set -e aborts before it otherwise) and the box shuts down either way. + return `${header}${download}aws s3 sync /opt/llm/model/ 's3://${env.bucket}/${cfg.weightsPrefix}' --region '${env.region}' +shutdown -h now +`; +} + +/** + * Launch the seed instance. Returns its id, or throws if no AMI is baked yet. + */ +export async function launchSeedInstance(cfg: DeployConfig, env: SeedEnv): Promise { + const roleFilter = { Name: `tag:${env.amiRoleTagKey}`, Values: [env.amiRoleTagValue] }; + // Prefer the vLLM AMI (it has the venv); fall back to any runtime AMI for + // images baked before the runner tag existed. + const imageId = + (await findLatestAmi([roleFilter, { Name: `tag:${env.amiRunnerTagKey}`, Values: ['vllm'] }])) ?? + (await findLatestAmi([roleFilter])); + if (!imageId) { + throw new Error('no baked runtime AMI found — run `pnpm bake vllm` first'); + } + return runInstance({ + imageId, + instanceType: env.instanceType, + subnetId: env.subnetId, + securityGroupId: env.securityGroupId, + instanceProfileArn: env.instanceProfileArn, + userData: buildSeedUserData(cfg, env), + tags: { Name: 'cloud-vm-llm-seed' }, + terminateOnShutdown: true, + }); +} diff --git a/remote/lambda/start/index.ts b/remote/lambda/start/index.ts new file mode 100644 index 00000000..edc8ef05 --- /dev/null +++ b/remote/lambda/start/index.ts @@ -0,0 +1,349 @@ +import { GetSecretValueCommand, SecretsManagerClient } from '@aws-sdk/client-secrets-manager'; +import type { Context, LambdaFunctionURLEvent, LambdaFunctionURLResult } from 'aws-lambda'; +import { + associateEip, + errorName, + findLatestAmi, + findManagedInstance, + getInstance, + isCapacityError, + isSsmAgentOnline, + readDeployConfig, + readState, + requireEnv, + runInstance, + runShellCommand, + sleep, + writeState, +} from '../shared/aws'; +import { buildServeCommand, type DeployConfig } from '../shared/deploy-config'; +import { jsonResponse } from '../shared/http'; + +const secretsManager = new SecretsManagerClient({}); + +const TAG_KEY = requireEnv('TAG_KEY'); +const TAG_VALUE = requireEnv('TAG_VALUE'); +const VLLM_PORT = requireEnv('VLLM_PORT'); +const BASE_URL = requireEnv('BASE_URL'); +const STATE_PARAM_NAME = requireEnv('STATE_PARAM_NAME'); +const AMI_ROLE_TAG_KEY = requireEnv('AMI_ROLE_TAG_KEY'); +const AMI_ROLE_TAG_VALUE = requireEnv('AMI_ROLE_TAG_VALUE'); +const AMI_RUNNER_TAG_KEY = requireEnv('AMI_RUNNER_TAG_KEY'); +const INSTANCE_TYPE = requireEnv('INSTANCE_TYPE'); +const SUBNET_IDS = requireEnv('SUBNET_IDS').split(','); +const SECURITY_GROUP_ID = requireEnv('SECURITY_GROUP_ID'); +const INSTANCE_PROFILE_ARN = requireEnv('INSTANCE_PROFILE_ARN'); +const EIP_ALLOCATION_ID = requireEnv('EIP_ALLOCATION_ID'); +const API_KEY_SECRET_ARN = requireEnv('API_KEY_SECRET_ARN'); +const WEIGHTS_BUCKET = requireEnv('WEIGHTS_BUCKET'); +const DEPLOY_CONFIG_PARAM = requireEnv('DEPLOY_CONFIG_PARAM'); +const REGION = requireEnv('AWS_REGION'); + +const DEADLINE_MARGIN_MS = 20_000; +const POLL_MS = 5_000; +const HEALTH_POLL_MS = 10_000; +const TERMINAL_STATES = new Set(['shutting-down', 'terminated', 'stopping', 'stopped']); + +const HEALTH_COMMAND = + `curl -s -o /dev/null -w "%{http_code}" --max-time 5 http://localhost:${VLLM_PORT}/health || true`; + +export async function handler( + event: LambdaFunctionURLEvent, + context: Context, +): Promise { + const method = event.requestContext?.http?.method ?? 'POST'; + if (method === 'GET') { + return status(); + } + return wake(context); +} + +/** GET — report state without side effects. Used by `outfit remote status`. */ +async function status(): Promise { + const instance = await findManagedInstance(TAG_KEY, TAG_VALUE); + if (!instance || instance.state !== 'running') { + return jsonResponse(200, { state: instance?.state ?? 'stopped', healthy: false, base_url: BASE_URL }); + } + const healthy = (await isSsmAgentOnline(instance.instanceId)) && (await checkHealth(instance.instanceId)); + return jsonResponse(200, { state: 'running', healthy, base_url: BASE_URL }); +} + +/** POST — launch the instance if needed and block until vLLM is serving. */ +async function wake(context: Context): Promise { + const deadline = Date.now() + context.getRemainingTimeInMillis() - DEADLINE_MARGIN_MS; + + // What to serve comes from the deploy-config parameter. No default runner: an + // unset/invalid config fails the wake loudly rather than launching a guess. + let deployConfig: DeployConfig; + try { + deployConfig = await readDeployConfig(DEPLOY_CONFIG_PARAM); + } catch (err) { + return jsonResponse( + 503, + { + state: 'unconfigured', + message: `${(err as Error).message} — run \`outfit remote deploy\``, + retry_after_seconds: 300, + }, + { 'retry-after': '300' }, + ); + } + + const existing = await findManagedInstance(TAG_KEY, TAG_VALUE); + let instanceId: string; + if (existing) { + // Idempotent: a managed instance is already up (or coming up). + instanceId = existing.instanceId; + console.log(JSON.stringify({ phase: 'existing', instanceId, state: existing.state })); + } else { + const launched = await launchAcrossAzs(deployConfig); + if ('error' in launched) { + return launched.error; + } + instanceId = launched.instanceId; + } + + // Phase 1: EC2 state -> running (then pin the EIP so BASE_URL resolves). + // Right after RunInstances, DescribeInstances is eventually consistent and + // may briefly 404 the brand-new instance — tolerate that and keep polling. + while (Date.now() < deadline) { + let state: string; + try { + state = (await getInstance(instanceId)).state; + } catch (err) { + if (errorName(err) === 'InvalidInstanceID.NotFound') { + await sleep(POLL_MS); + continue; + } + throw err; + } + if (state === 'running') { + break; + } + if (TERMINAL_STATES.has(state)) { + return jsonResponse(500, { state, message: `instance went ${state} while starting` }); + } + await sleep(POLL_MS); + } + try { + await associateEip(EIP_ALLOCATION_ID, instanceId); + } catch (err) { + console.log(JSON.stringify({ phase: 'eip', error: errorName(err) })); + } + + // Phase 2: SSM agent online (registers 30-60 s after boot). + while (Date.now() < deadline) { + if (await isSsmAgentOnline(instanceId)) { + break; + } + await sleep(POLL_MS); + } + + // Phase 3: vLLM health. The server binds its port only once the engine has + // loaded the weights, so any HTTP answer (200, or 401 from the api-key + // middleware) means ready; connection refused means still loading. + while (Date.now() < deadline) { + if (await checkHealth(instanceId)) { + return ready(); + } + await sleep(HEALTH_POLL_MS); + } + + console.log(JSON.stringify({ phase: 'deadline', instanceId })); + return jsonResponse(503, { state: 'starting', retry_after_seconds: 60 }, { 'retry-after': '60' }); +} + +/** Try each AZ's subnet in turn, skipping ones without capacity. */ +async function launchAcrossAzs( + deployConfig: DeployConfig, +): Promise<{ instanceId: string } | { error: LambdaFunctionURLResult }> { + // Pick the newest AMI baked for THIS runner (role + runner tags). + const amiId = await findLatestAmi([ + { Name: `tag:${AMI_ROLE_TAG_KEY}`, Values: [AMI_ROLE_TAG_VALUE] }, + { Name: `tag:${AMI_RUNNER_TAG_KEY}`, Values: [deployConfig.runner] }, + { Name: 'state', Values: ['available'] }, + ]); + if (!amiId) { + return { + error: jsonResponse(503, { + state: 'no-ami', + message: `no baked AMI for runner "${deployConfig.runner}"; run \`pnpm bake ${deployConfig.runner}\` and wait`, + retry_after_seconds: 300, + }), + }; + } + const userData = buildUserData(deployConfig); + const tried: string[] = []; + for (const subnetId of SUBNET_IDS) { + try { + const instanceId = await runInstance({ + imageId: amiId, + instanceType: INSTANCE_TYPE, + subnetId, + securityGroupId: SECURITY_GROUP_ID, + instanceProfileArn: INSTANCE_PROFILE_ARN, + userData, + tags: { Name: 'cloud-vm-llm', [TAG_KEY]: TAG_VALUE }, + }); + console.log(JSON.stringify({ phase: 'launched', instanceId, subnetId })); + return { instanceId }; + } catch (err) { + if (isCapacityError(err)) { + console.log(JSON.stringify({ phase: 'capacity', subnetId, error: errorName(err) })); + tried.push(subnetId); + continue; + } + // The vCPU quota is regional, so trying other AZs won't help — return a + // clear message instead of crashing. Usually means an instance is already + // running, or the G-instance quota needs raising. + if (errorName(err) === 'VcpuLimitExceeded') { + return { + error: jsonResponse( + 503, + { + state: 'quota-exceeded', + message: + 'G-instance vCPU quota exhausted — an instance may already be running, or request a quota increase', + retry_after_seconds: 60, + }, + { 'retry-after': '60' }, + ), + }; + } + throw err; + } + } + return { + error: jsonResponse( + 503, + { + state: 'no-capacity', + message: `no g6e capacity in any of ${tried.length} availability zone(s); retry shortly`, + retry_after_seconds: 120, + }, + { 'retry-after': '120' }, + ), + }; +} + +function buildUserData(cfg: DeployConfig): string { + const modelDir = '/opt/llm/model'; + const serveCommand = buildServeCommand(cfg, { modelDir, port: Number(VLLM_PORT) }); + const runnerUnit = cfg.runner === 'vllm' ? vllmUnit(serveCommand) : llamacppUnit(serveCommand); + // Common boot: log the GPU, add swap for the load spike, sync the weights + // from S3, fetch the API key. Then the runner-specific env file + systemd + // unit take over. + return `#!/bin/bash +set -euxo pipefail +# Log the GPU state up front so cloud-init-output.log shows whether the driver +# loaded — the fastest way to tell a driver problem from a serving one. +nvidia-smi || echo "NVIDIA_SMI_FAILED" + +# Swap for OOM safety during model load. The FP8 checkpoint (~29 GB) is close +# to the 32 GB host RAM on g6e.xlarge, so a transient host-memory spike while +# loading could be OOM-killed. 16 GB of swap backstops that (the weights still +# live in VRAM; swap only catches host-RAM spikes). Created per boot rather +# than baked in, to keep the AMI slim; fallocate keeps it near-instant. +if ! swapon --show | grep -q /swapfile; then + fallocate -l 16G /swapfile || dd if=/dev/zero of=/swapfile bs=1M count=16384 + chmod 600 /swapfile + mkswap /swapfile + swapon /swapfile +fi +free -h + +MODEL_DIR=${modelDir} +mkdir -p "$MODEL_DIR" +aws s3 sync "s3://${WEIGHTS_BUCKET}/${cfg.weightsPrefix}" "$MODEL_DIR/" --region '${REGION}' + +API_KEY=$(aws secretsmanager get-secret-value --secret-id '${API_KEY_SECRET_ARN}' --region '${REGION}' --query SecretString --output text) +umask 077 + +${runnerUnit} +`; +} + +/** vLLM: env file (API key, offline, native sampler) plus its systemd unit. */ +function vllmUnit(serveCommand: string): string { + return `# Python dev headers: Triton JIT-compiles a CUDA stub against Python.h on the +# first model load (Qwen3.6's linear-attention path); baked into recipe 2.0.3+, +# this is a safety net for instances off an older AMI and a no-op once present. +if [ ! -f /usr/include/python3.12/Python.h ]; then + apt-get update && apt-get install -y python3.12-dev +fi + +cat >/etc/vllm.env </etc/systemd/system/vllm.service <<'UNIT' +[Unit] +Description=vLLM server +After=network-online.target +Wants=network-online.target +[Service] +EnvironmentFile=/etc/vllm.env +ExecStart=${serveCommand} +Restart=on-failure +RestartSec=5 +[Install] +WantedBy=multi-user.target +UNIT + +systemctl daemon-reload +systemctl enable --now vllm.service`; +} + +/** llama.cpp: the API key in a root-only file (--api-key-file) plus its unit. */ +function llamacppUnit(serveCommand: string): string { + return `mkdir -p /etc/llm +printf '%s' "$API_KEY" >/etc/llm/api-key +chmod 600 /etc/llm/api-key + +cat >/etc/systemd/system/llama-server.service <<'UNIT' +[Unit] +Description=llama.cpp server +After=network-online.target +Wants=network-online.target +[Service] +ExecStart=${serveCommand} +Restart=on-failure +RestartSec=5 +[Install] +WantedBy=multi-user.target +UNIT + +systemctl daemon-reload +systemctl enable --now llama-server.service`; +} + +async function checkHealth(instanceId: string): Promise { + try { + const result = await runShellCommand(instanceId, HEALTH_COMMAND, 30); + const code = result.stdout.trim(); + return result.status === 'Success' && code !== '' && code !== '000'; + } catch (err) { + console.log(JSON.stringify({ phase: 'health', error: errorName(err) })); + return false; + } +} + +async function ready(): Promise { + // Record the wake so the idle check gives the first request time to land. + const state = await readState(STATE_PARAM_NAME); + await writeState(STATE_PARAM_NAME, { ...state, last_wake_at: new Date().toISOString() }); + + const secret = await secretsManager.send( + new GetSecretValueCommand({ SecretId: API_KEY_SECRET_ARN }), + ); + console.log(JSON.stringify({ phase: 'ready' })); + return jsonResponse(200, { + state: 'ready', + base_url: BASE_URL, + api_key: secret.SecretString ?? '', + }); +} diff --git a/remote/lambda/stop/index.ts b/remote/lambda/stop/index.ts new file mode 100644 index 00000000..856c8696 --- /dev/null +++ b/remote/lambda/stop/index.ts @@ -0,0 +1,126 @@ +import type { LambdaFunctionURLEvent, LambdaFunctionURLResult, ScheduledEvent } from 'aws-lambda'; +import { + errorName, + findManagedInstance, + isSsmAgentOnline, + readDeployConfig, + readState, + requireEnv, + runShellCommand, + terminateInstance, + writeState, +} from '../shared/aws'; +import type { Runner } from '../shared/deploy-config'; +import { decideIdle, metricsGrepPattern, parseMetrics, type MetricsResult } from '../shared/idle'; +import { jsonResponse } from '../shared/http'; + +const TAG_KEY = requireEnv('TAG_KEY'); +const TAG_VALUE = requireEnv('TAG_VALUE'); +const VLLM_PORT = requireEnv('VLLM_PORT'); +const STATE_PARAM_NAME = requireEnv('STATE_PARAM_NAME'); +const IDLE_THRESHOLD_MINUTES = Number(requireEnv('IDLE_THRESHOLD_MINUTES')); +const GRACE_PERIOD_MINUTES = Number(requireEnv('GRACE_PERIOD_MINUTES')); +const MAX_RUNTIME_MINUTES = Number(requireEnv('MAX_RUNTIME_MINUTES')); +const DEPLOY_CONFIG_PARAM = requireEnv('DEPLOY_CONFIG_PARAM'); + +// Grep on the instance (GetCommandInvocation truncates stdout at 24 000 chars, +// and /metrics is far larger). The metric names are runner-specific, and +// llama.cpp gates /metrics behind the API key (vLLM leaves it open) — the key +// is in the same root-only file the server reads, and the SSM command runs as +// root, so the scrape can pass it. +function metricsCommand(runner: Runner): string { + const auth = runner === 'llamacpp' ? '-H "Authorization: Bearer $(cat /etc/llm/api-key)" ' : ''; + return ( + `curl -s --max-time 5 ${auth}http://localhost:${VLLM_PORT}/metrics` + + ` | grep -E '${metricsGrepPattern(runner)}'` + + ` || echo SCRAPE_FAILED` + ); +} + +type StopEvent = ScheduledEvent | LambdaFunctionURLEvent; + +export function isScheduledEvent(event: StopEvent): event is ScheduledEvent { + return (event as ScheduledEvent).source === 'aws.events'; +} + +export async function handler(event: StopEvent): Promise { + if (isScheduledEvent(event)) { + await idleCheck(); + return; + } + return manualStop(event); +} + +/** Function URL — POST terminates immediately; GET just reports state. */ +async function manualStop(event: LambdaFunctionURLEvent): Promise { + const instance = await findManagedInstance(TAG_KEY, TAG_VALUE); + const method = event.requestContext?.http?.method ?? 'POST'; + if (method === 'GET') { + return jsonResponse(200, { state: instance?.state ?? 'stopped' }); + } + if (instance) { + await terminateInstance(instance.instanceId); + console.log(JSON.stringify({ mode: 'manual', action: 'terminate', instanceId: instance.instanceId })); + return jsonResponse(200, { state: 'terminating' }); + } + console.log(JSON.stringify({ mode: 'manual', action: 'noop' })); + return jsonResponse(200, { state: 'stopped' }); +} + +/** EventBridge tick — terminate the instance if it has been idle long enough. */ +async function idleCheck(): Promise { + const instance = await findManagedInstance(TAG_KEY, TAG_VALUE); + if (!instance || instance.state !== 'running' || !instance.launchTime) { + console.log(JSON.stringify({ mode: 'idle', action: 'noop', state: instance?.state ?? 'none' })); + return; + } + + // The runner (which metric names to scrape) comes from the deploy-config. + // If it is unreadable, fall through with no metrics: decideIdle then treats + // this as "no activity observed" rather than crashing the tick. + let metrics: MetricsResult = { ok: false }; + try { + const { runner } = await readDeployConfig(DEPLOY_CONFIG_PARAM); + metrics = await scrapeMetrics(instance.instanceId, runner); + } catch (err) { + console.log(JSON.stringify({ mode: 'idle', warning: `deploy-config unreadable: ${errorName(err)}` })); + } + const state = await readState(STATE_PARAM_NAME); + const decision = decideIdle({ + now: new Date(), + launchTime: instance.launchTime, + metrics, + state, + idleThresholdMinutes: IDLE_THRESHOLD_MINUTES, + gracePeriodMinutes: GRACE_PERIOD_MINUTES, + maxRuntimeMinutes: MAX_RUNTIME_MINUTES, + retainUntil: instance.retainUntil, + }); + console.log(JSON.stringify({ mode: 'idle', decision: decision.action, reason: decision.reason })); + + if (decision.action === 'update') { + await writeState(STATE_PARAM_NAME, decision.newState); + } else if (decision.action === 'stop') { + await terminateInstance(instance.instanceId); + } +} + +async function scrapeMetrics(instanceId: string, runner: Runner): Promise { + try { + if (!(await isSsmAgentOnline(instanceId))) { + console.log(JSON.stringify({ mode: 'idle', warning: 'ssm agent offline' })); + return { ok: false }; + } + const result = await runShellCommand(instanceId, metricsCommand(runner), 30); + if (result.status !== 'Success') { + console.log(JSON.stringify({ mode: 'idle', warning: `scrape ${result.status}` })); + return { ok: false }; + } + return parseMetrics(result.stdout, runner); + } catch (err) { + // Treated as "no activity observed" by decideIdle: a crashed container + // gets terminated at the threshold rather than running up GPU-hours. + console.log(JSON.stringify({ mode: 'idle', warning: `scrape error ${errorName(err)}` })); + return { ok: false }; + } +} diff --git a/remote/lib/config.ts b/remote/lib/config.ts new file mode 100644 index 00000000..2bad9768 --- /dev/null +++ b/remote/lib/config.ts @@ -0,0 +1,243 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import type { App } from 'aws-cdk-lib'; +import { isRunner, RUNNERS, type Runner } from '../lambda/shared/deploy-config'; + +// Tag the image stack puts on every baked AMI, and the runtime start Lambda +// filters on to find the newest one to launch. The AMI is model-agnostic +// (just the driver + vLLM), so there is no model tag — the model comes from +// S3 at boot. +export const AMI_ROLE_TAG_KEY = 'cloud-vm-llm:role'; +export const AMI_ROLE_TAG_VALUE = 'runtime-ami'; +// The runner an AMI was baked for. Each runner has its own recipe/pipeline and +// its own AMI lineage, so the start Lambda (and seed) filter by both tags. +export const AMI_RUNNER_TAG_KEY = 'cloud-vm-llm:runner'; + +export interface LlmConfig { + /** CIDR allowed to reach the vLLM port, e.g. your home IP as a /32. Required. */ + allowedCidr: string; + region: string; + /** + * Which inference server to run: `vllm` or `llamacpp`. Required, no default — + * one must be chosen. CDK writes it into the initial deploy-config; thereafter + * `outfit remote deploy` owns it. + */ + runner: Runner; + /** + * Hugging Face model id, served at runtime from S3. Must be a pre-quantised + * checkpoint: BF16 weights for a 27B model are ~54 GB and do not fit the + * L40S's 48 GB. FP8 is hardware-native on the L40S (Ada); + * nvidia/Qwen3.6-27B-NVFP4 is the Blackwell-oriented alternative. + */ + modelId: string; + /** Optional Hugging Face token, used only for the first-boot seed of a gated repo. */ + hfToken: string; + instanceType: string; + /** Pinned vLLM version installed into the AMI's venv (uv pip install vllm==...). */ + vllmVersion: string; + /** + * Pinned ai-dock/llama.cpp-cuda release tag (e.g. "b10107") baked into the + * llamacpp AMI — a prebuilt CUDA `llama-server` (CUDA 12.8, amd64). ai-dock + * tracks upstream llama.cpp; pick a build new enough for MTP (PR #22673). + */ + llamacppRelease: string; + /** + * NVIDIA driver package installed in the AMI. The host needs only the driver + * — vLLM's torch wheels bring CUDA — so this is the "-server-open" headless + * driver (open kernel modules, required for Ada/L40S), not the CUDA toolkit. + */ + nvidiaDriverPackage: string; + maxModelLen: number; + /** Extra args appended to `vllm serve`, e.g. "--kv-cache-dtype fp8". */ + vllmExtraArgs: string; + /** + * vLLM tool-call parser (model-specific). When set, the server is started + * with `--enable-auto-tool-choice --tool-call-parser `, which coding + * agents need — they send `tool_choice: "auto"`. Qwen3.6 emits tool calls in + * Qwen's XML format, parsed by `qwen3_xml`. Empty disables tool calling. + */ + toolCallParser: string; + /** + * vLLM reasoning parser (model-specific). When set, adds `--reasoning-parser + * `, which splits the model's `` block into a separate field so + * it neither pollutes the reply nor blocks tool-call parsing. `qwen3` for + * Qwen3.6. Empty leaves reasoning inline in the content. + */ + reasoningParser: string; + /** Terminate the instance after this many minutes without requests. */ + idleThresholdMinutes: number; + /** Never idle-terminate within this many minutes of launch (model load time). */ + gracePeriodMinutes: number; + /** + * Hard cap on a running session: terminate this many minutes after launch, + * even if requests are still flowing. + */ + maxRuntimeMinutes: number; + vllmPort: number; + /** + * AZs the start Lambda tries, in order, when launching the instance — the + * g6e-capable zones. It launches into the first with capacity, so this + * replaces a single hard AZ pin with automatic per-AZ fallback. + */ + availabilityZones: string[]; + /** + * Instance type EC2 Image Builder uses to bake the AMI. The bake installs + * the driver and the vLLM venv but never runs the GPU, so a cheap non-GPU + * type is fine. + */ + builderInstanceType: string; + /** Root volume size (GB) of the baked AMI — fits the OS + driver + vLLM venv. */ + imageVolumeGb: number; +} + +const DEFAULTS = { + region: 'us-east-1', + modelId: 'Qwen/Qwen3.6-27B-FP8', + hfToken: '', + instanceType: 'g6e.xlarge', + // Must be new enough for the model's architecture — vLLM 0.11 predates + // Qwen3.6 (Qwen3_5ForConditionalGeneration) and rejects it at load. Pin a + // version that lists the model's architecture as supported. + vllmVersion: '0.26.0', + // ai-dock/llama.cpp-cuda release with CUDA 12.8; pin a specific build for + // reproducible bakes. Must post-date the MTP merge (PR #22673). + llamacppRelease: 'b10107', + nvidiaDriverPackage: 'nvidia-driver-570-server-open', + // 32k, not 64k: ~30 GB of FP8 weights leave ~14 GB on the 48 GB L40S, and a + // 64k KV cache for a 27B model is ~13 GB for one sequence — enough to OOM at + // startup. 32k halves the KV cache and is ample for a coding agent. Raise it + // (or add `--kv-cache-dtype fp8` via vllmExtraArgs) if you have headroom. + maxModelLen: 32768, + // --enforce-eager skips vLLM's torch.compile / CUDA-graph capture, which on + // a cold start with no cache takes 15-20 min and pins every core (starving + // the SSM agent and blowing past the idle timer). Eager mode makes wakes + // ~5 min at a modest single-user throughput cost — the right trade for a + // scale-to-zero box. Drop it if you want peak throughput and longer waits. + vllmExtraArgs: '--enforce-eager', + // Qwen3.6 emits tool calls in Qwen's XML format and wraps reasoning in + // ; coding agents send tool_choice: "auto", so both parsers are + // needed. qwen3_coder and qwen3_xml are aliases for the same parser. + toolCallParser: 'qwen3_coder', + reasoningParser: 'qwen3', + idleThresholdMinutes: 15, + // Must exceed the whole cold start (S3 sync ~4 min + weight/CUDA load), or + // the idle check terminates the instance mid-load (the metrics scrape fails + // while vLLM is still loading, which reads as "idle"). + gracePeriodMinutes: 30, + maxRuntimeMinutes: 240, + vllmPort: 8000, + availabilityZones: ['us-east-1b', 'us-east-1c', 'us-east-1d', 'us-east-1e'], + builderInstanceType: 'm5.xlarge', + // Big enough for the OS + driver + vLLM venv AND the ~30 GB model synced + // from S3 at boot. The snapshot only copies used blocks (~20 GB), so this + // does not slow the bake. + imageVolumeGb: 80, +} as const; + +function contextString(app: App, key: string, fallback: string): string { + const value = app.node.tryGetContext(key); + return value === undefined ? fallback : String(value); +} + +function contextList(app: App, key: string, fallback: readonly string[]): string[] { + const value = app.node.tryGetContext(key); + if (value === undefined) { + return [...fallback]; + } + const items = String(value) + .split(',') + .map((item) => item.trim()) + .filter(Boolean); + if (items.length === 0) { + throw new Error(`Context value "${key}" must be a non-empty comma-separated list`); + } + return items; +} + +function contextNumber(app: App, key: string, fallback: number): number { + const value = app.node.tryGetContext(key); + if (value === undefined) { + return fallback; + } + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`Context value "${key}" must be a positive number, got: ${value}`); + } + return parsed; +} + +/** + * Minimal .env parser (KEY=value lines, # comments) — enough to avoid a + * dotenv dependency. Recognised keys: ALLOWED_CIDR, HF_TOKEN. + */ +function loadDotEnv(dotEnvPath: string): Record { + if (!fs.existsSync(dotEnvPath)) { + return {}; + } + const values: Record = {}; + for (const line of fs.readFileSync(dotEnvPath, 'utf8').split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) { + continue; + } + const eq = trimmed.indexOf('='); + if (eq === -1) { + continue; + } + values[trimmed.slice(0, eq).trim()] = trimmed.slice(eq + 1).trim(); + } + return values; +} + +export function loadConfig( + app: App, + dotEnvPath: string = path.join(__dirname, '..', '.env'), +): LlmConfig { + // Secrets and machine-specific values live in the gitignored .env file; + // explicit -c context values always win over it. + const dotEnv = loadDotEnv(dotEnvPath); + + const allowedCidr = app.node.tryGetContext('allowedCidr') ?? dotEnv.ALLOWED_CIDR; + if (!allowedCidr) { + throw new Error( + 'allowedCidr is not set: run `pnpm set-ip` to write your current public ' + + 'IP to .env, or pass it explicitly with `-c allowedCidr=/32`.', + ); + } + if (!/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/.test(String(allowedCidr))) { + throw new Error(`Context value "allowedCidr" must be an IPv4 CIDR, got: ${allowedCidr}`); + } + + const region = contextString(app, 'region', DEFAULTS.region); + + // No default: one runner must be chosen explicitly (like allowedCidr). + const runner = contextString(app, 'runner', ''); + if (!isRunner(runner)) { + throw new Error( + `runner must be one of ${RUNNERS.join('/')} — set it with \`-c runner=\` (no default)`, + ); + } + + return { + allowedCidr: String(allowedCidr), + region, + runner, + modelId: contextString(app, 'modelId', DEFAULTS.modelId), + hfToken: contextString(app, 'hfToken', dotEnv.HF_TOKEN ?? DEFAULTS.hfToken), + instanceType: contextString(app, 'instanceType', DEFAULTS.instanceType), + vllmVersion: contextString(app, 'vllmVersion', DEFAULTS.vllmVersion), + llamacppRelease: contextString(app, 'llamacppRelease', DEFAULTS.llamacppRelease), + nvidiaDriverPackage: contextString(app, 'nvidiaDriverPackage', DEFAULTS.nvidiaDriverPackage), + maxModelLen: contextNumber(app, 'maxModelLen', DEFAULTS.maxModelLen), + vllmExtraArgs: contextString(app, 'vllmExtraArgs', DEFAULTS.vllmExtraArgs), + toolCallParser: contextString(app, 'toolCallParser', DEFAULTS.toolCallParser), + reasoningParser: contextString(app, 'reasoningParser', DEFAULTS.reasoningParser), + idleThresholdMinutes: contextNumber(app, 'idleThresholdMinutes', DEFAULTS.idleThresholdMinutes), + gracePeriodMinutes: contextNumber(app, 'gracePeriodMinutes', DEFAULTS.gracePeriodMinutes), + maxRuntimeMinutes: contextNumber(app, 'maxRuntimeMinutes', DEFAULTS.maxRuntimeMinutes), + vllmPort: DEFAULTS.vllmPort, + availabilityZones: contextList(app, 'availabilityZones', DEFAULTS.availabilityZones), + builderInstanceType: contextString(app, 'builderInstanceType', DEFAULTS.builderInstanceType), + imageVolumeGb: contextNumber(app, 'imageVolumeGb', DEFAULTS.imageVolumeGb), + }; +} diff --git a/remote/lib/image-stack.ts b/remote/lib/image-stack.ts new file mode 100644 index 00000000..49f018ac --- /dev/null +++ b/remote/lib/image-stack.ts @@ -0,0 +1,290 @@ +import * as cdk from 'aws-cdk-lib'; +import { + aws_ec2 as ec2, + aws_iam as iam, + aws_imagebuilder as imagebuilder, + aws_ssm as ssm, +} from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import { + AMI_ROLE_TAG_KEY, + AMI_ROLE_TAG_VALUE, + AMI_RUNNER_TAG_KEY, + type LlmConfig, +} from './config'; + +export interface ImageStackProps extends cdk.StackProps { + config: LlmConfig; +} + +// A plain Ubuntu 24.04 base — the bake adds only the NVIDIA driver and the +// chosen runner, so each AMI stays small and snapshots quickly. +const UBUNTU_SSM_PARAMETER = + '/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id'; + +// Root device of the Ubuntu 24.04 AMI (verify with describe-images if a bake +// adds a second volume instead of resizing the root). +const AMI_ROOT_DEVICE = '/dev/sda1'; + +// Per-runner recipe/component version. Image Builder treats a version as +// immutable, so bump a runner's version to force a fresh AMI for just it. +const RUNNER_VERSION = { vllm: '3.0.0', llamacpp: '3.0.1' } as const; + +/** + * Bakes a slim, model-agnostic AMI **per runner** — vLLM (a `uv` venv) and + * llama.cpp (a prebuilt CUDA `llama-server` from ai-dock) — each on the NVIDIA + * driver, no Docker. The model is NOT baked in; it lives in S3 and is synced at + * boot. Each runner has its own recipe/pipeline and its AMIs are tagged with + * the runner, so the runtime picks the right one. Defines Image Builder + * *pipelines*, so deploying never runs a build; trigger one with `pnpm bake`. + */ +export class ImageStack extends cdk.Stack { + constructor(scope: Construct, id: string, props: ImageStackProps) { + super(scope, id, props); + const cfg = props.config; + + // Minimal public VPC for the transient builder — internet + SSM, no NAT. + // Pinned to the first configured AZ so it doesn't default to us-east-1a, + // which lacks the builder instance type for this account. + const vpc = new ec2.Vpc(this, 'BuilderVpc', { + availabilityZones: [cfg.availabilityZones[0]], + natGateways: 0, + restrictDefaultSecurityGroup: false, + subnetConfiguration: [{ name: 'public', subnetType: ec2.SubnetType.PUBLIC }], + }); + const builderSg = new ec2.SecurityGroup(this, 'BuilderSg', { + vpc, + description: 'Image Builder bake instance - outbound only', + allowAllOutbound: true, + }); + + const builderRole = new iam.Role(this, 'BuilderRole', { + assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'), + managedPolicies: [ + iam.ManagedPolicy.fromAwsManagedPolicyName('EC2InstanceProfileForImageBuilder'), + iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'), + ], + }); + const builderProfile = new iam.CfnInstanceProfile(this, 'BuilderProfile', { + roles: [builderRole.roleName], + }); + + // One infrastructure config shared by both runners' pipelines. + const infrastructure = new imagebuilder.CfnInfrastructureConfiguration(this, 'Infra', { + name: `${this.stackName}-infra`, + instanceProfileName: builderProfile.ref, + instanceTypes: [cfg.builderInstanceType], + subnetId: vpc.publicSubnets[0].subnetId, + securityGroupIds: [builderSg.securityGroupId], + terminateInstanceOnFailure: true, + }); + + const parentImage = ssm.StringParameter.valueForStringParameter(this, UBUNTU_SSM_PARAMETER); + const nvidiaParam = { name: 'NvidiaDriverPackage', value: [cfg.nvidiaDriverPackage] }; + + const runnerBuilds = [ + { + runner: 'vllm', + data: vllmComponentDoc(), + parameters: [{ name: 'VllmVersion', value: [cfg.vllmVersion] }, nvidiaParam], + }, + { + runner: 'llamacpp', + data: llamacppComponentDoc(), + parameters: [{ name: 'LlamacppRelease', value: [cfg.llamacppRelease] }, nvidiaParam], + }, + ]; + + for (const build of runnerBuilds) { + const component = new imagebuilder.CfnComponent(this, `${build.runner}Component`, { + name: `${this.stackName}-${build.runner}-bake`, + platform: 'Linux', + version: RUNNER_VERSION[build.runner as keyof typeof RUNNER_VERSION], + data: build.data, + }); + + const recipe = new imagebuilder.CfnImageRecipe(this, `${build.runner}Recipe`, { + name: `${this.stackName}-${build.runner}-recipe`, + version: RUNNER_VERSION[build.runner as keyof typeof RUNNER_VERSION], + parentImage, + components: [{ componentArn: component.attrArn, parameters: [...build.parameters] }], + blockDeviceMappings: [ + { + deviceName: AMI_ROOT_DEVICE, + ebs: { + volumeSize: cfg.imageVolumeGb, + volumeType: 'gp3', + deleteOnTermination: true, + encrypted: true, + }, + }, + ], + }); + + // Tag each AMI with the role the runtime filters on AND the runner it was + // baked for, so the two runners' AMIs never collide under one tag. + const distribution = new imagebuilder.CfnDistributionConfiguration( + this, + `${build.runner}Distribution`, + { + name: `${this.stackName}-${build.runner}-distribution`, + distributions: [ + { + region: this.region, + amiDistributionConfiguration: { + Name: `cloud-vm-llm-${build.runner}-{{ imagebuilder:buildDate }}`, + AmiTags: { + [AMI_ROLE_TAG_KEY]: AMI_ROLE_TAG_VALUE, + [AMI_RUNNER_TAG_KEY]: build.runner, + }, + }, + }, + ], + }, + ); + + const pipeline = new imagebuilder.CfnImagePipeline(this, `${build.runner}Pipeline`, { + name: `${this.stackName}-${build.runner}-pipeline`, + imageRecipeArn: recipe.attrArn, + infrastructureConfigurationArn: infrastructure.attrArn, + distributionConfigurationArn: distribution.attrArn, + // No GPU test — the builder has no GPU; the driver is validated at + // runtime and the DKMS build is checked in the bake. + imageTestsConfiguration: { imageTestsEnabled: false }, + status: 'ENABLED', + }); + + new cdk.CfnOutput(this, `${build.runner}PipelineName`, { value: pipeline.name! }); + } + } +} + +/** + * Shared bake preamble: grow the root fs, install build tools + the AWS CLI, + * and the NVIDIA driver (open kernel modules, required for Ada/L40S). Both + * runners need the driver; the DKMS build needs build-essential + headers. + */ +function commonPreamble(): string { + return ` set -euxo pipefail + export DEBIAN_FRONTEND=noninteractive + + ROOT_SRC=$(findmnt -no SOURCE /) + ROOT_DISK=$(lsblk -no PKNAME "$ROOT_SRC") + ROOT_PARTNUM=$(echo "$ROOT_SRC" | grep -oE '[0-9]+$') + growpart "/dev/$ROOT_DISK" "$ROOT_PARTNUM" || true + resize2fs "$ROOT_SRC" || true + df -h / + + apt-get update + apt-get install -y build-essential "linux-headers-$(uname -r)" \\ + curl ca-certificates gnupg unzip + + # AWS CLI v2 — the runtime uses it to sync weights from S3 and read + # the API key from Secrets Manager. + curl -fsSL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o /tmp/awscliv2.zip + unzip -q /tmp/awscliv2.zip -d /tmp + /tmp/aws/install + aws --version + + # NVIDIA driver. The host needs only the driver; the runner brings + # its own CUDA (vLLM's torch wheels, or the ai-dock llama-server + # tarball). The DKMS module loads on the GPU instance at boot. + curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb -o /tmp/cuda-keyring.deb + dpkg -i /tmp/cuda-keyring.deb + apt-get update + apt-get install -y {{ NvidiaDriverPackage }} + dkms status | grep -i nvidia`; +} + +const CLEANUP = ` apt-get clean + rm -rf /var/lib/apt/lists/* /tmp/aws /tmp/awscliv2.zip /tmp/cuda-keyring.deb + echo "bake complete"`; + +/** vLLM AMI: the driver + a vLLM venv via uv (no Docker, no model). */ +function vllmComponentDoc(): string { + return `name: cloud-vm-llm-vllm-bake +description: Slim AMI with the NVIDIA driver and vLLM (via uv), model-agnostic +schemaVersion: 1.0 +parameters: + - VllmVersion: + type: string + - NvidiaDriverPackage: + type: string +phases: + - name: build + steps: + - name: Bake + action: ExecuteBash + inputs: + commands: + - | +${commonPreamble()} + + # python3.12-dev (Python.h) is a runtime need: Qwen3.6's + # linear-attention path imports Triton, which JIT-compiles a CUDA + # stub against the Python headers on first load. + apt-get install -y python3.12 python3.12-venv python3.12-dev + + # uv + vLLM into a venv baked into the AMI. + curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/usr/local/bin sh + /usr/local/bin/uv venv --python 3.12 /opt/llm/venv + /usr/local/bin/uv pip install --python /opt/llm/venv "vllm=={{ VllmVersion }}" + # Verify via package metadata — do NOT run "vllm --version", which + # probes for a GPU and fails on the CPU-only builder. + /opt/llm/venv/bin/python -c "import importlib.metadata as m; print('vllm', m.version('vllm'))" + +${CLEANUP} +`; +} + +/** + * llama.cpp AMI: the driver + a prebuilt CUDA `llama-server` from + * ai-dock/llama.cpp-cuda (tracks upstream; CUDA 12.8). No toolkit, no build — + * just download, extract, and install the binary plus its bundled libs. + */ +function llamacppComponentDoc(): string { + return `name: cloud-vm-llm-llamacpp-bake +description: Slim AMI with the NVIDIA driver and a prebuilt CUDA llama-server +schemaVersion: 1.0 +parameters: + - LlamacppRelease: + type: string + - NvidiaDriverPackage: + type: string +phases: + - name: build + steps: + - name: Bake + action: ExecuteBash + inputs: + commands: + - | +${commonPreamble()} + + # CUDA runtime libraries the prebuilt llama-server links against. + # The ai-dock tarball ships the llama.cpp/ggml libs but NOT the CUDA + # runtime; the driver provides libcuda, these provide + # libcudart/libcublas + NCCL (from the cuda repo added above). + apt-get install -y cuda-cudart-12-8 libcublas-12-8 libnccl2 + + # Prebuilt CUDA llama-server (ai-dock/llama.cpp-cuda, CUDA 12.8). + # Install the binary and every shared lib it ships beside it, then + # register the path so the runtime resolver finds them. + mkdir -p /opt/llm/bin /tmp/llama + REL='{{ LlamacppRelease }}' + curl -fsSL "https://github.com/ai-dock/llama.cpp-cuda/releases/download/$REL/llama.cpp-$REL-cuda-12.8-amd64.tar.gz" -o /tmp/llama.tar.gz + tar -xzf /tmp/llama.tar.gz -C /tmp/llama + SRV=$(find /tmp/llama -type f -name llama-server | head -1) + test -n "$SRV" + install -D -m 0755 "$SRV" /opt/llm/bin/llama-server + find "$(dirname "$SRV")" -name '*.so*' -exec cp -a {} /opt/llm/bin/ \\; + echo /opt/llm/bin > /etc/ld.so.conf.d/llm.conf + ldconfig + ln -sf /opt/llm/bin/llama-server /usr/local/bin/llama-server + # Verify it is an ELF binary (no GPU on the builder to run it). + file /opt/llm/bin/llama-server | grep -q ELF + + rm -rf /tmp/llama /tmp/llama.tar.gz +${CLEANUP} +`; +} diff --git a/remote/lib/llm-stack.ts b/remote/lib/llm-stack.ts new file mode 100644 index 00000000..3f158b39 --- /dev/null +++ b/remote/lib/llm-stack.ts @@ -0,0 +1,378 @@ +import * as cdk from 'aws-cdk-lib'; +import { + aws_ec2 as ec2, + aws_events as events, + aws_events_targets as targets, + aws_iam as iam, + aws_lambda as lambda, + aws_lambda_nodejs as nodejs, + aws_s3 as s3, + aws_secretsmanager as secretsmanager, + aws_ssm as ssm, +} from 'aws-cdk-lib'; +import { Construct } from 'constructs'; +import * as path from 'path'; +import { + AMI_ROLE_TAG_KEY, + AMI_ROLE_TAG_VALUE, + AMI_RUNNER_TAG_KEY, + type LlmConfig, +} from './config'; +import { UNCONFIGURED_DEPLOY_CONFIG, weightsPrefixFor } from '../lambda/shared/deploy-config'; + +export interface LlmStackProps extends cdk.StackProps { + config: LlmConfig; +} + +// Instances the start Lambda launches carry this tag; the idle/status Lambdas +// find "the current instance" by it. +const TAG_KEY = 'cloud-vm-llm'; +const TAG_VALUE = 'endpoint'; + +/** + * The scale-to-zero runtime. It holds no EC2 instance of its own: the start + * Lambda launches one from the slim baked AMI (found by tag) on demand, trying + * each g6e AZ until one has capacity, and the idle Lambda terminates it. The + * model weights live in the S3 bucket here (seeded once by `pnpm seed-model`) + * and are synced onto the instance at boot, so the instance is stateless and + * the AMI is model-agnostic. + */ +export class LlmStack extends cdk.Stack { + constructor(scope: Construct, id: string, props: LlmStackProps) { + super(scope, id, props); + const cfg = props.config; + + // Where the weights live in S3. modelId contains "/", which is a fine S3 + // key prefix. Seeded by `pnpm seed-model`, synced down at boot. + const weightsBucket = new s3.Bucket(this, 'Weights', { + // Retain on destroy — the weights are ~30 GB and re-seeding takes time. + removalPolicy: cdk.RemovalPolicy.RETAIN, + encryption: s3.BucketEncryption.S3_MANAGED, + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + enforceSSL: true, + }); + // Same derivation the deploy Lambda uses, so the initial config and the + // seed agree on where a model's weights live. + const weightsPrefix = weightsPrefixFor(cfg.runner, cfg.modelId, ''); + + // One public subnet per g6e AZ, so the start Lambda has a target in each + // zone to try. No NAT gateways — the instance gets a public IP directly. + const vpc = new ec2.Vpc(this, 'Vpc', { + availabilityZones: cfg.availabilityZones, + natGateways: 0, + restrictDefaultSecurityGroup: false, + subnetConfiguration: [{ name: 'public', subnetType: ec2.SubnetType.PUBLIC }], + }); + + const securityGroup = new ec2.SecurityGroup(this, 'VllmSg', { + vpc, + description: 'vLLM endpoint - ingress restricted to the allowed CIDR', + allowAllOutbound: true, // S3, Secrets Manager, SSM agent outbound + }); + securityGroup.addIngressRule( + ec2.Peer.ipv4(cfg.allowedCidr), + ec2.Port.tcp(cfg.vllmPort), + 'vLLM OpenAI-compatible API', + ); + + // vLLM's API key. The launched instance reads it via its role at boot; the + // start Lambda returns it to the client. + const apiKey = new secretsmanager.Secret(this, 'ApiKey', { + description: 'API key for the vLLM endpoint', + generateSecretString: { + excludePunctuation: true, + includeSpace: false, + passwordLength: 48, + }, + }); + + // Optional Hugging Face token, used only by the seed job for gated repos. + let hfSecret: secretsmanager.Secret | undefined; + if (cfg.hfToken) { + hfSecret = new secretsmanager.Secret(this, 'HfToken', { + description: 'Hugging Face token used only when seeding weights to S3', + secretStringValue: cdk.SecretValue.unsafePlainText(cfg.hfToken), + }); + } + + // Stable address so the endpoint base URL survives across launches. The + // start Lambda associates it with each freshly launched instance. + const eip = new ec2.CfnEIP(this, 'Eip', { domain: 'vpc' }); + const baseUrl = `http://${eip.attrPublicIp}:${cfg.vllmPort}/v1`; + + const idleState = new ssm.StringParameter(this, 'IdleState', { + parameterName: '/cloud-vm-llm/idle-state', + stringValue: '{}', + }); + + // What to serve — runner/model/context/serveArgs. The start Lambda reads it + // at each wake, so switching model/runner is a parameter write, not a + // redeploy. The parameter is outfit/manual-owned: CDK creates it with a + // constant placeholder (below) so a later `cdk deploy` never clobbers a + // real config, and `pnpm deploy` seeds this cfg-derived initial config over + // the placeholder ONLY while it is still unconfigured (see + // scripts/seed-deploy-config.mjs). The initial config is emitted as an + // output rather than baked into the parameter value for that reason. + const vllmServeArgs = [ + ...cfg.vllmExtraArgs.split(/\s+/).filter(Boolean), + ...(cfg.toolCallParser + ? ['--enable-auto-tool-choice', '--tool-call-parser', cfg.toolCallParser] + : []), + ...(cfg.reasoningParser ? ['--reasoning-parser', cfg.reasoningParser] : []), + ]; + const initialDeployConfig = { + runner: cfg.runner, + modelId: cfg.modelId, + quant: '', + weightsPrefix, + contextSize: cfg.maxModelLen, + servedModelName: cfg.modelId, + // llama.cpp's initial serveArgs come from the Outfit via `outfit remote + // deploy`; there is no sensible CDK-side default for them. + serveArgs: cfg.runner === 'vllm' ? vllmServeArgs : [], + }; + const deployConfigParam = new ssm.StringParameter(this, 'DeployConfig', { + parameterName: '/cloud-vm-llm/deploy-config', + stringValue: UNCONFIGURED_DEPLOY_CONFIG, + }); + + // Role assumed by the launched runtime instance — SSM (for the Lambdas' + // health/idle checks), read the API-key secret, and read the weights. + const instanceRole = new iam.Role(this, 'InstanceRole', { + assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'), + managedPolicies: [ + iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'), + ], + }); + apiKey.grantRead(instanceRole); + // Broad over the whole models/ tree: the deploy-config chooses the prefix + // (per runner + model + quant), so the grant can't be pinned to one. + weightsBucket.grantRead(instanceRole, 'models/*'); + const instanceProfile = new iam.InstanceProfile(this, 'InstanceProfile', { + role: instanceRole, + }); + + // Role for the disposable seed instance — write the weights it downloads + // from Hugging Face, and read the HF token if the repo is gated. + const seedRole = new iam.Role(this, 'SeedRole', { + assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'), + managedPolicies: [ + iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'), + ], + }); + weightsBucket.grantReadWrite(seedRole, 'models/*'); + hfSecret?.grantRead(seedRole); + const seedProfile = new iam.InstanceProfile(this, 'SeedProfile', { role: seedRole }); + + const runShellScriptDocArn = `arn:${cdk.Aws.PARTITION}:ssm:${cdk.Aws.REGION}::document/AWS-RunShellScript`; + + const describeStatement = new iam.PolicyStatement({ + actions: ['ec2:DescribeInstances', 'ssm:DescribeInstanceInformation', 'ssm:GetCommandInvocation'], + resources: ['*'], + }); + // ssm:SendCommand authorises the document AND the target instance as + // separate resources. A tag condition can only hold on the instance — the + // AWS-managed AWS-RunShellScript document is untagged, so putting both ARNs + // in one conditioned statement denies the document and the whole call fails + // with AccessDenied (which silently broke every health-check and idle + // scrape). Grant them in two statements: instance tag-scoped, document open. + const sendCommandStatements = () => [ + new iam.PolicyStatement({ + actions: ['ssm:SendCommand'], + resources: [`arn:${cdk.Aws.PARTITION}:ec2:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:instance/*`], + conditions: { StringEquals: { [`ssm:resourceTag/${TAG_KEY}`]: TAG_VALUE } }, + }), + new iam.PolicyStatement({ + actions: ['ssm:SendCommand'], + resources: [runShellScriptDocArn], + }), + ]; + + const commonEnv = { + TAG_KEY, + TAG_VALUE, + VLLM_PORT: String(cfg.vllmPort), + BASE_URL: baseUrl, + STATE_PARAM_NAME: idleState.parameterName, + }; + + const startFn = new nodejs.NodejsFunction(this, 'StartFn', { + description: 'Launches the baked AMI (per-AZ capacity fallback) and waits until vLLM serves', + entry: path.join(__dirname, '..', 'lambda', 'start', 'index.ts'), + handler: 'handler', + runtime: lambda.Runtime.NODEJS_22_X, + architecture: lambda.Architecture.ARM_64, + // 15 min (Lambda max) so a cold wake — S3 sync + weight load — can return + // "ready" in one call rather than making outfit poll through a 503. + timeout: cdk.Duration.seconds(900), + memorySize: 256, + environment: { + ...commonEnv, + AMI_ROLE_TAG_KEY, + AMI_ROLE_TAG_VALUE, + AMI_RUNNER_TAG_KEY, + INSTANCE_TYPE: cfg.instanceType, + SUBNET_IDS: vpc.publicSubnets.map((s) => s.subnetId).join(','), + SECURITY_GROUP_ID: securityGroup.securityGroupId, + INSTANCE_PROFILE_ARN: instanceProfile.instanceProfileArn, + EIP_ALLOCATION_ID: eip.attrAllocationId, + API_KEY_SECRET_ARN: apiKey.secretArn, + WEIGHTS_BUCKET: weightsBucket.bucketName, + // The model/runner/context/serveArgs come from the deploy-config + // parameter, read at wake — not baked into the Lambda env. + DEPLOY_CONFIG_PARAM: deployConfigParam.parameterName, + }, + }); + // RunInstances resource-level scoping is impractical (it spans instance, + // volume, network-interface, subnet, sg, image, profile), so it is granted + // broadly; CreateTags is limited to the launch. Terminate (in the stop + // Lambda) is tag-scoped, which bounds what these credentials can destroy. + startFn.addToRolePolicy( + new iam.PolicyStatement({ + actions: ['ec2:RunInstances', 'ec2:AssociateAddress'], + resources: ['*'], + }), + ); + startFn.addToRolePolicy( + new iam.PolicyStatement({ + actions: ['ec2:CreateTags'], + resources: [`arn:${cdk.Aws.PARTITION}:ec2:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:*/*`], + conditions: { StringEquals: { 'ec2:CreateAction': 'RunInstances' } }, + }), + ); + startFn.addToRolePolicy( + new iam.PolicyStatement({ + actions: ['iam:PassRole'], + resources: [instanceRole.roleArn], + conditions: { StringEquals: { 'iam:PassedToService': 'ec2.amazonaws.com' } }, + }), + ); + startFn.addToRolePolicy(describeStatement); + sendCommandStatements().forEach((s) => startFn.addToRolePolicy(s)); + // Find the latest baked AMI by tag. DescribeImages has no resource-level + // scoping, so it is granted broadly. + startFn.addToRolePolicy( + new iam.PolicyStatement({ actions: ['ec2:DescribeImages'], resources: ['*'] }), + ); + idleState.grantRead(startFn); + idleState.grantWrite(startFn); + deployConfigParam.grantRead(startFn); + apiKey.grantRead(startFn); + + const stopFn = new nodejs.NodejsFunction(this, 'StopFn', { + description: 'Terminates the instance - immediately (manual) or after idle (scheduled)', + entry: path.join(__dirname, '..', 'lambda', 'stop', 'index.ts'), + handler: 'handler', + runtime: lambda.Runtime.NODEJS_22_X, + architecture: lambda.Architecture.ARM_64, + timeout: cdk.Duration.seconds(120), + memorySize: 256, + environment: { + ...commonEnv, + IDLE_THRESHOLD_MINUTES: String(cfg.idleThresholdMinutes), + GRACE_PERIOD_MINUTES: String(cfg.gracePeriodMinutes), + MAX_RUNTIME_MINUTES: String(cfg.maxRuntimeMinutes), + // The idle scrape reads the runner from the deploy-config to pick the + // right /metrics names (vLLM and llama.cpp expose different ones). + DEPLOY_CONFIG_PARAM: deployConfigParam.parameterName, + }, + }); + stopFn.addToRolePolicy( + new iam.PolicyStatement({ + actions: ['ec2:TerminateInstances'], + resources: ['*'], + conditions: { StringEquals: { [`ec2:ResourceTag/${TAG_KEY}`]: TAG_VALUE } }, + }), + ); + stopFn.addToRolePolicy(describeStatement); + sendCommandStatements().forEach((s) => stopFn.addToRolePolicy(s)); + idleState.grantRead(stopFn); + idleState.grantWrite(stopFn); + deployConfigParam.grantRead(stopFn); + + // The control plane `outfit remote deploy` calls: it validates the posted + // DeployConfig and writes it to the deploy-config parameter. outfit needs + // only Lambda invoke (SigV4), no SSM/EC2 perms of its own. + const deployFn = new nodejs.NodejsFunction(this, 'DeployFn', { + description: 'Sets the deploy-config (runner/model/context) the next wake serves', + entry: path.join(__dirname, '..', 'lambda', 'deploy', 'index.ts'), + handler: 'handler', + runtime: lambda.Runtime.NODEJS_22_X, + architecture: lambda.Architecture.ARM_64, + timeout: cdk.Duration.seconds(60), + memorySize: 256, + environment: { + DEPLOY_CONFIG_PARAM: deployConfigParam.parameterName, + // Seeding: the Lambda launches the disposable download instance itself + // when the posted config names weights that are not in S3 yet. + WEIGHTS_BUCKET: weightsBucket.bucketName, + SEED_INSTANCE_TYPE: cfg.builderInstanceType, + SEED_SUBNET_ID: vpc.publicSubnets[0].subnetId, + SEED_SECURITY_GROUP_ID: securityGroup.securityGroupId, + SEED_INSTANCE_PROFILE_ARN: seedProfile.instanceProfileArn, + HF_TOKEN_SECRET_ARN: hfSecret?.secretArn ?? '', + AMI_ROLE_TAG_KEY, + AMI_ROLE_TAG_VALUE, + AMI_RUNNER_TAG_KEY, + }, + }); + deployConfigParam.grantRead(deployFn); + deployConfigParam.grantWrite(deployFn); + // Read-only on the weights: the Lambda only checks whether the sentinel + // object exists; the seed instance itself does the writing. + weightsBucket.grantRead(deployFn, 'models/*'); + deployFn.addToRolePolicy( + new iam.PolicyStatement({ + actions: ['ec2:DescribeImages', 'ec2:RunInstances', 'ec2:CreateTags'], + resources: ['*'], + }), + ); + // Only the seed role, and only to EC2 — so this cannot be used to hand the + // Lambda's caller a more privileged role. + deployFn.addToRolePolicy( + new iam.PolicyStatement({ + actions: ['iam:PassRole'], + resources: [seedRole.roleArn], + conditions: { StringEquals: { 'iam:PassedToService': 'ec2.amazonaws.com' } }, + }), + ); + + const startUrl = startFn.addFunctionUrl({ authType: lambda.FunctionUrlAuthType.AWS_IAM }); + const stopUrl = stopFn.addFunctionUrl({ authType: lambda.FunctionUrlAuthType.AWS_IAM }); + const deployUrl = deployFn.addFunctionUrl({ authType: lambda.FunctionUrlAuthType.AWS_IAM }); + + new events.Rule(this, 'IdleCheckRule', { + description: 'Periodic idle check for the vLLM instance', + schedule: events.Schedule.rate(cdk.Duration.minutes(5)), + targets: [new targets.LambdaFunction(stopFn)], + }); + + new cdk.CfnOutput(this, 'StartUrl', { value: startUrl.url }); + new cdk.CfnOutput(this, 'StopUrl', { value: stopUrl.url }); + new cdk.CfnOutput(this, 'DeployUrl', { value: deployUrl.url }); + new cdk.CfnOutput(this, 'BaseUrl', { value: baseUrl }); + new cdk.CfnOutput(this, 'EipAddress', { value: eip.attrPublicIp }); + new cdk.CfnOutput(this, 'ModelId', { value: cfg.modelId }); + new cdk.CfnOutput(this, 'Region', { value: this.region }); + // Consumed by `pnpm seed-model` to launch the disposable seed instance. + new cdk.CfnOutput(this, 'WeightsBucket', { value: weightsBucket.bucketName }); + // No WeightsPrefix output: the prefix is derived per deployed model and + // stored in the deploy-config parameter, so a stack-level value would only + // ever be a guess from the CDK defaults. + // `pnpm deploy`'s seed step (scripts/seed-deploy-config.mjs) writes this + // cfg-derived initial config over the parameter's `unconfigured` + // placeholder the first time, and never overwrites an outfit/manual value. + new cdk.CfnOutput(this, 'DeployConfigParam', { value: deployConfigParam.parameterName }); + new cdk.CfnOutput(this, 'InitialDeployConfig', { value: JSON.stringify(initialDeployConfig) }); + new cdk.CfnOutput(this, 'SeedInstanceProfileArn', { value: seedProfile.instanceProfileArn }); + new cdk.CfnOutput(this, 'SeedInstanceType', { value: cfg.builderInstanceType }); + new cdk.CfnOutput(this, 'SeedSubnetId', { value: vpc.publicSubnets[0].subnetId }); + new cdk.CfnOutput(this, 'SeedSecurityGroupId', { value: securityGroup.securityGroupId }); + new cdk.CfnOutput(this, 'HfTokenSecretArn', { value: hfSecret?.secretArn ?? '' }); + // `pnpm write-config` reads this from the outputs file to generate + // remote.json; the stable BaseUrl above becomes the Outfit's BASEURL. + new cdk.CfnOutput(this, 'OutfitRemoteConfig', { + value: `{"start_url":"${startUrl.url}","stop_url":"${stopUrl.url}","deploy_url":"${deployUrl.url}","region":"${this.region}"}`, + }); + } +} diff --git a/remote/package.json b/remote/package.json new file mode 100644 index 00000000..af6c67e6 --- /dev/null +++ b/remote/package.json @@ -0,0 +1,43 @@ +{ + "name": "cloud-vm-llm", + "version": "0.1.0", + "private": true, + "description": "Scale-to-zero self-hosted Qwen3.6-27B endpoint on AWS (vLLM on EC2, CDK)", + "packageManager": "pnpm@10.33.0", + "scripts": { + "build": "tsc --noEmit", + "set-ip": "bash scripts/set-ip", + "test": "vitest run", + "synth": "cdk synth", + "deploy:image": "cdk deploy cloud-vm-llm-image", + "bake": "bash scripts/bake", + "console": "bash scripts/console", + "seed-model": "node scripts/seed-model.mjs", + "seed-deploy-config": "node scripts/seed-deploy-config.mjs", + "deploy": "cdk deploy cloud-vm-llm --outputs-file cdk-outputs.json && node scripts/write-config.mjs && node scripts/seed-deploy-config.mjs", + "write-config": "node scripts/write-config.mjs", + "cdk": "cdk" + }, + "dependencies": { + "@aws-sdk/client-ec2": "^3.700.0", + "@aws-sdk/client-s3": "^3.1095.0", + "@aws-sdk/client-secrets-manager": "^3.700.0", + "@aws-sdk/client-ssm": "^3.700.0" + }, + "devDependencies": { + "@types/aws-lambda": "^8.10.145", + "@types/node": "^22.10.0", + "aws-cdk": "^2.170.0", + "aws-cdk-lib": "^2.170.0", + "constructs": "^10.4.2", + "esbuild": "^0.25.0", + "tsx": "^4.19.2", + "typescript": "^5.7.2", + "vitest": "^3.0.0" + }, + "pnpm": { + "onlyBuiltDependencies": [ + "esbuild" + ] + } +} diff --git a/remote/pnpm-lock.yaml b/remote/pnpm-lock.yaml new file mode 100644 index 00000000..b5cce5e1 --- /dev/null +++ b/remote/pnpm-lock.yaml @@ -0,0 +1,1939 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@aws-sdk/client-ec2': + specifier: ^3.700.0 + version: 3.1094.0 + '@aws-sdk/client-s3': + specifier: ^3.1095.0 + version: 3.1095.0 + '@aws-sdk/client-secrets-manager': + specifier: ^3.700.0 + version: 3.1094.0 + '@aws-sdk/client-ssm': + specifier: ^3.700.0 + version: 3.1094.0 + devDependencies: + '@types/aws-lambda': + specifier: ^8.10.145 + version: 8.10.162 + '@types/node': + specifier: ^22.10.0 + version: 22.20.1 + aws-cdk: + specifier: ^2.170.0 + version: 2.1133.0 + aws-cdk-lib: + specifier: ^2.170.0 + version: 2.262.0(constructs@10.7.1) + constructs: + specifier: ^10.4.2 + version: 10.7.1 + esbuild: + specifier: ^0.25.0 + version: 0.25.12 + tsx: + specifier: ^4.19.2 + version: 4.23.1 + typescript: + specifier: ^5.7.2 + version: 5.9.3 + vitest: + specifier: ^3.0.0 + version: 3.2.7(@types/node@22.20.1)(tsx@4.23.1) + +packages: + + '@aws-cdk/asset-awscli-v1@2.2.282': + resolution: {integrity: sha512-7hKMi5tTxDcKGIMIOq14PnY0GBcugW33Uh/2YHDZiEwSxLeFOCYBwhR+BFXONb/EJeVI3RETFgailNZbkcKF6g==} + + '@aws-cdk/asset-node-proxy-agent-v6@2.1.2': + resolution: {integrity: sha512-pDiuqH+qY3zM9lhhLjbKJ1tnKOHzQ2V4Wr/3qsxyKeKAkuPMI/BVGvZG1PbrikUw949cGVTfVEt4ETKKYnrj0Q==} + + '@aws-cdk/cloud-assembly-schema@54.14.0': + resolution: {integrity: sha512-JCZCzgp3SuXQVljaKqXnttHzcezEHt9Ag/YipK0XwUFD+Iz2T4jY7gUc3pA25Uq6pzY2n9DvO/nEU++dPXW4Rw==} + engines: {node: '>= 18.0.0'} + bundledDependencies: + - jsonschema + - semver + + '@aws-sdk/checksums@3.1000.20': + resolution: {integrity: sha512-uc5PMNcLcs+UBSLGsRjbMZKi3mgnPSalXpABw9Xjo0DMrv4gjIa/E4a02yJsuU4FF6Tmvis/JgGdKNIOoN4DQw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-ec2@3.1094.0': + resolution: {integrity: sha512-/ciub9Q2o4baii+4E7irh6de7V53hDu1zOW2BQC3BUisNAA0TTORr0fjaaPOGVKO79H2l1E4EYRz3ixiaQ2v7A==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-s3@3.1095.0': + resolution: {integrity: sha512-eeobm3TKci47CTTHIxc5s5UDjLL0gTKfjlcyzKU+NMoeIJ3flKPq51Fhi2nUDiMNbzsY5HAynM3bHAQFHxRTUw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-secrets-manager@3.1094.0': + resolution: {integrity: sha512-pOPasPHgz0+z6qcbX3gNzlOPuZ3JCgkUFczqdR4dC67g8nr5BCcJZ3La5GUxteCV1S2X98+3Q6mnZ23AtTbsNQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-ssm@3.1094.0': + resolution: {integrity: sha512-WWKlz+wzQggLnjkBDWpfBb9hV6bGk9El2gEhgUFddQlr8M8hwrEgLKue1+beaoDLOmRr/K3MHgL4PEw2jlVcgw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.976.0': + resolution: {integrity: sha512-0cjRaEdlVoOrsNb9pP5q1Syyc8pXw5xSj2Np2ryReRTr9FppIIRVSdZK4lbnfmc2Hvgux/xBOUU6baB7z8//uA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.977.1': + resolution: {integrity: sha512-KVtQRtc00ES/y+Sc3vYXeP6pCIcNlBJCZOwvqSy8ZpVGmbM5+IG+AfhuTKQ2oXmIVqZJewaGMMpzPkywC6xg0w==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.60': + resolution: {integrity: sha512-BAkxdoe7tpDDqCghGpuOeHQRbm/2znVvOQm0AvpQbA2tbfMN46doN4zx65fv85ImP3KADwc2zQPmbrlI9MPfMg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.61': + resolution: {integrity: sha512-qihs2ekMb89Nxd2JenCgVFhjbkb3EIo7HEBCBzyZACKVJdrLUZBLOmAE3xr0Sayml8n/jZSzwO/IufIiIzO7PQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.62': + resolution: {integrity: sha512-g/0fGqKTb9xpKdd9AtpmV5Eo3DFKbnkpA2+w0peISSlu7NfAoWOuYBFxsu+yWBtxU89ka55ezoZBCbFaS8pjYQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.63': + resolution: {integrity: sha512-yfozsS8wkWZEi/n6IsrodcFKBWZ0iNAezhJbTReMNc0z1Px17qdeAeuL1/wziCAmCZyXiW7QzP75ggJkBQv8jQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.973.5': + resolution: {integrity: sha512-ylubazcRfq2TVus/qXucSXeC42Qdjp5HQxTu68K/BsdMiZlcSLD1zkpoCgApXZX1Y6YJhtGGs7ZHhO/GuIgBlw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.973.6': + resolution: {integrity: sha512-jGLTW1bj148GL/6/IMlfY2fMYS9FtHOG+NahkFD4y0qkzYudNUahelxryY68/HGMslYuHClk1XaS/3b3eJzEkg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.67': + resolution: {integrity: sha512-CCygIKJ9YbI3n84OClSaSppkgKKHVj2TGT33c6FRORZrYNZQ1POmD+ip0FLYokiJAK7sSdc3YVkOsBm90oxWMQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.68': + resolution: {integrity: sha512-w6tNci6g7RqFpLhj1f5xseBvaNojb4Pkgp5Jp5apl9hrJtaf2AA+rX9+qlhlWUK6kcyAFYPA7emO+55zj+S98Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.71': + resolution: {integrity: sha512-HIg7Q2osBzajQwL+1Vkyh2E7Gim3eTNb9RHIsOxDGjW0eZg4oEKtRs5sioCnc73ilhaOm4gX2lHVF8J7+nt2rg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.72': + resolution: {integrity: sha512-blQ7F5QGzylnzeh5549zQLoCAiMHkXFLjFovEMaVy4b2X8JhUu+u9NXro1hyK95YHdVFNmBHKs2hIHtZchxKlQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.60': + resolution: {integrity: sha512-YIo3f99hM43QdYG8hDzwGemnR/pU95b0kramqSJUTleCqaB7+HwKf7YZFHqvOgTqZTPx/mRmNIqoDRr3U0Z3Tw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.61': + resolution: {integrity: sha512-xzRuj+fUVO4nkafKQJVKAF97kGpeQbfjuwmRrtGZNf42/1dkmcz6o7dswBy7alY0htQn5sCL1GWQYEykviWZkA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.973.4': + resolution: {integrity: sha512-BPdmL8sSBOCv4ngZ+3LHxyc3CNqDCEK37CHioCk7zGrTMY5sUtkH8q+o6qA80nn6w3/fyBPGNE7OIRlmoOxRQA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.973.5': + resolution: {integrity: sha512-fZRjjWhLFelsDoOYjqShQTrIGYC3Pf9Mx9Czf+1ikfQDgktxjze33dVo1q1/ZQ+T0qbtejVoHNHrfD5aJVpv/w==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.66': + resolution: {integrity: sha512-kSAziJboOmZmsR9/MTbiNjowl2BPes1bQuJpne4qAZ62ubi8fjfr/aupJSQje6udBoYxXTQbsL0e0kby2la3ng==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.67': + resolution: {integrity: sha512-FTNZ05gkPBA6CKbU3N4zPgybV+stdazwMOya75CmGdcJL7p8Fw/BdHP8WVxJd0mvzyPK2cg/C3gli58Ir4HgCw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-sdk-ec2@3.972.48': + resolution: {integrity: sha512-KIqurwP5A7AwAA2S3cd8y41IbxNFmxtTHi1ZzdEkDRLhreKFaDKk9zIJgclGfsOMaOmao+nb0S/CInOgdwS7Ig==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-sdk-s3@3.972.66': + resolution: {integrity: sha512-sTZKP1+SvyzWc/3alO8AM/PUiEzDCmn2P/KOcFFSk2UKzKZkB5ZxXdlHVtACE7DHaMFvx6DY0bognwHa1qTv4g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/nested-clients@3.997.34': + resolution: {integrity: sha512-Y9REVrSwmLM+Qy6sZJ7ofMC2S3Hr3tPP/4CzL5U1olPP7OGoF+6+Px0E49cVQBtSxJtyeLJMf0UaBErfeSahAA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/nested-clients@3.997.35': + resolution: {integrity: sha512-2MJfseVG/aXvIyOIBlYA/Oaf6qFDdsu4D8RKsEUdOQpVuLaor0BdxIBBtJLBNQQEe6Ku3YMvLljwb1MwVUpzRw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.996.41': + resolution: {integrity: sha512-QMUytg+FQMGouc8gHS00KoYih3+N6cqmVI/pQGOIo7Nr7OpQaiXjSYOuL+vsPZ1tymY4LAQ8MYcHJmws5LRxng==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.996.42': + resolution: {integrity: sha512-DBV4naZP6HYBlAvPpoQzOP12Wvfou/5rN8yJPXjBTBylU5qwCbh/tXr2MddHoIjgoRkEl/eS+IljiUqvmwey1Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1092.0': + resolution: {integrity: sha512-hBYUAr6iBLNFcsiWTgtBb0stdSw39VOUq4Sp4A5caCNf66BAZplWN4FleKrVpJx5li2YgdnK2DqoFSMWC642FQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1095.0': + resolution: {integrity: sha512-65SudS6y4nzaYHybtqcpm3sHe5jLhdMn68HRKS1nUx690BtQeaAQOoujQ+dpOjBATIVGVgKKjEP8tR+U06QJQA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.974.2': + resolution: {integrity: sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/xml-builder@3.972.36': + resolution: {integrity: sha512-RdGmS1GLrtaTOLE1ElSluMldNrpk9Emq6uYs8SS8iHlu5xTAmM9rRkM91o48+rIRryBtyO9t+uLYCoMG6jVMVA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/xml-builder@3.972.37': + resolution: {integrity: sha512-zKq4HQum8JwDyEuyfuI4bbiAcU0KxP6qy+9PR/IsR92IyE/DaBAikzAS50tjxip4bqIIANpCcG+Yyj6CVhXupg==} + engines: {node: '>=20.0.0'} + + '@aws/lambda-invoke-store@0.3.0': + resolution: {integrity: sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==} + engines: {node: '>=18.0.0'} + + '@esbuild/aix-ppc64@0.25.12': + resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/aix-ppc64@0.28.1': + resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.25.12': + resolution: {integrity: sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm64@0.28.1': + resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.25.12': + resolution: {integrity: sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-arm@0.28.1': + resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.25.12': + resolution: {integrity: sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/android-x64@0.28.1': + resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.25.12': + resolution: {integrity: sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-arm64@0.28.1': + resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.25.12': + resolution: {integrity: sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.1': + resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.25.12': + resolution: {integrity: sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-arm64@0.28.1': + resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.25.12': + resolution: {integrity: sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.1': + resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.25.12': + resolution: {integrity: sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm64@0.28.1': + resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.25.12': + resolution: {integrity: sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-arm@0.28.1': + resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.25.12': + resolution: {integrity: sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-ia32@0.28.1': + resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.25.12': + resolution: {integrity: sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-loong64@0.28.1': + resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.25.12': + resolution: {integrity: sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-mips64el@0.28.1': + resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.25.12': + resolution: {integrity: sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-ppc64@0.28.1': + resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.25.12': + resolution: {integrity: sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.1': + resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.25.12': + resolution: {integrity: sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-s390x@0.28.1': + resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.25.12': + resolution: {integrity: sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/linux-x64@0.28.1': + resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.25.12': + resolution: {integrity: sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-arm64@0.28.1': + resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.25.12': + resolution: {integrity: sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.1': + resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.25.12': + resolution: {integrity: sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-arm64@0.28.1': + resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.25.12': + resolution: {integrity: sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.1': + resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.25.12': + resolution: {integrity: sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/openharmony-arm64@0.28.1': + resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.25.12': + resolution: {integrity: sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/sunos-x64@0.28.1': + resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.25.12': + resolution: {integrity: sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-arm64@0.28.1': + resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.25.12': + resolution: {integrity: sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-ia32@0.28.1': + resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.25.12': + resolution: {integrity: sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@esbuild/win32-x64@0.28.1': + resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@rollup/rollup-android-arm-eabi@4.62.2': + resolution: {integrity: sha512-6o7ZLZK+BeenkZCFNDXqpbjw9bD6nuWonvS/lwQJp7NoVVxm6p3qE7qQ5jGuBjiFsgvqjD8mZAU5oWxTmbOeOg==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.62.2': + resolution: {integrity: sha512-BaH7BllCACHoH1LguOU56UItGfUWjujlO65kS9LAodViaN4bwIKd7oeW/ZHJ/4ljr/7MIiENnNy3HJ0zXv8Zkw==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.62.2': + resolution: {integrity: sha512-v39RCCvj4He82I9sFmk+M1VZ0PLM9sfsLVikjfx2hYBNALhrrOR2D3JjQA6AhlaSOgcR+RzrKY7e1+bT6SUO/A==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.62.2': + resolution: {integrity: sha512-yl0y2vq3S3lHeuXhEdss6TWfKW8vkujImO12tn4ZkG/4oghr09LvdYm2RElVjokTQiUvDUGXLGsYeLqUMCKpGA==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.62.2': + resolution: {integrity: sha512-tT4pvt4qXD+vEoezupCWi+a1F0vvDiksiHc+PxRlYTOH1I6/X4id9jPxTP+Fg+545euaFT1jJVs4CEdHZAU1vw==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.62.2': + resolution: {integrity: sha512-6nU5F2wCW+qvCBhTn1pdIU3bzsIoF7EUwsCDRxilWGprQR6yd508YnH9+OKFCwpfS8pjZqDUmnCAr7exax0XCg==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.62.2': + resolution: {integrity: sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm-musleabihf@4.62.2': + resolution: {integrity: sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==} + cpu: [arm] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-arm64-gnu@4.62.2': + resolution: {integrity: sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm64-musl@4.62.2': + resolution: {integrity: sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-loong64-gnu@4.62.2': + resolution: {integrity: sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==} + cpu: [loong64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-loong64-musl@4.62.2': + resolution: {integrity: sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==} + cpu: [loong64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-ppc64-gnu@4.62.2': + resolution: {integrity: sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-ppc64-musl@4.62.2': + resolution: {integrity: sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==} + cpu: [ppc64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-riscv64-gnu@4.62.2': + resolution: {integrity: sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-riscv64-musl@4.62.2': + resolution: {integrity: sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-s390x-gnu@4.62.2': + resolution: {integrity: sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-gnu@4.62.2': + resolution: {integrity: sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-musl@4.62.2': + resolution: {integrity: sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rollup/rollup-openbsd-x64@4.62.2': + resolution: {integrity: sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.62.2': + resolution: {integrity: sha512-8hOJnxgbyObnCm5AlRA3A931xX19xq80RjVTKgJOvEKWqJruP/Uf12IbAOaDjjEXYRewwHLfmF0YRIdK3OwKWA==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.62.2': + resolution: {integrity: sha512-mmF4AY1i0hG/bLWUctUq59gtmgaSIRa3cu/A3JFRp/sCNEme2bgDEiDS22P9FbnJB8NJNF4jPJiSP5RHQpUTDg==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.62.2': + resolution: {integrity: sha512-DZgkknc6jhHrk46V25vbAM0zZkyP0nSDkJB8/dRkLTxv470dOmWDqGoEJl/9A0dFfS7yE3REOwNDxpHwSLSt0Q==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.62.2': + resolution: {integrity: sha512-T6xr6ucWSFto+VGajA8YH26LdpHRuP4YLHEKAtCWvJDOlnmWcDZVCI2Jmjr+IFHDlt2zRaTAKE4tfjTaWLgJBg==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.62.2': + resolution: {integrity: sha512-BfzEnDJOt9T8M989/lA37EcJgat01wLRnoi5dQf3QzOH7jzpqTAzdDbVfRljVr5r+jzKqpbHeyOfAaXxAd0PAA==} + cpu: [x64] + os: [win32] + + '@smithy/core@3.29.8': + resolution: {integrity: sha512-rpCbCV+TimOBi3VLNBMmtTvgfOWcFIEAru3+TFlG87SL2F+te4jOnnNR+cf3uR4eJ5Qf4LnT80fqnBKgPRS6zA==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.4.13': + resolution: {integrity: sha512-X+2HNZhWi5i3rJsCas0LPf6fTQUaKyJ40zd8aTO/bwpRfpU3biYaqLr7C1WMibL7PVKJalpi1PyybjGPNoHC8Q==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.6.10': + resolution: {integrity: sha512-5/Yj9mS2JjTsB3B8ZX7euh77mrY9aXW23ag1yAmFykSRmA6vldqBrgqmSeQ50EjY+5SB8+aE4w14B6LKbBVEhQ==} + engines: {node: '>=18.0.0'} + + '@smithy/node-http-handler@4.9.10': + resolution: {integrity: sha512-ETQz9v/Z+nTQc6fRWTXxUpxJqwpmzB3Tn3WKAdHwWkeT+m+HE5czs6GNG8vW+4vyxXSls65RVcvOZwk7Q/PS/Q==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.6.9': + resolution: {integrity: sha512-g5rnEii/mkT0mjVJmlsaOfyNBtHNTecD9Lo4NP8D5HzMUEnZNpz7/FbvBCjNcV4vteHFAxOGiLUYNxPkDZZAPw==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.16.1': + resolution: {integrity: sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==} + engines: {node: '>=18.0.0'} + + '@types/aws-lambda@8.10.162': + resolution: {integrity: sha512-Fn658grtLOci1oxi1391vvDWJRKNGWRSqfxRkmN/Iy3c0tQH1USMKEXcPYHLvope+ZgTFocx9FRQJx1muBL6qw==} + + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@types/node@22.20.1': + resolution: {integrity: sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==} + + '@vitest/expect@3.2.7': + resolution: {integrity: sha512-E8eBXaKibuvH2pSZErOjdVb5vF4PbKYcrnluBTYxEk1l/VhhwZg1kZQsdtjq+CsF5CFydf2Rdkz7jDHKSisi3w==} + + '@vitest/mocker@3.2.7': + resolution: {integrity: sha512-Trr0hYO9CM3Wj6ksWHRhK9IZpIY6wTMO5u/MqXurMxT57sWBaOPEtP3Oq60ihZuh5JsiagKfz95OcxdEP6dBrA==} + peerDependencies: + msw: ^2.4.9 + vite: ^5.0.0 || ^6.0.0 || ^7.0.0-0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@3.2.7': + resolution: {integrity: sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==} + + '@vitest/runner@3.2.7': + resolution: {integrity: sha512-sB9y4ovltoQP+WaUPwmSxO9WIg9Ig694Di5PalVPsYHklAdE027mehpWF2SQSVq+k6sFgaivbTjTJwZLSHbedA==} + + '@vitest/snapshot@3.2.7': + resolution: {integrity: sha512-7C+MwShwtBSI5Buwoyg3s/iY1eHL9PKAf+O1wVh/TdnjXUtkoL/9YQtre90i4MtNXM6edP1wJ2zOBpfCyhIS7g==} + + '@vitest/spy@3.2.7': + resolution: {integrity: sha512-Q2eQGI6d2L/hBtZ0qNuKcAGid68XK6cv1xsoaIma6PaJhHPoqcEJhYpXZ/5myCMqkNgtP6UKuBhbc0nHKnrkuQ==} + + '@vitest/utils@3.2.7': + resolution: {integrity: sha512-x6BDOd7dyo3PFLY3I9/HJ25X/6OurhGXk2/B9gOZNPF7XDVjeBK4k01lQE5uvDpbuheErh91qYuE1E2OEjK3Rw==} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + + aws-cdk-lib@2.262.0: + resolution: {integrity: sha512-6zRVoWRd8kQs9ZZ9xhERSm36W8uWS2vW3/g9Zx0xlhvRRiUwTc80bzfJkohKGdT/5AOW+wy6fSDvohavG94pcA==} + engines: {node: '>= 20.0.0'} + peerDependencies: + constructs: ^10.5.0 + bundledDependencies: + - '@aws/cloudformation-validate' + - '@balena/dockerignore' + - '@aws-cdk/cloud-assembly-api' + - case + - fs-extra + - ignore + - jsonschema + - minimatch + - punycode + - semver + - yaml + - mime-types + + aws-cdk@2.1133.0: + resolution: {integrity: sha512-DGlCBwqxSHTe1u/IoT5WV+Bbd2K3UhwFHsdMqb2nQa1fkJmaQgoNFu0It+p3HxyMWeW+gKsVzT5KcjQPQ6Kzuw==} + engines: {node: '>= 18.0.0'} + hasBin: true + + bowser@2.14.1: + resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} + + cac@6.7.14: + resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} + engines: {node: '>=8'} + + chai@5.3.3: + resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} + engines: {node: '>=18'} + + check-error@2.1.3: + resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} + engines: {node: '>= 16'} + + constructs@10.7.1: + resolution: {integrity: sha512-ulK25Sg2Nv1jW+A9VeV7fu9GFN9KdVTNWdXMoapNRwqkQzSdToro/Tttk3Ak5BGI80NRA/d2nSzKnoZ27LizLw==} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + deep-eql@5.0.2: + resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} + engines: {node: '>=6'} + + es-module-lexer@1.7.0: + resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + + esbuild@0.25.12: + resolution: {integrity: sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==} + engines: {node: '>=18'} + hasBin: true + + esbuild@0.28.1: + resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} + engines: {node: '>=18'} + hasBin: true + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + expect-type@1.4.0: + resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} + engines: {node: '>=12.0.0'} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + js-tokens@9.0.1: + resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} + + loupe@3.2.1: + resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + nanoid@3.3.16: + resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + pathval@2.0.1: + resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} + engines: {node: '>= 14.16'} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.5: + resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==} + engines: {node: '>=12'} + + postcss@8.5.22: + resolution: {integrity: sha512-KBDEIpLrvpv16pp3K0Fw+UCoZfopFjjgeB+0tA/aaThfEE74kKDLrgg603YvOWJyg3+WYtyq3xYsQWsIyZlPqQ==} + engines: {node: ^10 || ^12 || >=14} + + rollup@4.62.2: + resolution: {integrity: sha512-RFnrW4lhXA3s3eqHDZvN654g8OTjzRfqpIRJYczCGB6HzphckVAi/Qh4tbPUbRuDi7s1Llv8g/NspLkttY3gTA==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@3.10.0: + resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + + strip-literal@3.1.0: + resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} + + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@0.3.2: + resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + + tinypool@1.1.1: + resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} + engines: {node: ^18.0.0 || >=20.0.0} + + tinyrainbow@2.0.0: + resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} + engines: {node: '>=14.0.0'} + + tinyspy@4.0.4: + resolution: {integrity: sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==} + engines: {node: '>=14.0.0'} + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + tsx@4.23.1: + resolution: {integrity: sha512-GQHnkIfxyx1wYCOS/wonik5MVRZU9hi1TEZmzGZSCJB1y9YgoZ8H6itNE/u4suE+yLmOzuE4E5S4TZ/ZX2wcWQ==} + engines: {node: '>=18.0.0'} + hasBin: true + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + + vite-node@3.2.4: + resolution: {integrity: sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + + vite@7.3.6: + resolution: {integrity: sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + jiti: '>=1.21.0' + less: ^4.0.0 + lightningcss: ^1.21.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + jiti: + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitest@3.2.7: + resolution: {integrity: sha512-KrxIJ62Fd89gfysR4WotlgZABiz2dqFPgqGzX7s+CwsqLFomRH7777ZcrOD6+WVAh7khPQP41A+BKbpcJFrdEg==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@types/debug': ^4.1.12 + '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 + '@vitest/browser': 3.2.7 + '@vitest/ui': 3.2.7 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@types/debug': + optional: true + '@types/node': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + +snapshots: + + '@aws-cdk/asset-awscli-v1@2.2.282': {} + + '@aws-cdk/asset-node-proxy-agent-v6@2.1.2': {} + + '@aws-cdk/cloud-assembly-schema@54.14.0': {} + + '@aws-sdk/checksums@3.1000.20': + dependencies: + '@aws-sdk/core': 3.977.1 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/client-ec2@3.1094.0': + dependencies: + '@aws-sdk/core': 3.976.0 + '@aws-sdk/credential-provider-node': 3.972.71 + '@aws-sdk/middleware-sdk-ec2': 3.972.48 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/fetch-http-handler': 5.6.10 + '@smithy/node-http-handler': 4.9.10 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/client-s3@3.1095.0': + dependencies: + '@aws-sdk/checksums': 3.1000.20 + '@aws-sdk/core': 3.977.1 + '@aws-sdk/credential-provider-node': 3.972.72 + '@aws-sdk/middleware-sdk-s3': 3.972.66 + '@aws-sdk/signature-v4-multi-region': 3.996.42 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/fetch-http-handler': 5.6.10 + '@smithy/node-http-handler': 4.9.10 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/client-secrets-manager@3.1094.0': + dependencies: + '@aws-sdk/core': 3.976.0 + '@aws-sdk/credential-provider-node': 3.972.71 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/fetch-http-handler': 5.6.10 + '@smithy/node-http-handler': 4.9.10 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/client-ssm@3.1094.0': + dependencies: + '@aws-sdk/core': 3.976.0 + '@aws-sdk/credential-provider-node': 3.972.71 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/fetch-http-handler': 5.6.10 + '@smithy/node-http-handler': 4.9.10 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/core@3.976.0': + dependencies: + '@aws-sdk/types': 3.974.2 + '@aws-sdk/xml-builder': 3.972.36 + '@aws/lambda-invoke-store': 0.3.0 + '@smithy/core': 3.29.8 + '@smithy/signature-v4': 5.6.9 + '@smithy/types': 4.16.1 + bowser: 2.14.1 + tslib: 2.8.1 + + '@aws-sdk/core@3.977.1': + dependencies: + '@aws-sdk/types': 3.974.2 + '@aws-sdk/xml-builder': 3.972.37 + '@aws/lambda-invoke-store': 0.3.0 + '@smithy/core': 3.29.8 + '@smithy/signature-v4': 5.6.9 + '@smithy/types': 4.16.1 + bowser: 2.14.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-env@3.972.60': + dependencies: + '@aws-sdk/core': 3.976.0 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-env@3.972.61': + dependencies: + '@aws-sdk/core': 3.977.1 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-http@3.972.62': + dependencies: + '@aws-sdk/core': 3.976.0 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/fetch-http-handler': 5.6.10 + '@smithy/node-http-handler': 4.9.10 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-http@3.972.63': + dependencies: + '@aws-sdk/core': 3.977.1 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/fetch-http-handler': 5.6.10 + '@smithy/node-http-handler': 4.9.10 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-ini@3.973.5': + dependencies: + '@aws-sdk/core': 3.976.0 + '@aws-sdk/credential-provider-env': 3.972.60 + '@aws-sdk/credential-provider-http': 3.972.62 + '@aws-sdk/credential-provider-login': 3.972.67 + '@aws-sdk/credential-provider-process': 3.972.60 + '@aws-sdk/credential-provider-sso': 3.973.4 + '@aws-sdk/credential-provider-web-identity': 3.972.66 + '@aws-sdk/nested-clients': 3.997.34 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/credential-provider-imds': 4.4.13 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-ini@3.973.6': + dependencies: + '@aws-sdk/core': 3.977.1 + '@aws-sdk/credential-provider-env': 3.972.61 + '@aws-sdk/credential-provider-http': 3.972.63 + '@aws-sdk/credential-provider-login': 3.972.68 + '@aws-sdk/credential-provider-process': 3.972.61 + '@aws-sdk/credential-provider-sso': 3.973.5 + '@aws-sdk/credential-provider-web-identity': 3.972.67 + '@aws-sdk/nested-clients': 3.997.35 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/credential-provider-imds': 4.4.13 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-login@3.972.67': + dependencies: + '@aws-sdk/core': 3.976.0 + '@aws-sdk/nested-clients': 3.997.34 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-login@3.972.68': + dependencies: + '@aws-sdk/core': 3.977.1 + '@aws-sdk/nested-clients': 3.997.35 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-node@3.972.71': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.60 + '@aws-sdk/credential-provider-http': 3.972.62 + '@aws-sdk/credential-provider-ini': 3.973.5 + '@aws-sdk/credential-provider-process': 3.972.60 + '@aws-sdk/credential-provider-sso': 3.973.4 + '@aws-sdk/credential-provider-web-identity': 3.972.66 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/credential-provider-imds': 4.4.13 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-node@3.972.72': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.61 + '@aws-sdk/credential-provider-http': 3.972.63 + '@aws-sdk/credential-provider-ini': 3.973.6 + '@aws-sdk/credential-provider-process': 3.972.61 + '@aws-sdk/credential-provider-sso': 3.973.5 + '@aws-sdk/credential-provider-web-identity': 3.972.67 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/credential-provider-imds': 4.4.13 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-process@3.972.60': + dependencies: + '@aws-sdk/core': 3.976.0 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-process@3.972.61': + dependencies: + '@aws-sdk/core': 3.977.1 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.973.4': + dependencies: + '@aws-sdk/core': 3.976.0 + '@aws-sdk/nested-clients': 3.997.34 + '@aws-sdk/token-providers': 3.1092.0 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.973.5': + dependencies: + '@aws-sdk/core': 3.977.1 + '@aws-sdk/nested-clients': 3.997.35 + '@aws-sdk/token-providers': 3.1095.0 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-web-identity@3.972.66': + dependencies: + '@aws-sdk/core': 3.976.0 + '@aws-sdk/nested-clients': 3.997.34 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-web-identity@3.972.67': + dependencies: + '@aws-sdk/core': 3.977.1 + '@aws-sdk/nested-clients': 3.997.35 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/middleware-sdk-ec2@3.972.48': + dependencies: + '@aws-sdk/core': 3.976.0 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/signature-v4': 5.6.9 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/middleware-sdk-s3@3.972.66': + dependencies: + '@aws-sdk/core': 3.977.1 + '@aws-sdk/signature-v4-multi-region': 3.996.42 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.997.34': + dependencies: + '@aws-sdk/core': 3.976.0 + '@aws-sdk/signature-v4-multi-region': 3.996.41 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/fetch-http-handler': 5.6.10 + '@smithy/node-http-handler': 4.9.10 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.997.35': + dependencies: + '@aws-sdk/core': 3.977.1 + '@aws-sdk/signature-v4-multi-region': 3.996.42 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/fetch-http-handler': 5.6.10 + '@smithy/node-http-handler': 4.9.10 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/signature-v4-multi-region@3.996.41': + dependencies: + '@aws-sdk/types': 3.974.2 + '@smithy/signature-v4': 5.6.9 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/signature-v4-multi-region@3.996.42': + dependencies: + '@aws-sdk/types': 3.974.2 + '@smithy/signature-v4': 5.6.9 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.1092.0': + dependencies: + '@aws-sdk/core': 3.976.0 + '@aws-sdk/nested-clients': 3.997.34 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.1095.0': + dependencies: + '@aws-sdk/core': 3.977.1 + '@aws-sdk/nested-clients': 3.997.35 + '@aws-sdk/types': 3.974.2 + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/types@3.974.2': + dependencies: + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/xml-builder@3.972.36': + dependencies: + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/xml-builder@3.972.37': + dependencies: + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws/lambda-invoke-store@0.3.0': {} + + '@esbuild/aix-ppc64@0.25.12': + optional: true + + '@esbuild/aix-ppc64@0.28.1': + optional: true + + '@esbuild/android-arm64@0.25.12': + optional: true + + '@esbuild/android-arm64@0.28.1': + optional: true + + '@esbuild/android-arm@0.25.12': + optional: true + + '@esbuild/android-arm@0.28.1': + optional: true + + '@esbuild/android-x64@0.25.12': + optional: true + + '@esbuild/android-x64@0.28.1': + optional: true + + '@esbuild/darwin-arm64@0.25.12': + optional: true + + '@esbuild/darwin-arm64@0.28.1': + optional: true + + '@esbuild/darwin-x64@0.25.12': + optional: true + + '@esbuild/darwin-x64@0.28.1': + optional: true + + '@esbuild/freebsd-arm64@0.25.12': + optional: true + + '@esbuild/freebsd-arm64@0.28.1': + optional: true + + '@esbuild/freebsd-x64@0.25.12': + optional: true + + '@esbuild/freebsd-x64@0.28.1': + optional: true + + '@esbuild/linux-arm64@0.25.12': + optional: true + + '@esbuild/linux-arm64@0.28.1': + optional: true + + '@esbuild/linux-arm@0.25.12': + optional: true + + '@esbuild/linux-arm@0.28.1': + optional: true + + '@esbuild/linux-ia32@0.25.12': + optional: true + + '@esbuild/linux-ia32@0.28.1': + optional: true + + '@esbuild/linux-loong64@0.25.12': + optional: true + + '@esbuild/linux-loong64@0.28.1': + optional: true + + '@esbuild/linux-mips64el@0.25.12': + optional: true + + '@esbuild/linux-mips64el@0.28.1': + optional: true + + '@esbuild/linux-ppc64@0.25.12': + optional: true + + '@esbuild/linux-ppc64@0.28.1': + optional: true + + '@esbuild/linux-riscv64@0.25.12': + optional: true + + '@esbuild/linux-riscv64@0.28.1': + optional: true + + '@esbuild/linux-s390x@0.25.12': + optional: true + + '@esbuild/linux-s390x@0.28.1': + optional: true + + '@esbuild/linux-x64@0.25.12': + optional: true + + '@esbuild/linux-x64@0.28.1': + optional: true + + '@esbuild/netbsd-arm64@0.25.12': + optional: true + + '@esbuild/netbsd-arm64@0.28.1': + optional: true + + '@esbuild/netbsd-x64@0.25.12': + optional: true + + '@esbuild/netbsd-x64@0.28.1': + optional: true + + '@esbuild/openbsd-arm64@0.25.12': + optional: true + + '@esbuild/openbsd-arm64@0.28.1': + optional: true + + '@esbuild/openbsd-x64@0.25.12': + optional: true + + '@esbuild/openbsd-x64@0.28.1': + optional: true + + '@esbuild/openharmony-arm64@0.25.12': + optional: true + + '@esbuild/openharmony-arm64@0.28.1': + optional: true + + '@esbuild/sunos-x64@0.25.12': + optional: true + + '@esbuild/sunos-x64@0.28.1': + optional: true + + '@esbuild/win32-arm64@0.25.12': + optional: true + + '@esbuild/win32-arm64@0.28.1': + optional: true + + '@esbuild/win32-ia32@0.25.12': + optional: true + + '@esbuild/win32-ia32@0.28.1': + optional: true + + '@esbuild/win32-x64@0.25.12': + optional: true + + '@esbuild/win32-x64@0.28.1': + optional: true + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@rollup/rollup-android-arm-eabi@4.62.2': + optional: true + + '@rollup/rollup-android-arm64@4.62.2': + optional: true + + '@rollup/rollup-darwin-arm64@4.62.2': + optional: true + + '@rollup/rollup-darwin-x64@4.62.2': + optional: true + + '@rollup/rollup-freebsd-arm64@4.62.2': + optional: true + + '@rollup/rollup-freebsd-x64@4.62.2': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.62.2': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.62.2': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.62.2': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.62.2': + optional: true + + '@rollup/rollup-linux-x64-musl@4.62.2': + optional: true + + '@rollup/rollup-openbsd-x64@4.62.2': + optional: true + + '@rollup/rollup-openharmony-arm64@4.62.2': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.62.2': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.62.2': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.62.2': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.62.2': + optional: true + + '@smithy/core@3.29.8': + dependencies: + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@smithy/credential-provider-imds@4.4.13': + dependencies: + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@smithy/fetch-http-handler@5.6.10': + dependencies: + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@smithy/node-http-handler@4.9.10': + dependencies: + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@smithy/signature-v4@5.6.9': + dependencies: + '@smithy/core': 3.29.8 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@smithy/types@4.16.1': + dependencies: + tslib: 2.8.1 + + '@types/aws-lambda@8.10.162': {} + + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + + '@types/deep-eql@4.0.2': {} + + '@types/estree@1.0.9': {} + + '@types/node@22.20.1': + dependencies: + undici-types: 6.21.0 + + '@vitest/expect@3.2.7': + dependencies: + '@types/chai': 5.2.3 + '@vitest/spy': 3.2.7 + '@vitest/utils': 3.2.7 + chai: 5.3.3 + tinyrainbow: 2.0.0 + + '@vitest/mocker@3.2.7(vite@7.3.6(@types/node@22.20.1)(tsx@4.23.1))': + dependencies: + '@vitest/spy': 3.2.7 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.3.6(@types/node@22.20.1)(tsx@4.23.1) + + '@vitest/pretty-format@3.2.7': + dependencies: + tinyrainbow: 2.0.0 + + '@vitest/runner@3.2.7': + dependencies: + '@vitest/utils': 3.2.7 + pathe: 2.0.3 + strip-literal: 3.1.0 + + '@vitest/snapshot@3.2.7': + dependencies: + '@vitest/pretty-format': 3.2.7 + magic-string: 0.30.21 + pathe: 2.0.3 + + '@vitest/spy@3.2.7': + dependencies: + tinyspy: 4.0.4 + + '@vitest/utils@3.2.7': + dependencies: + '@vitest/pretty-format': 3.2.7 + loupe: 3.2.1 + tinyrainbow: 2.0.0 + + assertion-error@2.0.1: {} + + aws-cdk-lib@2.262.0(constructs@10.7.1): + dependencies: + '@aws-cdk/asset-awscli-v1': 2.2.282 + '@aws-cdk/asset-node-proxy-agent-v6': 2.1.2 + '@aws-cdk/cloud-assembly-schema': 54.14.0 + constructs: 10.7.1 + + aws-cdk@2.1133.0: {} + + bowser@2.14.1: {} + + cac@6.7.14: {} + + chai@5.3.3: + dependencies: + assertion-error: 2.0.1 + check-error: 2.1.3 + deep-eql: 5.0.2 + loupe: 3.2.1 + pathval: 2.0.1 + + check-error@2.1.3: {} + + constructs@10.7.1: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + deep-eql@5.0.2: {} + + es-module-lexer@1.7.0: {} + + esbuild@0.25.12: + optionalDependencies: + '@esbuild/aix-ppc64': 0.25.12 + '@esbuild/android-arm': 0.25.12 + '@esbuild/android-arm64': 0.25.12 + '@esbuild/android-x64': 0.25.12 + '@esbuild/darwin-arm64': 0.25.12 + '@esbuild/darwin-x64': 0.25.12 + '@esbuild/freebsd-arm64': 0.25.12 + '@esbuild/freebsd-x64': 0.25.12 + '@esbuild/linux-arm': 0.25.12 + '@esbuild/linux-arm64': 0.25.12 + '@esbuild/linux-ia32': 0.25.12 + '@esbuild/linux-loong64': 0.25.12 + '@esbuild/linux-mips64el': 0.25.12 + '@esbuild/linux-ppc64': 0.25.12 + '@esbuild/linux-riscv64': 0.25.12 + '@esbuild/linux-s390x': 0.25.12 + '@esbuild/linux-x64': 0.25.12 + '@esbuild/netbsd-arm64': 0.25.12 + '@esbuild/netbsd-x64': 0.25.12 + '@esbuild/openbsd-arm64': 0.25.12 + '@esbuild/openbsd-x64': 0.25.12 + '@esbuild/openharmony-arm64': 0.25.12 + '@esbuild/sunos-x64': 0.25.12 + '@esbuild/win32-arm64': 0.25.12 + '@esbuild/win32-ia32': 0.25.12 + '@esbuild/win32-x64': 0.25.12 + + esbuild@0.28.1: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.1 + '@esbuild/android-arm': 0.28.1 + '@esbuild/android-arm64': 0.28.1 + '@esbuild/android-x64': 0.28.1 + '@esbuild/darwin-arm64': 0.28.1 + '@esbuild/darwin-x64': 0.28.1 + '@esbuild/freebsd-arm64': 0.28.1 + '@esbuild/freebsd-x64': 0.28.1 + '@esbuild/linux-arm': 0.28.1 + '@esbuild/linux-arm64': 0.28.1 + '@esbuild/linux-ia32': 0.28.1 + '@esbuild/linux-loong64': 0.28.1 + '@esbuild/linux-mips64el': 0.28.1 + '@esbuild/linux-ppc64': 0.28.1 + '@esbuild/linux-riscv64': 0.28.1 + '@esbuild/linux-s390x': 0.28.1 + '@esbuild/linux-x64': 0.28.1 + '@esbuild/netbsd-arm64': 0.28.1 + '@esbuild/netbsd-x64': 0.28.1 + '@esbuild/openbsd-arm64': 0.28.1 + '@esbuild/openbsd-x64': 0.28.1 + '@esbuild/openharmony-arm64': 0.28.1 + '@esbuild/sunos-x64': 0.28.1 + '@esbuild/win32-arm64': 0.28.1 + '@esbuild/win32-ia32': 0.28.1 + '@esbuild/win32-x64': 0.28.1 + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + expect-type@1.4.0: {} + + fdir@6.5.0(picomatch@4.0.5): + optionalDependencies: + picomatch: 4.0.5 + + fsevents@2.3.3: + optional: true + + js-tokens@9.0.1: {} + + loupe@3.2.1: {} + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + + ms@2.1.3: {} + + nanoid@3.3.16: {} + + pathe@2.0.3: {} + + pathval@2.0.1: {} + + picocolors@1.1.1: {} + + picomatch@4.0.5: {} + + postcss@8.5.22: + dependencies: + nanoid: 3.3.16 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + rollup@4.62.2: + dependencies: + '@types/estree': 1.0.9 + optionalDependencies: + '@rollup/rollup-android-arm-eabi': 4.62.2 + '@rollup/rollup-android-arm64': 4.62.2 + '@rollup/rollup-darwin-arm64': 4.62.2 + '@rollup/rollup-darwin-x64': 4.62.2 + '@rollup/rollup-freebsd-arm64': 4.62.2 + '@rollup/rollup-freebsd-x64': 4.62.2 + '@rollup/rollup-linux-arm-gnueabihf': 4.62.2 + '@rollup/rollup-linux-arm-musleabihf': 4.62.2 + '@rollup/rollup-linux-arm64-gnu': 4.62.2 + '@rollup/rollup-linux-arm64-musl': 4.62.2 + '@rollup/rollup-linux-loong64-gnu': 4.62.2 + '@rollup/rollup-linux-loong64-musl': 4.62.2 + '@rollup/rollup-linux-ppc64-gnu': 4.62.2 + '@rollup/rollup-linux-ppc64-musl': 4.62.2 + '@rollup/rollup-linux-riscv64-gnu': 4.62.2 + '@rollup/rollup-linux-riscv64-musl': 4.62.2 + '@rollup/rollup-linux-s390x-gnu': 4.62.2 + '@rollup/rollup-linux-x64-gnu': 4.62.2 + '@rollup/rollup-linux-x64-musl': 4.62.2 + '@rollup/rollup-openbsd-x64': 4.62.2 + '@rollup/rollup-openharmony-arm64': 4.62.2 + '@rollup/rollup-win32-arm64-msvc': 4.62.2 + '@rollup/rollup-win32-ia32-msvc': 4.62.2 + '@rollup/rollup-win32-x64-gnu': 4.62.2 + '@rollup/rollup-win32-x64-msvc': 4.62.2 + fsevents: 2.3.3 + + siginfo@2.0.0: {} + + source-map-js@1.2.1: {} + + stackback@0.0.2: {} + + std-env@3.10.0: {} + + strip-literal@3.1.0: + dependencies: + js-tokens: 9.0.1 + + tinybench@2.9.0: {} + + tinyexec@0.3.2: {} + + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.5) + picomatch: 4.0.5 + + tinypool@1.1.1: {} + + tinyrainbow@2.0.0: {} + + tinyspy@4.0.4: {} + + tslib@2.8.1: {} + + tsx@4.23.1: + dependencies: + esbuild: 0.28.1 + optionalDependencies: + fsevents: 2.3.3 + + typescript@5.9.3: {} + + undici-types@6.21.0: {} + + vite-node@3.2.4(@types/node@22.20.1)(tsx@4.23.1): + dependencies: + cac: 6.7.14 + debug: 4.4.3 + es-module-lexer: 1.7.0 + pathe: 2.0.3 + vite: 7.3.6(@types/node@22.20.1)(tsx@4.23.1) + transitivePeerDependencies: + - '@types/node' + - jiti + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + + vite@7.3.6(@types/node@22.20.1)(tsx@4.23.1): + dependencies: + esbuild: 0.28.1 + fdir: 6.5.0(picomatch@4.0.5) + picomatch: 4.0.5 + postcss: 8.5.22 + rollup: 4.62.2 + tinyglobby: 0.2.17 + optionalDependencies: + '@types/node': 22.20.1 + fsevents: 2.3.3 + tsx: 4.23.1 + + vitest@3.2.7(@types/node@22.20.1)(tsx@4.23.1): + dependencies: + '@types/chai': 5.2.3 + '@vitest/expect': 3.2.7 + '@vitest/mocker': 3.2.7(vite@7.3.6(@types/node@22.20.1)(tsx@4.23.1)) + '@vitest/pretty-format': 3.2.7 + '@vitest/runner': 3.2.7 + '@vitest/snapshot': 3.2.7 + '@vitest/spy': 3.2.7 + '@vitest/utils': 3.2.7 + chai: 5.3.3 + debug: 4.4.3 + expect-type: 1.4.0 + magic-string: 0.30.21 + pathe: 2.0.3 + picomatch: 4.0.5 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.17 + tinypool: 1.1.1 + tinyrainbow: 2.0.0 + vite: 7.3.6(@types/node@22.20.1)(tsx@4.23.1) + vite-node: 3.2.4(@types/node@22.20.1)(tsx@4.23.1) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 22.20.1 + transitivePeerDependencies: + - jiti + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 diff --git a/remote/preset.ini b/remote/preset.ini new file mode 100644 index 00000000..e3c6f220 --- /dev/null +++ b/remote/preset.ini @@ -0,0 +1,29 @@ +# llama.cpp preset for the endpoint this repo deploys — the section name matches +# the Outfit's ALIAS, which is what selects it. +# +# `outfit remote deploy` turns this into the deploy config: `hf` names the +# weights (repo:QUANT) and the rest become the server's flags. Settings the +# cloud owns are dropped from it — host, port, api-key, ctx-size (CONTEXT), the +# alias, and metrics — so this same file also runs the model locally with +# `outfit serve`, where those values do apply. +# See https://github.com/ggml-org/llama.cpp/blob/master/docs/preset.md + +[*] +host = 127.0.0.1 # local serve only; the cloud binds 0.0.0.0 +port = 8080 # local serve only +jinja = 1 # required for Qwen3.6 tool calling +ngl = 99 # offload every layer to the GPU + +[qwen3.6-27b] +# Unsloth's MTP build: the multi-token-prediction draft is inside this GGUF, so +# no separate draft file is downloaded. +hf = unsloth/Qwen3.6-27B-MTP-GGUF:UD-Q6_K_XL +ctx-size = 131072 # keep in step with the Outfit's CONTEXT +fa = on # flash attention, needed for the quantised KV cache +cache-type-k = q8_0 # a q8 KV cache keeps 128k of context inside 48 GB +cache-type-v = q8_0 +np = 1 # MTP does not support parallel sequences +# Multi-token prediction: the draft proposes tokens the model verifies in one +# pass. Worth ~1.5-2x on decode; llama-server logs the acceptance rate. +spec-type = draft-mtp +spec-draft-n-max = 2 diff --git a/remote/scripts/bake b/remote/scripts/bake new file mode 100755 index 00000000..771acf24 --- /dev/null +++ b/remote/scripts/bake @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# +# Triggers an EC2 Image Builder bake of a runner's AMI. Usage: +# +# aws-vault exec outofcoffee -- pnpm bake +# +# The image stack (pnpm deploy:image) only defines the pipelines; this starts a +# build. Builds are asynchronous and take ~20-40 min — this returns as soon as +# the build is queued, printing how to check on it. A failed build leaves the +# stack and the previous AMI untouched. + +set -euo pipefail + +# Not AWS_REGION — the aws-vault profile may default it to a different region +# than the stacks are deployed to. Override with CLOUD_VM_LLM_REGION if needed. +readonly REGION="${CLOUD_VM_LLM_REGION:-us-east-1}" + +readonly RUNNER="${1:-}" +case "${RUNNER}" in + vllm | llamacpp) ;; + *) + echo "usage: pnpm bake " >&2 + exit 2 + ;; +esac + +readonly PIPELINE_NAME="cloud-vm-llm-image-${RUNNER}-pipeline" + +pipeline_arn="$(aws imagebuilder list-image-pipelines --region "${REGION}" \ + --query "imagePipelineList[?name=='${PIPELINE_NAME}'].arn | [0]" --output text)" + +if [[ -z "${pipeline_arn}" || "${pipeline_arn}" == "None" ]]; then + echo "Error: pipeline '${PIPELINE_NAME}' not found in ${REGION}." >&2 + echo "Run 'pnpm deploy:image' first (or set CLOUD_VM_LLM_REGION)." >&2 + exit 1 +fi + +build_arn="$(aws imagebuilder start-image-pipeline-execution --region "${REGION}" \ + --image-pipeline-arn "${pipeline_arn}" --query imageBuildVersionArn --output text)" + +echo "Bake started (${RUNNER}): ${build_arn}" +echo +echo "Check progress (BUILDING -> ... -> AVAILABLE, ~20-40 min):" +echo " aws imagebuilder get-image --region ${REGION} \\" +echo " --image-build-version-arn ${build_arn} --query 'image.state' --output json" +echo +echo "The runtime start Lambda picks up the new AMI automatically once it is" +echo "AVAILABLE — no runtime redeploy needed." diff --git a/remote/scripts/console b/remote/scripts/console new file mode 100755 index 00000000..56b205a4 --- /dev/null +++ b/remote/scripts/console @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# +# Opens an interactive shell on the running cloud-vm-llm instance via SSM +# Session Manager — no SSH and no extra open port, over the same SSM channel +# the health and idle checks already use. Run it under your AWS creds: +# +# aws-vault exec outofcoffee -- pnpm console +# +# Pass an instance id to target one explicitly; otherwise it finds the running +# managed instance by tag. Once connected, handy commands: +# sudo journalctl -u vllm -f # tail the vLLM logs +# nvidia-smi # GPU state +# systemctl status vllm # service state + +set -euo pipefail + +# Not AWS_REGION — the aws-vault profile may default it elsewhere than the +# stacks are deployed to. Override with CLOUD_VM_LLM_REGION if needed. +readonly REGION="${CLOUD_VM_LLM_REGION:-us-east-1}" + +if ! command -v session-manager-plugin >/dev/null 2>&1; then + echo "Error: session-manager-plugin is not installed (required by 'aws ssm start-session')." >&2 + echo " macOS: brew install --cask session-manager-plugin" >&2 + echo " docs: https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html" >&2 + exit 1 +fi + +instance_id="${1:-}" +if [[ -z "${instance_id}" ]]; then + instance_id="$(aws ec2 describe-instances --region "${REGION}" \ + --filters Name=tag:cloud-vm-llm,Values=endpoint Name=instance-state-name,Values=running \ + --query "Reservations[0].Instances[0].InstanceId" --output text)" +fi + +if [[ -z "${instance_id}" || "${instance_id}" == "None" ]]; then + echo "Error: no running cloud-vm-llm instance found in ${REGION}." >&2 + echo "Start it first (e.g. 'outfit remote start'), then retry." >&2 + exit 1 +fi + +echo "Connecting to ${instance_id} (${REGION}) via SSM Session Manager..." >&2 +echo "Tip: 'sudo journalctl -u vllm -f' tails the vLLM logs; Ctrl-D to exit." >&2 +exec aws ssm start-session --region "${REGION}" --target "${instance_id}" diff --git a/remote/scripts/seed-deploy-config.mjs b/remote/scripts/seed-deploy-config.mjs new file mode 100644 index 00000000..584e851f --- /dev/null +++ b/remote/scripts/seed-deploy-config.mjs @@ -0,0 +1,94 @@ +#!/usr/bin/env node +// Seeds the deploy-config SSM parameter with CDK's initial config the FIRST +// time — while the parameter still holds the `unconfigured` placeholder CDK +// created it with. It never overwrites a real config: `outfit remote deploy` +// and manual edits own the parameter after the first seed, and a plain +// `cdk deploy` leaves it untouched (the parameter's CloudFormation value is a +// constant, so a deploy cannot clobber what is actually being served). +// +// Runs as the last step of `pnpm deploy`, reading the stack outputs that +// `cdk deploy --outputs-file` wrote. + +import { existsSync, readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { GetParameterCommand, PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; + +// Keep in step with UNCONFIGURED_DEPLOY_CONFIG in lambda/shared/deploy-config.ts. +const UNCONFIGURED = 'unconfigured'; + +const repoRoot = dirname(dirname(fileURLToPath(import.meta.url))); +const outputsPath = process.argv[2] ?? join(repoRoot, 'cdk-outputs.json'); + +function fail(message) { + console.error(`Error: ${message}`); + process.exit(1); +} + +if (!existsSync(outputsPath)) { + fail(`${outputsPath} not found — run \`pnpm deploy\` first (it writes the stack outputs).`); +} +const allOutputs = JSON.parse(readFileSync(outputsPath, 'utf8')); +const stack = allOutputs['cloud-vm-llm'] ?? Object.values(allOutputs)[0]; +if (!stack) { + fail(`no stack outputs found in ${outputsPath}`); +} + +const { DeployConfigParam, InitialDeployConfig, Region } = stack; +for (const [name, value] of Object.entries({ DeployConfigParam, InitialDeployConfig, Region })) { + if (!value) { + fail(`stack output ${name} is missing from ${outputsPath}`); + } +} + +function describe(raw) { + try { + const cfg = JSON.parse(raw); + return `runner=${cfg.runner} model=${cfg.modelId} ctx=${cfg.contextSize}`; + } catch { + return raw; + } +} + +// Local checks first, so the common "nothing to do" paths need no AWS call. +// +// CDK only knows a runner's full serve config when it has the serve args. +// llama.cpp's args come from an Outfit (via `outfit remote deploy`), so CDK's +// initial config for it is incomplete — seeding it would produce a config that +// fails obscurely at wake. Leave the parameter unconfigured (it fails loudly +// with a clear message) for outfit/manual to set. +const initial = JSON.parse(InitialDeployConfig); +if (!Array.isArray(initial.serveArgs) || initial.serveArgs.length === 0) { + console.log( + `deploy-config left unconfigured: CDK has no complete serve config for runner=${initial.runner} ` + + `(its serve args come from an Outfit). Set it with \`outfit remote deploy\` or a manual put.`, + ); + process.exit(0); +} + +const ssm = new SSMClient({ region: Region }); + +const current = await ssm + .send(new GetParameterCommand({ Name: DeployConfigParam })) + .then((r) => r.Parameter?.Value) + .catch((err) => { + if (err?.name === 'ParameterNotFound') { + return undefined; + } + throw err; + }); + +if (current && current.trim() !== UNCONFIGURED) { + console.log(`deploy-config already set (${describe(current)}) — leaving it untouched.`); + process.exit(0); +} + +await ssm.send( + new PutParameterCommand({ + Name: DeployConfigParam, + Type: 'String', + Overwrite: true, + Value: InitialDeployConfig, + }), +); +console.log(`Seeded deploy-config (${describe(InitialDeployConfig)}).`); diff --git a/remote/scripts/seed-model.mjs b/remote/scripts/seed-model.mjs new file mode 100644 index 00000000..d615b484 --- /dev/null +++ b/remote/scripts/seed-model.mjs @@ -0,0 +1,148 @@ +#!/usr/bin/env node +// Manual model seed: launches a disposable EC2 instance that downloads the +// model from Hugging Face and syncs it to the runtime's S3 bucket, then +// terminates itself. WHAT to seed (runner, model id, quant, S3 prefix) comes +// from the deploy-config SSM parameter — set it with `outfit remote deploy` +// (or the deploy Lambda) first. +// +// The deploy Lambda now seeds automatically when a posted config names weights +// that are not in S3 (see lambda/shared/seed.ts), so this script is only needed +// to force a re-seed of weights that are already there. Infra (bucket, seed profile/subnet/sg) comes +// from cdk-outputs.json (`pnpm deploy`). The seed always runs on the vLLM AMI +// because that one carries a Python venv with huggingface_hub; it can fetch +// either safetensors (vLLM) or a single GGUF file (llama.cpp). + +import { existsSync, readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { DescribeImagesCommand, EC2Client, RunInstancesCommand } from '@aws-sdk/client-ec2'; +import { GetParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; + +const AMI_ROLE_TAG_KEY = 'cloud-vm-llm:role'; +const AMI_ROLE_TAG_VALUE = 'runtime-ami'; +const AMI_RUNNER_TAG_KEY = 'cloud-vm-llm:runner'; +const DEPLOY_CONFIG_PARAM = '/cloud-vm-llm/deploy-config'; + +const repoRoot = dirname(dirname(fileURLToPath(import.meta.url))); + +function fail(message) { + console.error(`Error: ${message}`); + process.exit(1); +} + +const outputsPath = join(repoRoot, 'cdk-outputs.json'); +if (!existsSync(outputsPath)) { + fail('cdk-outputs.json not found — run `pnpm deploy` first.'); +} +const stack = JSON.parse(readFileSync(outputsPath, 'utf8'))['cloud-vm-llm']; +if (!stack) { + fail('no cloud-vm-llm outputs in cdk-outputs.json'); +} +const { + Region: region, + WeightsBucket: bucket, + SeedInstanceProfileArn: profileArn, + SeedInstanceType: instanceType, + SeedSubnetId: subnetId, + SeedSecurityGroupId: securityGroupId, + HfTokenSecretArn: hfSecretArn, +} = stack; + +// What to seed comes from the deploy-config parameter, not the stack outputs. +const ssm = new SSMClient({ region }); +const param = await ssm.send(new GetParameterCommand({ Name: DEPLOY_CONFIG_PARAM })); +let deploy; +try { + deploy = JSON.parse(param.Parameter?.Value ?? '{}'); +} catch { + fail('deploy-config is not valid JSON — set it via `outfit remote deploy` first.'); +} +const { runner, modelId, quant, weightsPrefix } = deploy; +if (runner !== 'vllm' && runner !== 'llamacpp') { + fail(`deploy-config.runner must be vllm or llamacpp (got ${JSON.stringify(runner)}).`); +} +if (!modelId || !weightsPrefix) { + fail('deploy-config is missing modelId/weightsPrefix.'); +} + +const ec2 = new EC2Client({ region }); + +// The seed runs on the vLLM AMI (it has the Python venv + huggingface_hub), +// regardless of which runner the weights are for. +async function newestAmi(filters) { + const r = await ec2.send(new DescribeImagesCommand({ Owners: ['self'], Filters: filters })); + return (r.Images ?? []).sort((a, b) => + (b.CreationDate ?? '').localeCompare(a.CreationDate ?? ''), + )[0]; +} +const roleFilter = { Name: `tag:${AMI_ROLE_TAG_KEY}`, Values: [AMI_ROLE_TAG_VALUE] }; +const stateFilter = { Name: 'state', Values: ['available'] }; +let image = await newestAmi([ + roleFilter, + { Name: `tag:${AMI_RUNNER_TAG_KEY}`, Values: ['vllm'] }, + stateFilter, +]); +if (!image) { + // Transition fallback: older vLLM AMIs predate the runner tag. The seed only + // needs huggingface_hub + the AWS CLI, which any runtime AMI carries. + image = await newestAmi([roleFilter, stateFilter]); +} +if (!image) { + fail('no baked runtime AMI found — run `pnpm bake vllm` and wait for it to finish.'); +} + +const header = `#!/bin/bash +set -euxo pipefail +HF_TOKEN="" +${hfSecretArn ? `HF_TOKEN=$(aws secretsmanager get-secret-value --secret-id '${hfSecretArn}' --region '${region}' --query SecretString --output text)` : ''} +export HF_TOKEN +export MODEL_ID='${modelId}' +mkdir -p /opt/llm/model +`; + +const download = + runner === 'llamacpp' + ? `export QUANT='${quant}' +mkdir -p /tmp/dl +/opt/llm/venv/bin/python -c "import os; from huggingface_hub import snapshot_download; snapshot_download(os.environ['MODEL_ID'], allow_patterns=['*'+os.environ['QUANT']+'*'], local_dir='/tmp/dl', token=(os.environ.get('HF_TOKEN') or None))" +# The main GGUF (MTP is embedded); normalise to model.gguf so the runtime need +# not guess the filename. mmproj/projector files are excluded. +mapfile -t GGUFS < <(find /tmp/dl -type f -name '*.gguf' ! -iname '*mmproj*' | sort) +test "\${#GGUFS[@]}" -ge 1 +cp "\${GGUFS[0]}" /opt/llm/model/model.gguf +[ "\${#GGUFS[@]}" -gt 1 ] && echo "WARNING: \${#GGUFS[@]} gguf files for $QUANT; used the first (split quant not handled)" >&2 || true +` + : `/opt/llm/venv/bin/python -c "import os; from huggingface_hub import snapshot_download; snapshot_download(os.environ['MODEL_ID'], local_dir='/opt/llm/model', token=(os.environ.get('HF_TOKEN') or None))" +`; + +const userData = `${header}${download}aws s3 sync /opt/llm/model/ 's3://${bucket}/${weightsPrefix}' --region '${region}' +shutdown -h now +`; + +const run = await ec2.send( + new RunInstancesCommand({ + ImageId: image.ImageId, + InstanceType: instanceType, + MinCount: 1, + MaxCount: 1, + SubnetId: subnetId, + SecurityGroupIds: [securityGroupId], + IamInstanceProfile: { Arn: profileArn }, + UserData: Buffer.from(userData).toString('base64'), + MetadataOptions: { HttpTokens: 'required' }, + // Self-terminate when the userdata runs `shutdown -h now`. + InstanceInitiatedShutdownBehavior: 'terminate', + TagSpecifications: [ + { ResourceType: 'instance', Tags: [{ Key: 'Name', Value: 'cloud-vm-llm-seed' }] }, + ], + }), +); +const instanceId = run.Instances?.[0]?.InstanceId; + +console.log(`Seeding ${runner} model ${modelId}${quant ? `:${quant}` : ''} -> s3://${bucket}/${weightsPrefix}`); +console.log(`Seed instance: ${instanceId} (${instanceType}, from ${image.ImageId})`); +console.log('It downloads from Hugging Face, syncs to S3, then terminates itself (~15-20 min).'); +console.log('\nWatch it:'); +console.log(` aws ssm start-session --target ${instanceId} # then: sudo tail -f /var/log/cloud-init-output.log`); +console.log('Done when it appears in S3:'); +console.log(` aws s3 ls s3://${bucket}/${weightsPrefix} --region ${region}`); diff --git a/remote/scripts/set-ip b/remote/scripts/set-ip new file mode 100755 index 00000000..231f7ba3 --- /dev/null +++ b/remote/scripts/set-ip @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Writes the current machine's public IP to the repo's .env file as +# ALLOWED_CIDR=/32. The CDK app reads this when the allowedCidr context +# value is not passed on the command line. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +readonly REPO_ROOT +readonly ENV_FILE="${REPO_ROOT}/.env" + +####################################### +# Fetch the machine's public IPv4 address. +# Outputs: +# Writes the address to stdout. +# Returns: +# 0 on success, 1 when no service answered with an IPv4 address. +####################################### +fetch_ip() { + local service ip + for service in 'https://checkip.amazonaws.com' 'https://ifconfig.me'; do + ip="$(curl -fsS --max-time 10 "${service}" 2>/dev/null || true)" + ip="${ip//[[:space:]]/}" + if [[ "${ip}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then + echo "${ip}" + return 0 + fi + done + return 1 +} + +main() { + local ip + if ! ip="$(fetch_ip)"; then + echo "Error: could not determine the public IP address" >&2 + exit 1 + fi + + local cidr="${ip}/32" + if [[ -f "${ENV_FILE}" ]] && grep -q '^ALLOWED_CIDR=' "${ENV_FILE}"; then + local tmp + tmp="$(mktemp)" + sed "s|^ALLOWED_CIDR=.*|ALLOWED_CIDR=${cidr}|" "${ENV_FILE}" > "${tmp}" + mv "${tmp}" "${ENV_FILE}" + else + echo "ALLOWED_CIDR=${cidr}" >> "${ENV_FILE}" + fi + echo "Wrote ALLOWED_CIDR=${cidr} to ${ENV_FILE}" +} + +main "$@" diff --git a/remote/scripts/write-config.mjs b/remote/scripts/write-config.mjs new file mode 100644 index 00000000..39b8e64a --- /dev/null +++ b/remote/scripts/write-config.mjs @@ -0,0 +1,47 @@ +#!/usr/bin/env node +// Generates the concrete, gitignored `Outfit` and `remote.json` from the CDK +// stack outputs that `pnpm deploy` writes (cdk deploy --outputs-file). +// +// The endpoint's base URL is a stable Elastic IP known at deploy time, so it +// lives in the Outfit's BASEURL as static config rather than being fetched +// from a runtime Lambda response. remote.json still carries the start/stop +// Lambda URLs that `outfit remote` drives. + +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const repoRoot = dirname(dirname(fileURLToPath(import.meta.url))); +const outputsPath = process.argv[2] ?? join(repoRoot, 'cdk-outputs.json'); + +function fail(message) { + console.error(`Error: ${message}`); + process.exit(1); +} + +if (!existsSync(outputsPath)) { + fail(`${outputsPath} not found — run \`pnpm deploy\` first (it writes the stack outputs).`); +} + +const allOutputs = JSON.parse(readFileSync(outputsPath, 'utf8')); +const stack = allOutputs['cloud-vm-llm'] ?? Object.values(allOutputs)[0]; +if (!stack) { + fail(`no stack outputs found in ${outputsPath}`); +} + +const { BaseUrl, ModelId, OutfitRemoteConfig } = stack; +for (const [name, value] of Object.entries({ BaseUrl, ModelId, OutfitRemoteConfig })) { + if (!value) { + fail(`stack output ${name} is missing from ${outputsPath}`); + } +} + +const template = readFileSync(join(repoRoot, 'Outfit.example'), 'utf8'); +const outfit = template.replaceAll('__BASEURL__', BaseUrl).replaceAll('__MODEL__', ModelId); +writeFileSync(join(repoRoot, 'Outfit'), outfit); + +// OutfitRemoteConfig is already a JSON string; re-emit it pretty-printed. +const remote = `${JSON.stringify(JSON.parse(OutfitRemoteConfig), null, 2)}\n`; +writeFileSync(join(repoRoot, 'remote.json'), remote); + +console.log(`Wrote Outfit (BASEURL ${BaseUrl}) and remote.json`); diff --git a/remote/test/deploy-config.test.ts b/remote/test/deploy-config.test.ts new file mode 100644 index 00000000..b42ecadf --- /dev/null +++ b/remote/test/deploy-config.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from 'vitest'; +import { + buildServeCommand, + parseDeployConfig, + UNCONFIGURED_DEPLOY_CONFIG, + weightsPrefixFor, + type DeployConfig, +} from '../lambda/shared/deploy-config'; + +const VLLM: DeployConfig = { + runner: 'vllm', + modelId: 'Qwen/Qwen3.6-27B-FP8', + quant: '', + weightsPrefix: 'models/vllm/Qwen/Qwen3.6-27B-FP8/', + contextSize: 32768, + servedModelName: 'Qwen/Qwen3.6-27B-FP8', + serveArgs: ['--enforce-eager', '--tool-call-parser', 'qwen3_coder'], +}; + +const LLAMACPP: DeployConfig = { + runner: 'llamacpp', + modelId: 'unsloth/Qwen3.6-27B-MTP-GGUF', + quant: 'UD-Q6_K_XL', + weightsPrefix: 'models/llamacpp/unsloth/Qwen3.6-27B-MTP-GGUF/UD-Q6_K_XL/', + contextSize: 131072, + servedModelName: 'qwen3.6-27b', + serveArgs: ['-ngl', '99', '-fa', 'on', '--spec-type', 'mtp', '--jinja'], +}; + +describe('weightsPrefixFor', () => { + it('includes the runner, and the quant only when there is one', () => { + expect(weightsPrefixFor('llamacpp', 'unsloth/Qwen3.6-27B-MTP-GGUF', 'UD-Q6_K_XL')).toBe( + 'models/llamacpp/unsloth/Qwen3.6-27B-MTP-GGUF/UD-Q6_K_XL/', + ); + expect(weightsPrefixFor('vllm', 'Qwen/Qwen3.6-27B-FP8', '')).toBe( + 'models/vllm/Qwen/Qwen3.6-27B-FP8/', + ); + }); + + it('keeps the two runners apart for the same model id', () => { + expect(weightsPrefixFor('vllm', 'org/m', '')).not.toBe(weightsPrefixFor('llamacpp', 'org/m', '')); + }); +}); + +describe('parseDeployConfig', () => { + it('derives weightsPrefix and ignores any sent on the wire', () => { + const cfg = parseDeployConfig( + JSON.stringify({ ...LLAMACPP, weightsPrefix: 'models/attacker-controlled/' }), + ); + expect(cfg.weightsPrefix).toBe('models/llamacpp/unsloth/Qwen3.6-27B-MTP-GGUF/UD-Q6_K_XL/'); + }); + + it('derives weightsPrefix when none is sent at all', () => { + const { weightsPrefix, ...withoutPrefix } = LLAMACPP; + expect(parseDeployConfig(JSON.stringify(withoutPrefix)).weightsPrefix).toBe(weightsPrefix); + }); + + it('round-trips a vllm config', () => { + expect(parseDeployConfig(JSON.stringify(VLLM))).toEqual(VLLM); + }); + + it('round-trips a llamacpp config', () => { + expect(parseDeployConfig(JSON.stringify(LLAMACPP))).toEqual(LLAMACPP); + }); + + it('defaults quant and serveArgs when omitted', () => { + const cfg = parseDeployConfig( + JSON.stringify({ ...VLLM, quant: undefined, serveArgs: undefined }), + ); + expect(cfg.quant).toBe(''); + expect(cfg.serveArgs).toEqual([]); + }); + + it('rejects an empty or unconfigured config', () => { + expect(() => parseDeployConfig('')).toThrow(/not set/); + expect(() => parseDeployConfig(undefined)).toThrow(/not set/); + // The placeholder CDK creates the parameter with must fail loudly, so a + // wake before the config is seeded is a clear error, not a silent default. + expect(() => parseDeployConfig(UNCONFIGURED_DEPLOY_CONFIG)).toThrow(/not set/); + }); + + it('rejects malformed JSON', () => { + expect(() => parseDeployConfig('{not json')).toThrow(/not valid JSON/); + }); + + it('rejects a missing or unknown runner (no default)', () => { + expect(() => parseDeployConfig(JSON.stringify({ ...VLLM, runner: undefined }))).toThrow(/runner/); + expect(() => parseDeployConfig(JSON.stringify({ ...VLLM, runner: 'tgi' }))).toThrow(/runner/); + }); + + it('rejects a non-positive context size', () => { + expect(() => parseDeployConfig(JSON.stringify({ ...VLLM, contextSize: 0 }))).toThrow(/contextSize/); + }); + + it('rejects a missing modelId', () => { + expect(() => parseDeployConfig(JSON.stringify({ ...VLLM, modelId: '' }))).toThrow(/modelId/); + }); + + it('rejects non-string serveArgs', () => { + expect(() => parseDeployConfig(JSON.stringify({ ...VLLM, serveArgs: [1, 2] }))).toThrow(/serveArgs/); + }); +}); + +describe('buildServeCommand', () => { + const opts = { modelDir: '/opt/llm/model', port: 8000 }; + + it('builds the vllm serve command with its args', () => { + const cmd = buildServeCommand(VLLM, opts); + expect(cmd).toContain('/opt/llm/venv/bin/vllm serve /opt/llm/model'); + expect(cmd).toContain("--served-model-name 'Qwen/Qwen3.6-27B-FP8'"); + expect(cmd).toContain('--max-model-len 32768'); + expect(cmd).toContain('--gpu-memory-utilization 0.92'); + expect(cmd).toContain('--enforce-eager --tool-call-parser qwen3_coder'); + expect(cmd).not.toContain('llama-server'); + }); + + it('builds the llama-server command with the gguf and its args', () => { + const cmd = buildServeCommand(LLAMACPP, opts); + expect(cmd).toContain('llama-server --model /opt/llm/model/model.gguf'); + expect(cmd).toContain('--ctx-size 131072'); + expect(cmd).toContain('--api-key-file /etc/llm/api-key'); + expect(cmd).toContain("--alias 'qwen3.6-27b'"); + // --metrics is forced on so the idle scrape can read /metrics. + expect(cmd).toContain('--metrics'); + expect(cmd).toContain('-ngl 99 -fa on --spec-type mtp --jinja'); + expect(cmd).not.toContain('vllm serve'); + }); +}); diff --git a/remote/test/idle.test.ts b/remote/test/idle.test.ts new file mode 100644 index 00000000..584a4a70 --- /dev/null +++ b/remote/test/idle.test.ts @@ -0,0 +1,231 @@ +import { describe, expect, it } from 'vitest'; +import { + decideIdle, + metricsGrepPattern, + parseMetrics, + type IdleDecisionInput, +} from '../lambda/shared/idle'; + +const SCRAPE = `vllm:num_requests_running{model_name="Qwen/Qwen3.6-27B-FP8"} 0.0 +vllm:num_requests_waiting{model_name="Qwen/Qwen3.6-27B-FP8"} 0.0 +vllm:prompt_tokens_total{model_name="Qwen/Qwen3.6-27B-FP8"} 12345.0 +vllm:generation_tokens_total{model_name="Qwen/Qwen3.6-27B-FP8"} 6789.0 +vllm:request_success_total{model_name="Qwen/Qwen3.6-27B-FP8",finished_reason="stop"} 42.0`; + +// Real names as observed on llama-server /metrics for Qwen3.6-27B. +const LLAMACPP_SCRAPE = `llamacpp:requests_processing 0 +llamacpp:requests_deferred 0 +llamacpp:prompt_tokens_total 5000 +llamacpp:tokens_predicted_total 2500 +llamacpp:n_decode_total 2500 +llamacpp:n_busy_slots_per_decode 1`; + +describe('parseMetrics', () => { + it('sums running and counter metrics from a vllm scrape', () => { + const result = parseMetrics(SCRAPE, 'vllm'); + expect(result).toEqual({ ok: true, running: 0, counter: 12345 + 6789 + 42 }); + }); + + it('reports in-flight vllm requests', () => { + const result = parseMetrics( + 'vllm:num_requests_running{model_name="m"} 2.0\nvllm:num_requests_waiting{model_name="m"} 1.0', + 'vllm', + ); + expect(result).toEqual({ ok: true, running: 3, counter: 0 }); + }); + + it('sums running and counter metrics from a llamacpp scrape', () => { + const result = parseMetrics(LLAMACPP_SCRAPE, 'llamacpp'); + // Ignores n_busy_slots_per_decode (not an activity metric). + expect(result).toEqual({ ok: true, running: 0, counter: 5000 + 2500 + 2500 }); + }); + + it('reports in-flight llamacpp requests', () => { + const result = parseMetrics( + 'llamacpp:requests_processing 1\nllamacpp:requests_deferred 2', + 'llamacpp', + ); + expect(result).toEqual({ ok: true, running: 3, counter: 0 }); + }); + + it('does not match the other runner’s metrics', () => { + expect(parseMetrics(SCRAPE, 'llamacpp')).toEqual({ ok: false }); + expect(parseMetrics(LLAMACPP_SCRAPE, 'vllm')).toEqual({ ok: false }); + }); + + it('fails on SCRAPE_FAILED marker', () => { + expect(parseMetrics('SCRAPE_FAILED', 'vllm')).toEqual({ ok: false }); + expect(parseMetrics('SCRAPE_FAILED', 'llamacpp')).toEqual({ ok: false }); + }); + + it('fails on empty output', () => { + expect(parseMetrics('', 'vllm')).toEqual({ ok: false }); + }); + + it('fails when no recognisable metrics are present', () => { + expect(parseMetrics('python_gc_objects_collected_total{generation="0"} 6.0', 'vllm')).toEqual({ + ok: false, + }); + }); +}); + +describe('metricsGrepPattern', () => { + it('scopes the grep to the runner prefix and its metric names', () => { + expect(metricsGrepPattern('vllm')).toBe( + '^vllm:(num_requests_running|num_requests_waiting|prompt_tokens_total|generation_tokens_total|request_success_total)', + ); + expect(metricsGrepPattern('llamacpp')).toBe( + '^llamacpp:(requests_processing|requests_deferred|prompt_tokens_total|tokens_predicted_total|n_decode_total)', + ); + }); +}); + +function input(overrides: Partial): IdleDecisionInput { + return { + now: new Date('2026-07-24T12:00:00Z'), + launchTime: new Date('2026-07-24T11:00:00Z'), + metrics: { ok: true, running: 0, counter: 100 }, + state: { counter: 100, last_change_at: '2026-07-24T11:30:00Z' }, + idleThresholdMinutes: 15, + gracePeriodMinutes: 10, + maxRuntimeMinutes: 240, + ...overrides, + }; +} + +describe('decideIdle', () => { + it('waits during the launch grace period', () => { + const decision = decideIdle(input({ launchTime: new Date('2026-07-24T11:55:00Z') })); + expect(decision.action).toBe('wait'); + expect(decision.reason).toContain('grace'); + }); + + it('records activity when requests are in flight', () => { + const decision = decideIdle(input({ metrics: { ok: true, running: 2, counter: 100 } })); + expect(decision.action).toBe('update'); + }); + + it('records activity when the counter moved', () => { + const decision = decideIdle(input({ metrics: { ok: true, running: 0, counter: 150 } })); + expect(decision).toMatchObject({ + action: 'update', + newState: { counter: 150, last_change_at: '2026-07-24T12:00:00.000Z' }, + }); + }); + + it('treats a counter reset (container restart) as activity', () => { + const decision = decideIdle(input({ metrics: { ok: true, running: 0, counter: 5 } })); + expect(decision.action).toBe('update'); + }); + + it('initialises state on first observation', () => { + const decision = decideIdle(input({ state: {} })); + expect(decision.action).toBe('update'); + }); + + it('preserves last_wake_at when updating', () => { + const decision = decideIdle( + input({ + metrics: { ok: true, running: 0, counter: 150 }, + state: { counter: 100, last_wake_at: '2026-07-24T11:45:00Z' }, + }), + ); + expect(decision).toMatchObject({ + action: 'update', + newState: { last_wake_at: '2026-07-24T11:45:00Z' }, + }); + }); + + it('stops once idle beyond the threshold', () => { + const decision = decideIdle(input({})); + expect(decision.action).toBe('stop'); // last change 30 min ago > 15 min threshold + }); + + it('waits when idle but within the threshold', () => { + const decision = decideIdle(input({ state: { counter: 100, last_change_at: '2026-07-24T11:50:00Z' } })); + expect(decision.action).toBe('wait'); + }); + + it('a recent wake blocks the stop even with a stale counter', () => { + const decision = decideIdle( + input({ + state: { + counter: 100, + last_change_at: '2026-07-24T11:30:00Z', + last_wake_at: '2026-07-24T11:55:00Z', + }, + }), + ); + expect(decision.action).toBe('wait'); + }); + + it('stops at the threshold when the scrape failed (crashed container)', () => { + const decision = decideIdle(input({ metrics: { ok: false } })); + expect(decision.action).toBe('stop'); + expect(decision.reason).toContain('scrape failed'); + }); + + it('anchors on launch time when there is no recorded state', () => { + const decision = decideIdle(input({ metrics: { ok: false }, state: {} })); + expect(decision.action).toBe('stop'); // launched 60 min ago, no signs of life + }); + + it('stops at the maximum runtime even with requests in flight', () => { + const decision = decideIdle( + input({ + launchTime: new Date('2026-07-24T07:00:00Z'), // 5 h ago > 4 h cap + metrics: { ok: true, running: 3, counter: 100 }, + }), + ); + expect(decision.action).toBe('stop'); + expect(decision.reason).toContain('maximum runtime'); + }); + + it('stops at the maximum runtime even when the counter is moving', () => { + const decision = decideIdle( + input({ + launchTime: new Date('2026-07-24T07:00:00Z'), + metrics: { ok: true, running: 0, counter: 9999 }, + }), + ); + expect(decision.action).toBe('stop'); + expect(decision.reason).toContain('maximum runtime'); + }); + + it('does not apply the maximum runtime before it is reached', () => { + const decision = decideIdle( + input({ + launchTime: new Date('2026-07-24T09:00:00Z'), // 3 h ago < 4 h cap + metrics: { ok: true, running: 1, counter: 100 }, + }), + ); + expect(decision.action).toBe('update'); // active, so no stop + }); + + it('a future Retain-Until blocks an otherwise-certain idle stop', () => { + const decision = decideIdle( + input({ metrics: { ok: false }, state: {}, retainUntil: new Date('2026-07-24T13:00:00Z') }), + ); + expect(decision.action).toBe('wait'); + expect(decision.reason).toContain('retained until'); + }); + + it('a future Retain-Until overrides even the maximum runtime cap', () => { + const decision = decideIdle( + input({ + launchTime: new Date('2026-07-24T07:00:00Z'), // 5 h ago > 4 h cap + metrics: { ok: true, running: 3, counter: 100 }, + retainUntil: new Date('2026-07-24T13:00:00Z'), + }), + ); + expect(decision.action).toBe('wait'); + expect(decision.reason).toContain('retained until'); + }); + + it('a past Retain-Until does not block a stop', () => { + const decision = decideIdle( + input({ metrics: { ok: false }, state: {}, retainUntil: new Date('2026-07-24T11:59:00Z') }), + ); + expect(decision.action).toBe('stop'); + }); +}); diff --git a/remote/test/seed.test.ts b/remote/test/seed.test.ts new file mode 100644 index 00000000..3d7ec7f0 --- /dev/null +++ b/remote/test/seed.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from 'vitest'; +import type { DeployConfig } from '../lambda/shared/deploy-config'; +import { buildSeedUserData, type SeedEnv } from '../lambda/shared/seed'; + +const ENV: SeedEnv = { + region: 'us-east-1', + bucket: 'weights-bucket', + instanceType: 'm5.xlarge', + subnetId: 'subnet-1', + securityGroupId: 'sg-1', + instanceProfileArn: 'arn:aws:iam::1:instance-profile/seed', + hfSecretArn: '', + amiRoleTagKey: 'cloud-vm-llm:role', + amiRoleTagValue: 'runtime-ami', + amiRunnerTagKey: 'cloud-vm-llm:runner', +}; + +const LLAMACPP: DeployConfig = { + runner: 'llamacpp', + modelId: 'unsloth/Qwen3.6-27B-MTP-GGUF', + quant: 'UD-Q6_K_XL', + weightsPrefix: 'models/llamacpp/unsloth/Qwen3.6-27B-MTP-GGUF/UD-Q6_K_XL/', + contextSize: 131072, + servedModelName: 'qwen3.6-27b', + serveArgs: [], +}; + +const VLLM: DeployConfig = { + runner: 'vllm', + modelId: 'Qwen/Qwen3.6-27B-FP8', + quant: '', + weightsPrefix: 'models/vllm/Qwen/Qwen3.6-27B-FP8/', + contextSize: 32768, + servedModelName: 'Qwen/Qwen3.6-27B-FP8', + serveArgs: [], +}; + +describe('buildSeedUserData', () => { + it('downloads only the requested quant and normalises it to model.gguf for llamacpp', () => { + const script = buildSeedUserData(LLAMACPP, ENV); + expect(script).toContain("export QUANT='UD-Q6_K_XL'"); + expect(script).toContain('allow_patterns'); + expect(script).toContain('/opt/llm/model/model.gguf'); + // mmproj/projector files would otherwise be picked up as "the" GGUF. + expect(script).toContain('mmproj'); + }); + + it('downloads the whole checkpoint for vllm', () => { + const script = buildSeedUserData(VLLM, ENV); + expect(script).toContain('snapshot_download'); + expect(script).not.toContain('allow_patterns'); + expect(script).not.toContain('model.gguf'); + }); + + it('syncs to the config’s derived prefix and self-terminates', () => { + const script = buildSeedUserData(LLAMACPP, ENV); + expect(script).toContain( + "aws s3 sync /opt/llm/model/ 's3://weights-bucket/models/llamacpp/unsloth/Qwen3.6-27B-MTP-GGUF/UD-Q6_K_XL/'", + ); + expect(script.trimEnd().endsWith('shutdown -h now')).toBe(true); + }); + + it('fetches the HF token only when one is configured', () => { + expect(buildSeedUserData(LLAMACPP, ENV)).not.toContain('get-secret-value'); + const withToken = buildSeedUserData(LLAMACPP, { + ...ENV, + hfSecretArn: 'arn:aws:secretsmanager:us-east-1:1:secret:hf', + }); + expect(withToken).toContain("get-secret-value --secret-id 'arn:aws:secretsmanager:us-east-1:1:secret:hf'"); + }); + + it('aborts on any failing step so a partial download is never synced', () => { + expect(buildSeedUserData(LLAMACPP, ENV)).toContain('set -euxo pipefail'); + }); +}); diff --git a/remote/test/stack.test.ts b/remote/test/stack.test.ts new file mode 100644 index 00000000..40b03c3d --- /dev/null +++ b/remote/test/stack.test.ts @@ -0,0 +1,311 @@ +import * as cdk from 'aws-cdk-lib'; +import { Match, Template } from 'aws-cdk-lib/assertions'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { loadConfig } from '../lib/config'; +import { ImageStack } from '../lib/image-stack'; +import { LlmStack } from '../lib/llm-stack'; +import { parseDeployConfig, UNCONFIGURED_DEPLOY_CONFIG } from '../lambda/shared/deploy-config'; + +const CONTEXT = { allowedCidr: '203.0.113.7/32', runner: 'vllm' }; +// Keep tests hermetic: never read the developer's real .env at the repo root. +const NO_DOTENV = path.join(os.tmpdir(), 'cloud-vm-llm-no-such-env'); + +function runtimeTemplate(context: Record = CONTEXT): Template { + const app = new cdk.App({ context }); + const config = loadConfig(app, NO_DOTENV); + return Template.fromStack(new LlmStack(app, 'test-runtime', { config, env: { region: config.region } })); +} + +function imageTemplate(context: Record = CONTEXT): Template { + const app = new cdk.App({ context }); + const config = loadConfig(app, NO_DOTENV); + return Template.fromStack(new ImageStack(app, 'test-image', { config, env: { region: config.region } })); +} + +function tempDotEnv(content: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cloud-vm-llm-test-')); + const file = path.join(dir, '.env'); + fs.writeFileSync(file, content); + return file; +} + +describe('config', () => { + it('requires allowedCidr', () => { + expect(() => loadConfig(new cdk.App(), NO_DOTENV)).toThrow(/allowedCidr/); + }); + + it('rejects a non-CIDR allowedCidr', () => { + const app = new cdk.App({ context: { allowedCidr: 'not-a-cidr', runner: 'vllm' } }); + expect(() => loadConfig(app, NO_DOTENV)).toThrow(/IPv4 CIDR/); + }); + + it('requires a runner to be chosen (no default)', () => { + const app = new cdk.App({ context: { allowedCidr: '203.0.113.7/32' } }); + expect(() => loadConfig(app, NO_DOTENV)).toThrow(/runner/); + const bad = new cdk.App({ context: { allowedCidr: '203.0.113.7/32', runner: 'tgi' } }); + expect(() => loadConfig(bad, NO_DOTENV)).toThrow(/runner/); + }); + + it('reads ALLOWED_CIDR and HF_TOKEN from a .env file', () => { + const dotEnv = tempDotEnv('# comment\nALLOWED_CIDR=203.0.113.9/32\nHF_TOKEN=hf_test\n'); + const config = loadConfig(new cdk.App({ context: { runner: 'vllm' } }), dotEnv); + expect(config.allowedCidr).toBe('203.0.113.9/32'); + expect(config.hfToken).toBe('hf_test'); + }); + + it('applies defaults', () => { + const config = loadConfig(new cdk.App({ context: CONTEXT }), NO_DOTENV); + expect(config.runner).toBe('vllm'); + expect(config.region).toBe('us-east-1'); + expect(config.availabilityZones).toEqual(['us-east-1b', 'us-east-1c', 'us-east-1d', 'us-east-1e']); + expect(config.modelId).toBe('Qwen/Qwen3.6-27B-FP8'); + expect(config.instanceType).toBe('g6e.xlarge'); + expect(config.builderInstanceType).toBe('m5.xlarge'); + expect(config.imageVolumeGb).toBe(80); + expect(config.vllmVersion).toBe('0.26.0'); + expect(config.nvidiaDriverPackage).toContain('nvidia-driver'); + expect(config.maxRuntimeMinutes).toBe(240); + }); + + it('parses a comma-separated availabilityZones override', () => { + const app = new cdk.App({ context: { ...CONTEXT, availabilityZones: 'us-east-1b, us-east-1c' } }); + expect(loadConfig(app, NO_DOTENV).availabilityZones).toEqual(['us-east-1b', 'us-east-1c']); + }); +}); + +describe('LlmStack (runtime)', () => { + let template: Template; + beforeAll(() => { + template = runtimeTemplate(); + }); + + it('holds no EC2 instance and no persistent EBS volume', () => { + template.resourceCountIs('AWS::EC2::Instance', 0); + template.resourceCountIs('AWS::EC2::Volume', 0); + }); + + it('creates one public subnet per configured AZ', () => { + const subnets = template.findResources('AWS::EC2::Subnet'); + const azs = Object.values(subnets).map((s) => s.Properties.AvailabilityZone); + expect(azs.sort()).toEqual(['us-east-1b', 'us-east-1c', 'us-east-1d', 'us-east-1e']); + }); + + it('opens only the vLLM port, only to the allowed CIDR', () => { + template.hasResourceProperties('AWS::EC2::SecurityGroup', { + SecurityGroupIngress: [ + Match.objectLike({ CidrIp: '203.0.113.7/32', FromPort: 8000, ToPort: 8000, IpProtocol: 'tcp' }), + ], + }); + }); + + it('allocates an Elastic IP but does not attach it at deploy', () => { + template.resourceCountIs('AWS::EC2::EIP', 1); + for (const eip of Object.values(template.findResources('AWS::EC2::EIP'))) { + expect(eip.Properties?.InstanceId).toBeUndefined(); + } + }); + + it('creates the start, stop and deploy Lambdas with IAM-authenticated function URLs', () => { + template.resourceCountIs('AWS::Lambda::Function', 3); + const urls = template.findResources('AWS::Lambda::Url'); + expect(Object.keys(urls)).toHaveLength(3); + for (const url of Object.values(urls)) { + expect(url.Properties.AuthType).toBe('AWS_IAM'); + } + }); + + it('has a placeholder deploy-config parameter and a deploy Lambda that writes it', () => { + const params = template.findResources('AWS::SSM::Parameter'); + const deployParam = Object.values(params).find( + (p) => p.Properties.Name === '/cloud-vm-llm/deploy-config', + ); + expect(deployParam).toBeDefined(); + // The parameter's CloudFormation value is a constant placeholder so a later + // `cdk deploy` never clobbers a real (outfit/manual) config. The cfg-derived + // initial config is emitted as an output instead, which `pnpm deploy` seeds + // over the placeholder the first time. + expect(deployParam!.Properties.Value).toBe(UNCONFIGURED_DEPLOY_CONFIG); + const initialOutput = Object.values(template.findOutputs('InitialDeployConfig'))[0]; + const initial = parseDeployConfig(initialOutput.Value); + expect(initial.runner).toBe('vllm'); + expect(initial.contextSize).toBe(32768); + expect(initial.serveArgs).toContain('--enforce-eager'); + expect(initial.serveArgs).toContain('qwen3_coder'); + + const fns = template.findResources('AWS::Lambda::Function'); + const deploy = Object.values(fns).find((f) => + String(f.Properties.Description).includes('deploy-config'), + ); + expect(deploy).toBeDefined(); + expect(Object.keys(template.findOutputs('DeployUrl'))).toHaveLength(1); + }); + + it('lets the deploy Lambda seed the weights, with PassRole scoped to the seed role', () => { + const fns = template.findResources('AWS::Lambda::Function'); + const deploy = Object.values(fns).find((f) => + String(f.Properties.Description).includes('deploy-config'), + ); + const env = deploy!.Properties.Environment.Variables; + for (const key of [ + 'WEIGHTS_BUCKET', + 'SEED_INSTANCE_TYPE', + 'SEED_SUBNET_ID', + 'SEED_SECURITY_GROUP_ID', + 'SEED_INSTANCE_PROFILE_ARN', + 'AMI_ROLE_TAG_KEY', + 'AMI_RUNNER_TAG_KEY', + ]) { + expect(env).toHaveProperty(key); + } + + const passRole = allPolicyStatements(template).filter((s) => + [s.Action].flat().includes('iam:PassRole'), + ); + expect(passRole.length).toBeGreaterThan(0); + // A wildcard PassRole would let a caller hand EC2 any role in the account. + for (const statement of passRole) { + expect(JSON.stringify(statement.Resource)).not.toBe('"*"'); + expect(JSON.stringify(statement.Condition)).toContain('ec2.amazonaws.com'); + } + }); + + it('schedules the idle check every 5 minutes', () => { + template.hasResourceProperties('AWS::Events::Rule', { ScheduleExpression: 'rate(5 minutes)' }); + }); + + it('grants the start Lambda RunInstances, PassRole, AssociateAddress and DescribeImages', () => { + const actions = allPolicyActions(template); + expect(actions).toContain('ec2:RunInstances'); + expect(actions).toContain('ec2:AssociateAddress'); + expect(actions).toContain('iam:PassRole'); + expect(actions).toContain('ec2:DescribeImages'); + }); + + it('tag-scopes the terminate permission', () => { + const statements = allPolicyStatements(template); + const terminate = statements.find((s) => [s.Action].flat().includes('ec2:TerminateInstances')); + expect(terminate).toBeDefined(); + expect(JSON.stringify(terminate!.Condition)).toContain('cloud-vm-llm'); + }); + + it('passes the AMI role tag, weights bucket and subnet list to the start Lambda', () => { + const fns = template.findResources('AWS::Lambda::Function'); + const start = Object.values(fns).find((f) => + String(f.Properties.Description).includes('Launches the baked AMI'), + ); + const env = start!.Properties.Environment.Variables; + expect(env.AMI_ROLE_TAG_KEY).toBe('cloud-vm-llm:role'); + expect(env.WEIGHTS_BUCKET).toBeDefined(); + expect(env.DEPLOY_CONFIG_PARAM).toBeDefined(); + expect(env.SUBNET_IDS).toBeDefined(); + }); + + it('creates an S3 weights bucket retained on destroy', () => { + template.resourceCountIs('AWS::S3::Bucket', 1); + const bucket = Object.values(template.findResources('AWS::S3::Bucket'))[0]; + expect(bucket.DeletionPolicy).toBe('Retain'); + }); + + it('gives the runtime instance role read on the weights, and a separate seed profile', () => { + // Two instance profiles: the runtime instance and the disposable seed one. + template.resourceCountIs('AWS::IAM::InstanceProfile', 2); + const actions = allPolicyActions(template); + expect(actions.some((a) => a.startsWith('s3:GetObject'))).toBe(true); + expect(actions.some((a) => a.startsWith('s3:PutObject'))).toBe(true); + }); + + it('creates the HF-token secret only when a token is configured', () => { + // The runtime always has the API-key secret; the HF secret is conditional. + template.resourceCountIs('AWS::SecretsManager::Secret', 1); + runtimeTemplate({ ...CONTEXT, hfToken: 'hf_abc' }).resourceCountIs( + 'AWS::SecretsManager::Secret', + 2, + ); + }); + + it('outputs the outfit remote config, EIP and seed inputs', () => { + expect(Object.keys(template.findOutputs('OutfitRemoteConfig'))).toHaveLength(1); + expect(Object.keys(template.findOutputs('EipAddress'))).toHaveLength(1); + expect(Object.keys(template.findOutputs('WeightsBucket'))).toHaveLength(1); + expect(Object.keys(template.findOutputs('SeedInstanceProfileArn'))).toHaveLength(1); + }); +}); + +describe('ImageStack', () => { + let template: Template; + beforeAll(() => { + template = imageTemplate(); + }); + + it('defines a pipeline per runner and runs no build at deploy time', () => { + template.resourceCountIs('AWS::ImageBuilder::Component', 2); + template.resourceCountIs('AWS::ImageBuilder::ImageRecipe', 2); + // One shared infrastructure config; a distribution + pipeline per runner. + template.resourceCountIs('AWS::ImageBuilder::InfrastructureConfiguration', 1); + template.resourceCountIs('AWS::ImageBuilder::DistributionConfiguration', 2); + template.resourceCountIs('AWS::ImageBuilder::ImagePipeline', 2); + // No Image resource — a bake never blocks or fails the stack deploy. + template.resourceCountIs('AWS::ImageBuilder::Image', 0); + }); + + it('resizes the AMI root to the configured size on the right device', () => { + template.hasResourceProperties('AWS::ImageBuilder::ImageRecipe', { + BlockDeviceMappings: [ + Match.objectLike({ DeviceName: '/dev/sda1', Ebs: Match.objectLike({ VolumeSize: 80 }) }), + ], + }); + }); + + it('parameterises each runner recipe (vLLM version, llama.cpp release, driver)', () => { + const recipes = Object.values(template.findResources('AWS::ImageBuilder::ImageRecipe')); + const params = recipes.map((r) => r.Properties.Components[0].Parameters); + const names = (p: { Name: string }[]) => p.map((x) => x.Name); + const vllm = params.find((p) => names(p).includes('VllmVersion')); + const llamacpp = params.find((p) => names(p).includes('LlamacppRelease')); + expect(vllm!.find((p: { Name: string }) => p.Name === 'VllmVersion').Value).toEqual(['0.26.0']); + expect(llamacpp).toBeDefined(); + // Both need the driver. + for (const p of params) { + expect(names(p)).toContain('NvidiaDriverPackage'); + } + }); + + it('tags each AMI with the runtime role and its runner (no model tag)', () => { + const dists = Object.values( + template.findResources('AWS::ImageBuilder::DistributionConfiguration'), + ); + const tagSets = dists.map( + (d) => d.Properties.Distributions[0].AmiDistributionConfiguration.AmiTags, + ); + for (const tags of tagSets) { + expect(tags['cloud-vm-llm:role']).toBe('runtime-ami'); + expect(tags['cloud-vm-llm:model']).toBeUndefined(); + } + expect(tagSets.map((t) => t['cloud-vm-llm:runner']).sort()).toEqual(['llamacpp', 'vllm']); + }); + + it('pins the builder VPC to the first configured AZ', () => { + for (const subnet of Object.values(template.findResources('AWS::EC2::Subnet'))) { + expect(subnet.Properties.AvailabilityZone).toBe('us-east-1b'); + } + }); + + it('bakes no secret and no model into the AMI (model-agnostic)', () => { + template.resourceCountIs('AWS::SecretsManager::Secret', 0); + }); +}); + +function allPolicyStatements( + template: Template, +): { Action: string | string[]; Resource?: unknown; Condition?: unknown }[] { + return Object.values(template.findResources('AWS::IAM::Policy')).flatMap( + (p) => p.Properties.PolicyDocument.Statement, + ); +} + +function allPolicyActions(template: Template): string[] { + return allPolicyStatements(template).flatMap((s) => [s.Action].flat()); +} diff --git a/remote/tsconfig.json b/remote/tsconfig.json new file mode 100644 index 00000000..7042e01e --- /dev/null +++ b/remote/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "commonjs", + "moduleResolution": "node", + "lib": ["ES2022"], + "strict": true, + "noImplicitReturns": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "noEmit": true, + "types": ["node"] + }, + "include": ["bin/**/*.ts", "lib/**/*.ts", "lambda/**/*.ts", "test/**/*.ts", "vitest.config.ts"] +} diff --git a/remote/vitest.config.ts b/remote/vitest.config.ts new file mode 100644 index 00000000..1a65507c --- /dev/null +++ b/remote/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + include: ['test/**/*.test.ts'], + environment: 'node', + // Stack synth (with esbuild bundling of the Lambdas) is slow on first run. + testTimeout: 120_000, + }, +}); diff --git a/scripts/check-no-cloud-identifiers.sh b/scripts/check-no-cloud-identifiers.sh new file mode 100755 index 00000000..dccd3f63 --- /dev/null +++ b/scripts/check-no-cloud-identifiers.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Fails when anything tracked in this repository looks like a real cloud +# identifier. This repository is PUBLIC: account numbers, ARNs, endpoint +# hostnames, allocated addresses, bucket names and resource ids belong in a +# developer's working tree (gitignored) or in the deployment's own outputs — +# never in a commit. +# +# Examples in docs and tests must use reserved documentation values instead: +# addresses 198.51.100.0/24, 203.0.113.0/24, 192.0.2.0/24 (RFC 5737) +# accounts an obviously fake id — twelve zeroes, or a single digit +# hostnames an elided or example host (example.com, ) +# +# Usage: scripts/check-no-cloud-identifiers.sh [ref] +# With no argument it scans the tracked working tree. +set -uo pipefail + +cd "$(dirname "$0")/.." + +# name=pattern. Kept deliberately narrow so ordinary prose does not trip it. +patterns=( + "AWS account id=[^0-9A-Za-z][0-9]{12}[^0-9A-Za-z]" + "AWS ARN=arn:aws[a-z-]*:[a-z0-9-]+:[a-z0-9-]*:[0-9]{12}:" + "Lambda Function URL host=[a-z0-9]{20,}\.lambda-url\." + "EC2/EBS/VPC resource id=\b(i|vol|subnet|sg|vpc|ami|eni|snap)-[0-9a-f]{8,}\b" + "S3 bucket with a generated suffix=[a-z0-9-]*[0-9a-f]{8}-[a-z0-9]{12,}" +) + +# Reserved documentation ranges are the sanctioned stand-ins, so an address is +# only reported when it is outside them. +doc_ranges='(198\.51\.100\.|203\.0\.113\.|192\.0\.2\.|127\.0\.0\.|0\.0\.0\.0|10\.|192\.168\.|169\.254\.)' + +status=0 +# Lock files are nothing but hashes and pseudo-versions, which no pattern can +# tell from a generated resource name, and they never carry cloud identifiers. +files=$(git ls-files | grep -vE '(^|/)(go\.sum|go\.mod|pnpm-lock\.yaml|package-lock\.json)$') + +report() { + if [ "$status" -eq 0 ]; then + echo "Cloud identifiers must not be committed to this public repository." >&2 + echo >&2 + fi + status=1 + echo "$1" >&2 +} + +for entry in "${patterns[@]}"; do + name=${entry%%=*} + pattern=${entry#*=} + # shellcheck disable=SC2086 + if hits=$(echo "$files" | xargs grep -nEI "$pattern" 2>/dev/null); then + report "$name:" + echo "$hits" | sed 's/^/ /' >&2 + fi +done + +# A public IPv4 literal that is not a documentation/private address. +if hits=$(echo "$files" | xargs grep -nEI '\b([0-9]{1,3}\.){3}[0-9]{1,3}\b' 2>/dev/null | + grep -vE "$doc_ranges" | grep -vE '\b(0|1|2)\.[0-9]+\.[0-9]+\.[0-9]+\b'); then + report "Public IP address (use a documentation range from RFC 5737):" + echo "$hits" | sed 's/^/ /' >&2 +fi + +if [ "$status" -ne 0 ]; then + echo >&2 + echo "Replace them with the documentation values listed at the top of $0." >&2 + exit 1 +fi + +echo "No cloud identifiers found in tracked files."