diff --git a/README.md b/README.md index 3433d09b..49e9ad84 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,8 @@ it dresses the agent for you: - **One command, any model.** Pick from a built-in catalogue — OpenRouter, Bedrock, Ollama, llama.cpp, vLLM, oMLX, or any OpenAI-compatible endpoint. No - URLs or model ids to look up. + URLs to look up, and `outfit list --models` fetches the model ids straight from + the provider. - **Your config survives.** Settings are merged *into* what you already have. Other providers, your theme, even your comments stay exactly where you left them. - **Keys stay where they belong.** Secrets are read from a local `.env` and never @@ -86,6 +87,12 @@ See what's in the catalogue: outfit list ``` +Need a model id? Ask the provider itself — no memorising, no guessing: + +```sh +outfit list --models openrouter # the models it currently serves, live +``` + Add a provider and a model: ```sh @@ -143,7 +150,7 @@ quarter of the context for you. It can't exceed the context window. ## Usage ```sh -outfit list +outfit list [--models []] # the catalogue; --models fetches live model ids outfit show [--harness ] # show what the harness has configured outfit add --provider [--model ] [--alias ] [--context ] [--output ] [--base-url ] outfit remove --provider [--model ] [--alias ] diff --git a/cmd/outfit/complete.go b/cmd/outfit/complete.go index b2dab8e6..0b908352 100644 --- a/cmd/outfit/complete.go +++ b/cmd/outfit/complete.go @@ -20,7 +20,9 @@ import ( "github.com/lucinate-ai/outfit/internal/catalog" "github.com/lucinate-ai/outfit/internal/config" + "github.com/lucinate-ai/outfit/internal/discovery" "github.com/lucinate-ai/outfit/internal/harness" + "github.com/lucinate-ai/outfit/internal/opencode" ) //go:embed completion.bash @@ -61,6 +63,7 @@ const ( kindAliasOnly // registered aliases; a path would be meaningless kindHarness kindProvider + kindModel kindShell ) @@ -89,11 +92,11 @@ var selectionFlags = []string{ } // selectionValues maps those of them that take a value. The model id has no -// static source to complete against — the catalogue no longer enumerates models -// — so --model/-m consumes its value without offering candidates. +// static source in the catalogue; --model/-m completes from live per-provider +// discovery, scoped to the --provider already on the line. var selectionValues = map[string]candidateKind{ "--provider": kindProvider, "-p": kindProvider, - "--model": kindNone, "-m": kindNone, + "--model": kindModel, "-m": kindModel, "--alias": kindNone, "-a": kindNone, "--context": kindNone, "-c": kindNone, "--output": kindNone, "-o": kindNone, @@ -307,6 +310,19 @@ func candidatesFor(kind candidateKind, words []string) ([]string, string) { return nil, directiveNoFile } return cat.SortedProviderNames(), directiveNoFile + case kindModel: + p := providerOn(words) + if p == nil { + return nil, directiveNoFile + } + // Best-effort: source models live from the provider's endpoint. Any + // failure (offline, timeout, no key) yields no candidates, keeping + // completion quiet. + models, err := discovery.Models(p, "", opencode.EnvResolver("")) + if err != nil { + return nil, directiveNoFile + } + return models, directiveNoFile case kindShell: return completionShells, directiveNoFile default: @@ -330,6 +346,20 @@ func loadCatalogFor(words []string) (*catalog.Catalog, error) { return catalog.LoadFrom(catalog.ResolveCatalogPath(flagValue(words, "--providers"))) } +// providerOn returns the provider named by --provider/-p on the command line so +// far, or nil when there is none (or the catalogue will not load). +func providerOn(words []string) *catalog.Provider { + name := flagValue(words, "--provider", "-p") + if name == "" { + return nil + } + cat, err := loadCatalogFor(words) + if err != nil { + return nil + } + return cat.Providers[name] +} + // flagValue finds the value already given to one of names, in either the // attached (--flag=value, which bash splits into three words) or detached // (--flag value) form. It returns "" when the flag is not on the line. diff --git a/cmd/outfit/complete_test.go b/cmd/outfit/complete_test.go index 1676e7f1..2740eba4 100644 --- a/cmd/outfit/complete_test.go +++ b/cmd/outfit/complete_test.go @@ -1,6 +1,8 @@ package main import ( + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" @@ -231,21 +233,44 @@ func TestComplete_EqualsForm(t *testing.T) { func TestComplete_ModelValues(t *testing.T) { isolateConfig(t) - // No static model source, so no candidates and no error. - got, directive := complete(t, "add", "-p", "openrouter", "-m", "") + // amazon-bedrock has no models endpoint (no base URL), so discovery makes no + // network call and offers nothing — the offline, no-candidates path. + got, directive := complete(t, "add", "-p", "amazon-bedrock", "-m", "") if len(got) != 0 { - t.Errorf("expected no model candidates, got %v", got) + t.Errorf("expected no candidates for a non-discoverable provider, got %v", got) } if directive != directiveNoFile { t.Errorf("directive = %q, want %q", directive, directiveNoFile) } // -m consumes its value: the harness flag after it still completes. - if got, _ := complete(t, "add", "-p", "openrouter", "-m", "some-model", "-H", ""); !hasAll(got, "opencode", "pi") { + if got, _ := complete(t, "add", "-p", "amazon-bedrock", "-m", "some-model", "-H", ""); !hasAll(got, "opencode", "pi") { t.Errorf("a flag after --model did not complete; -m may not be consuming its value: %v", got) } } +// TestComplete_ModelDiscovery checks that --model/-m offers the models a +// provider's endpoint currently serves, sourced live from a stub. +func TestComplete_ModelDiscovery(t *testing.T) { + isolateConfig(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"data":[{"id":"disc-a"},{"id":"disc-b"}]}`)) + })) + defer srv.Close() + + dir := t.TempDir() + provPath := filepath.Join(dir, "providers.yaml") + mustWrite(t, provPath, "providers:\n stub:\n description: Stub\n npm: \"@ai-sdk/openai-compatible\"\n options:\n baseURL: "+srv.URL+"\n") + + got, directive := complete(t, "add", "--providers", provPath, "-p", "stub", "-m", "") + if !hasAll(got, "disc-a", "disc-b") { + t.Errorf("discovered models not offered: %v", got) + } + if directive != directiveNoFile { + t.Errorf("directive = %q, want %q", directive, directiveNoFile) + } +} + // TestComplete_AttachedEqualsForm checks the same flag=value case as it arrives // from zsh and PowerShell, which pass `--outfit=qw` as a single word rather than // splitting it on "=" the way bash does. diff --git a/cmd/outfit/main.go b/cmd/outfit/main.go index 368103da..d3cda4f5 100644 --- a/cmd/outfit/main.go +++ b/cmd/outfit/main.go @@ -60,6 +60,7 @@ import ( "github.com/lucinate-ai/outfit/internal/catalog" "github.com/lucinate-ai/outfit/internal/config" "github.com/lucinate-ai/outfit/internal/contextsize" + "github.com/lucinate-ai/outfit/internal/discovery" "github.com/lucinate-ai/outfit/internal/harness" "github.com/lucinate-ai/outfit/internal/opencode" "github.com/lucinate-ai/outfit/internal/outfit" @@ -1114,7 +1115,9 @@ func cmdShow(args []string) error { func cmdList(args []string) error { fs := flag.NewFlagSet("list", flag.ContinueOnError) var providers string + var showModels bool fs.StringVar(&providers, "providers", "", "path to a providers.yaml override") + fs.BoolVar(&showModels, "models", false, "also fetch each provider's current models live from its endpoint") if err := fs.Parse(args); err != nil { return err } @@ -1123,9 +1126,27 @@ func cmdList(args []string) error { if err != nil { return err } + + // An optional positional argument narrows the listing to one provider, which + // is the natural way to ask for a single provider's live models. + names := cat.SortedProviderNames() + if rest := fs.Args(); len(rest) > 0 { + if _, ok := cat.Providers[rest[0]]; !ok { + return fmt.Errorf("unknown provider %q (see `outfit list`)", rest[0]) + } + names = []string{rest[0]} + } + + // Only resolve keys and hit the network when --models is asked for; a plain + // list stays entirely offline. + var resolve func(string) string + if showModels { + resolve = opencode.EnvResolver("") + } + var b strings.Builder b.WriteString("Available providers:\n") - for _, name := range cat.SortedProviderNames() { + for _, name := range names { p := cat.Providers[name] fmt.Fprintf(&b, "\n %s — %s\n", name, p.Description) if p.APIKeyEnv != "" { @@ -1140,6 +1161,15 @@ func cmdList(args []string) error { harnesses = "opencode, pi" } fmt.Fprintf(&b, " harnesses: %s\n", harnesses) + if showModels { + if models, err := discovery.Models(p, "", resolve); err == nil && len(models) > 0 { + for _, m := range models { + fmt.Fprintf(&b, " model %s\n", m) + } + } else { + b.WriteString(" models: (none found)\n") + } + } } fmt.Print(b.String()) return nil diff --git a/cmd/outfit/main_test.go b/cmd/outfit/main_test.go index 0d71a682..9def590a 100644 --- a/cmd/outfit/main_test.go +++ b/cmd/outfit/main_test.go @@ -4,9 +4,12 @@ import ( "bytes" "encoding/json" "io" + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" + "sync/atomic" "testing" "github.com/lucinate-ai/outfit/internal/catalog" @@ -479,6 +482,53 @@ func TestCmdList(t *testing.T) { } } +// TestCmdList_Models covers `outfit list --models`: it fetches each provider's +// current models live, and a plain `outfit list` makes no network request. +func TestCmdList_Models(t *testing.T) { + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + _, _ = w.Write([]byte(`{"data":[{"id":"model-b"},{"id":"model-a"}]}`)) + })) + defer srv.Close() + + dir := t.TempDir() + provPath := filepath.Join(dir, "providers.yaml") + mustWrite(t, provPath, "providers:\n stub:\n description: Stub provider\n npm: \"@ai-sdk/openai-compatible\"\n options:\n baseURL: "+srv.URL+"\n") + + // --models on a single provider prints its live models (sorted). + out := captureStdout(t, func() { + if err := cmdList([]string{"--providers", provPath, "--models", "stub"}); err != nil { + t.Fatalf("cmdList --models: %v", err) + } + }) + if !strings.Contains(out, "model model-a") || !strings.Contains(out, "model model-b") { + t.Errorf("discovered models missing from output:\n%s", out) + } + if atomic.LoadInt32(&hits) == 0 { + t.Error("expected --models to make a discovery request") + } + + // A plain list makes no network request at all. + before := atomic.LoadInt32(&hits) + out = captureStdout(t, func() { + if err := cmdList([]string{"--providers", provPath}); err != nil { + t.Fatalf("cmdList: %v", err) + } + }) + if atomic.LoadInt32(&hits) != before { + t.Error("a plain `outfit list` should not hit the network") + } + if strings.Contains(out, "model model-a") { + t.Errorf("plain list should not print models:\n%s", out) + } + + // An unknown provider positional is an error. + if err := cmdList([]string{"--providers", provPath, "nope"}); err == nil { + t.Error("expected an error for an unknown provider filter") + } +} + func TestCmdInitProviders_EndToEnd(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "providers.yaml") diff --git a/docs/commands/list.md b/docs/commands/list.md index f38f7afd..efd47155 100644 --- a/docs/commands/list.md +++ b/docs/commands/list.md @@ -6,16 +6,25 @@ needs (if any), and which harnesses support it. ```sh outfit list outfit list --providers ./my-providers.yaml # a custom catalogue +outfit list --models # add each provider's live models +outfit list --models openrouter # just one provider's models ``` This is the *could* view — what's available to configure. For what your agent *currently has* configured, see [`outfit show`](show.md). +The catalogue names providers, not models — models change too often to curate. +`--models` fills that gap on demand: it asks each provider's own endpoint what it +currently serves (OpenRouter, an OpenAI-compatible gateway, a local llama.cpp or +Ollama server, …) and lists the ids you can drop into an `Outfit`'s `MODEL`. A +positional provider name narrows the query to one provider. + ## Flags | Flag | Meaning | | ---- | ------- | | `--providers` | Path to a custom catalogue (or set `OUTFIT_PROVIDERS`) — see [`outfit init-providers`](init-providers.md) | +| `--models` | Also fetch each listed provider's current models, live from its endpoint | ## Notes @@ -23,6 +32,10 @@ This is the *could* view — what's available to configure. For what your agent variable is set in a `.env` beside your `Outfit`, or in your environment. - Not every provider maps to every harness — the listing names which harnesses each supports (AWS Bedrock, for instance, is opencode-only). +- `--models` is best-effort: it uses the same base URL and key a selection + would, applies a short timeout, and prints `(none found)` when a provider is + unreachable or has no queryable endpoint (AWS Bedrock has none). A plain + `outfit list` makes no network request. ## See also diff --git a/docs/getting-started.md b/docs/getting-started.md index c7fbc19f..6560c734 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -19,6 +19,13 @@ outfit list That's the catalogue: every provider, the API key it needs (if any), and which harnesses support it. +Need a model id to go with a provider? Add `--models` and `outfit` asks the +provider's own endpoint what it currently serves: + +```sh +outfit list --models openrouter +``` + ## 3. Dress your agent Pick a provider and model. For a hosted one, drop the key in a `.env` or diff --git a/docs/outfit-file.md b/docs/outfit-file.md index 1f62b76c..5fd6d1fd 100644 --- a/docs/outfit-file.md +++ b/docs/outfit-file.md @@ -114,7 +114,9 @@ Rules: - **Comments** start with `#`, either on their own line or at the end of a line. Blank lines are ignored. -To see the available providers, run `outfit list`. +To see the available providers, run `outfit list`. To find a `MODEL` id for one, +run `outfit list --models `, which asks the provider's own endpoint +what it currently serves. ## Examples diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go new file mode 100644 index 00000000..c97d33ec --- /dev/null +++ b/internal/discovery/discovery.go @@ -0,0 +1,162 @@ +// Package discovery fetches the models a provider currently serves from the +// provider's own OpenAI-compatible `/models` endpoint. It is best-effort: any +// failure (unreachable endpoint, timeout, non-2xx status, unparseable body, +// missing key) returns an error that callers treat as "no models", never a +// panic, and a bounded timeout keeps a slow endpoint from hanging a command. +// +// The catalogue holds no models (see the provider-catalog spec); this is how +// `outfit list --models` and model tab-completion learn what a provider offers, +// live and without curation. Every discoverable provider here speaks the +// OpenAI-compatible `GET {baseURL}/models` shape — OpenRouter, vLLM, llama.cpp, +// the generic openai-compatible endpoint, and Ollama (whose compatibility layer +// serves `/v1/models`). A provider with no resolvable base URL (amazon-bedrock) +// is not discoverable over HTTP. +package discovery + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "sort" + "strings" + "sync" + "time" + + "github.com/lucinate-ai/outfit/internal/catalog" +) + +// baseURLEnv mirrors the catalogue's OUTFIT_BASE_URL override so discovery +// targets the same endpoint a selection would. +const baseURLEnv = "OUTFIT_BASE_URL" + +// requestTimeout bounds a single discovery request. Discovery is interactive +// (it backs `outfit list` and tab-completion), so a few seconds is the ceiling: +// a reachable endpoint answers in well under this, and an unreachable one fails +// fast enough not to strand the command. +const requestTimeout = 3 * time.Second + +// cacheTTL is how long a provider endpoint's models are reused within one +// process, so listing then completing does not re-hit the network. +const cacheTTL = 60 * time.Second + +var client = &http.Client{Timeout: requestTimeout} + +// ErrNotDiscoverable is returned for a provider with no resolvable base URL +// (for example amazon-bedrock), which has no HTTP `/models` endpoint to query. +var ErrNotDiscoverable = errors.New("provider has no discoverable models endpoint") + +type cacheEntry struct { + models []string + expires time.Time +} + +var ( + mu sync.Mutex + cache = map[string]cacheEntry{} +) + +// ResolveBaseURL determines the base URL to query for a provider, mirroring the +// base-URL precedence a selection uses: an explicit override, then +// OUTFIT_BASE_URL, then the provider's optionsFromEnv `baseURL`, then its static +// options.baseURL, then its Pi endpoint. It returns "" when none resolves, which +// marks the provider as not discoverable. +func ResolveBaseURL(p *catalog.Provider, override string, resolve func(string) string) string { + if override != "" { + return override + } + if v := resolve(baseURLEnv); v != "" { + return v + } + if env, ok := p.OptionsFromEnv["baseURL"]; ok { + if v := resolve(env); v != "" { + return v + } + } + if s, _ := p.Options["baseURL"].(string); s != "" { + return s + } + if p.Pi != nil && p.Pi.BaseURL != "" { + return p.Pi.BaseURL + } + return "" +} + +// Discoverable reports whether a provider can be queried for its models at all +// (i.e. a base URL resolves for it). +func Discoverable(p *catalog.Provider, resolve func(string) string) bool { + return ResolveBaseURL(p, "", resolve) != "" +} + +// Models returns, in stable order, the ids of the models a provider currently +// serves, querying its OpenAI-compatible `{baseURL}/models` endpoint. Results +// are cached per resolved endpoint for cacheTTL. When the provider declares an +// API key variable and it resolves to a value, that value is sent as a Bearer +// header and never stored. Any failure returns an error (callers treat it as +// "no models"); ErrNotDiscoverable means the provider has no endpoint to query. +func Models(p *catalog.Provider, baseURLOverride string, resolve func(string) string) ([]string, error) { + base := ResolveBaseURL(p, baseURLOverride, resolve) + if base == "" { + return nil, ErrNotDiscoverable + } + endpoint := strings.TrimRight(base, "/") + "/models" + + mu.Lock() + if e, ok := cache[endpoint]; ok && time.Now().Before(e.expires) { + models := e.models + mu.Unlock() + return models, nil + } + mu.Unlock() + + models, err := fetch(endpoint, p.APIKeyEnv, resolve) + if err != nil { + return nil, err + } + + mu.Lock() + cache[endpoint] = cacheEntry{models: models, expires: time.Now().Add(cacheTTL)} + mu.Unlock() + return models, nil +} + +// fetch performs the HTTP GET and parses the OpenAI `{"data":[{"id":...}]}` +// shape into a sorted list of model ids. +func fetch(endpoint, keyEnv string, resolve func(string) string) ([]string, error) { + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + if keyEnv != "" { + if key := resolve(keyEnv); key != "" { + req.Header.Set("Authorization", "Bearer "+key) + } + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("discovery %s: unexpected status %s", endpoint, resp.Status) + } + + var payload struct { + Data []struct { + ID string `json:"id"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + return nil, err + } + + ids := make([]string, 0, len(payload.Data)) + for _, m := range payload.Data { + if m.ID != "" { + ids = append(ids, m.ID) + } + } + sort.Strings(ids) + return ids, nil +} diff --git a/internal/discovery/discovery_test.go b/internal/discovery/discovery_test.go new file mode 100644 index 00000000..d9f36c7b --- /dev/null +++ b/internal/discovery/discovery_test.go @@ -0,0 +1,180 @@ +package discovery + +import ( + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/lucinate-ai/outfit/internal/catalog" +) + +func noEnv(string) string { return "" } + +// resetCache clears the in-process cache so tests don't see each other's +// entries. Endpoints already differ per httptest server, but this keeps the +// count-based assertions unambiguous. +func resetCache() { + mu.Lock() + cache = map[string]cacheEntry{} + mu.Unlock() +} + +func stubModels(body string, hits *int32) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if hits != nil { + atomic.AddInt32(hits, 1) + } + _, _ = w.Write([]byte(body)) + })) +} + +func providerAt(url string) *catalog.Provider { + return &catalog.Provider{Options: map[string]any{"baseURL": url}} +} + +func TestModels_OpenAICompatibleSortsIDs(t *testing.T) { + resetCache() + var path string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path = r.URL.Path + _, _ = w.Write([]byte(`{"data":[{"id":"zeta"},{"id":"alpha"},{"id":""}]}`)) + })) + defer srv.Close() + + got, err := Models(providerAt(srv.URL), "", noEnv) + if err != nil { + t.Fatalf("Models: %v", err) + } + if path != "/models" { + t.Errorf("queried path = %q, want /models", path) + } + if len(got) != 2 || got[0] != "alpha" || got[1] != "zeta" { + t.Errorf("got %v, want [alpha zeta] (sorted, empty id dropped)", got) + } +} + +func TestModels_SendsBearerKeyOnlyWhenResolved(t *testing.T) { + resetCache() + var auth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + auth = r.Header.Get("Authorization") + _, _ = w.Write([]byte(`{"data":[{"id":"m"}]}`)) + })) + defer srv.Close() + + p := &catalog.Provider{APIKeyEnv: "TOK", Options: map[string]any{"baseURL": srv.URL}} + resolve := func(n string) string { + if n == "TOK" { + return "secret" + } + return "" + } + if _, err := Models(p, "", resolve); err != nil { + t.Fatal(err) + } + if auth != "Bearer secret" { + t.Errorf("Authorization = %q, want %q", auth, "Bearer secret") + } + + // With the key unset, no Authorization header is sent. + resetCache() + auth = "" + if _, err := Models(p, "", noEnv); err != nil { + t.Fatal(err) + } + if auth != "" { + t.Errorf("Authorization = %q, want empty when key is unset", auth) + } +} + +func TestModels_CachesPerEndpoint(t *testing.T) { + resetCache() + var hits int32 + srv := stubModels(`{"data":[{"id":"m"}]}`, &hits) + defer srv.Close() + + p := providerAt(srv.URL) + for i := 0; i < 3; i++ { + if _, err := Models(p, "", noEnv); err != nil { + t.Fatal(err) + } + } + if got := atomic.LoadInt32(&hits); got != 1 { + t.Errorf("endpoint hit %d times, want 1 (later lookups cached)", got) + } +} + +func TestModels_FailuresYieldError(t *testing.T) { + resetCache() + + bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer bad.Close() + if _, err := Models(providerAt(bad.URL), "", noEnv); err == nil { + t.Error("expected error for a 500 status") + } + + junk := stubModels("not json at all", nil) + defer junk.Close() + if _, err := Models(providerAt(junk.URL), "", noEnv); err == nil { + t.Error("expected error for an unparseable body") + } + + // Port 1 refuses fast, standing in for an unreachable endpoint. + if _, err := Models(providerAt("http://127.0.0.1:1"), "", noEnv); err == nil { + t.Error("expected error for an unreachable endpoint") + } +} + +func TestModels_NotDiscoverable(t *testing.T) { + resetCache() + if _, err := Models(&catalog.Provider{}, "", noEnv); !errors.Is(err, ErrNotDiscoverable) { + t.Errorf("err = %v, want ErrNotDiscoverable for a provider with no base URL", err) + } +} + +func TestResolveBaseURL_Precedence(t *testing.T) { + p := &catalog.Provider{ + Options: map[string]any{"baseURL": "http://opt/v1"}, + OptionsFromEnv: map[string]string{"baseURL": "MYURL"}, + Pi: &catalog.PiConfig{BaseURL: "http://pi/v1"}, + } + + if got := ResolveBaseURL(p, "http://flag/v1", noEnv); got != "http://flag/v1" { + t.Errorf("override should win, got %q", got) + } + env := func(n string) string { + if n == "OUTFIT_BASE_URL" { + return "http://env/v1" + } + return "" + } + if got := ResolveBaseURL(p, "", env); got != "http://env/v1" { + t.Errorf("OUTFIT_BASE_URL should win over catalogue, got %q", got) + } + ofe := func(n string) string { + if n == "MYURL" { + return "http://ofe/v1" + } + return "" + } + if got := ResolveBaseURL(p, "", ofe); got != "http://ofe/v1" { + t.Errorf("optionsFromEnv should win over static, got %q", got) + } + if got := ResolveBaseURL(p, "", noEnv); got != "http://opt/v1" { + t.Errorf("static options.baseURL expected, got %q", got) + } + + // Pi endpoint is the last fallback (used by openrouter, which has no + // options.baseURL). + piOnly := &catalog.Provider{Pi: &catalog.PiConfig{BaseURL: "http://pi/v1"}} + if got := ResolveBaseURL(piOnly, "", noEnv); got != "http://pi/v1" { + t.Errorf("pi baseUrl fallback expected, got %q", got) + } + if got := ResolveBaseURL(&catalog.Provider{}, "", noEnv); got != "" { + t.Errorf("no base URL should resolve to empty, got %q", got) + } +} diff --git a/openspec/changes/live-model-discovery/.openspec.yaml b/openspec/changes/live-model-discovery/.openspec.yaml new file mode 100644 index 00000000..f205fc72 --- /dev/null +++ b/openspec/changes/live-model-discovery/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-29 diff --git a/openspec/changes/live-model-discovery/design.md b/openspec/changes/live-model-discovery/design.md new file mode 100644 index 00000000..63339174 --- /dev/null +++ b/openspec/changes/live-model-discovery/design.md @@ -0,0 +1,84 @@ +## Context + +`outfit` currently has no code that fetches model metadata over the network; the only HTTP +client (`internal/remote/remote.go`, a 10-minute-timeout client for AWS Lambda control) is +unrelated. After `retire-model-families`, the catalogue is provider plumbing only, so there +is no static list of models to browse. Each provider, however, exposes its own model list +over an OpenAI-compatible `GET {baseURL}/models` — including Ollama and llama.cpp. This +change adds a small, best-effort discovery layer that reads it. + +## Goals / Non-Goals + +**Goals:** +- A `internal/discovery` package that, given a provider and a resolve function, returns the + provider's currently-served model ids. +- Per-provider protocol handling (OpenAI-compatible `data[].id`; Ollama `models[].name`). +- Short-TTL in-process caching and a bounded per-request timeout. +- Best-effort semantics: any failure means "no models", never a command failure. +- Surfacing via `outfit list --models ` and model tab-completion. + +**Non-Goals:** +- Amazon Bedrock discovery (`ListFoundationModels`) — deferred; it needs the AWS SDK path. +- Fetching pricing, context windows, or other metadata — only model ids/names here. +- Persisting a discovery cache across processes. +- Making plain `outfit list` hit the network — discovery is opt-in via `--models`, and + completion (already latency-sensitive and failure-tolerant) sources it on demand. + +## Decisions + +**One uniform OpenAI-compatible query, no per-provider protocol switch.** Ollama's +compatibility layer serves `GET /v1/models` in the same `{"data":[{"id":…}]}` shape as +OpenRouter, vLLM, llama.cpp, and the generic endpoint, so a single `GET {baseURL}/models` +path covers every discoverable provider. This removed the need for a separate Ollama +`/api/tags` adapter and for any protocol field on the provider — discoverability is simply +"does a base URL resolve?", which keeps the catalogue plumbing-only. AWS Bedrock resolves +no base URL, so it is not discoverable (its SDK `ListFoundationModels` is out of scope). + +**Reuse the base-URL and key resolution the selection path already uses.** Discovery calls +the same `resolve` closure (`.env` beside the Outfit / working dir, then environment) and +the same base-URL precedence, so a provider that applies also discovers, with no new config +surface. The key is sent as an `Authorization: Bearer` header (or Ollama's keyless call) +and never written anywhere. + +**A dedicated short-timeout HTTP client**, separate from the remote package's 10-minute +client. Discovery is interactive; a few seconds is the right ceiling. Model completion uses +the same layer, so its cache and timeout keep completion instant. + +**Best-effort by construction.** The public function returns `([]string, error)` internally +but callers (`list`, completion) treat any error as "no models". `list --models` prints a +one-line "no models found" note; completion emits nothing. This matches the completion +spec's existing "never error, never stderr" rule. + +Alternatives considered: a build-time snapshot of each provider's models (rejected — that +is the curation treadmill again); querying an aggregator like OpenRouter for *all* +providers (rejected — only OpenRouter knows its own catalogue; other providers must be +asked directly); Hugging Face Hub as the source (rejected — it is a weights hub and does +not know a provider's served ids). + +## Risks / Trade-offs + +- [A provider endpoint is slow or down] → bounded timeout + best-effort fallback; the + command still succeeds on plumbing alone. +- [`--models` leaks that a key is set by succeeding/failing auth] → acceptable; the user is + querying their own configured provider. No key value is ever printed. +- [Completion latency if discovery is slow] → TTL cache + short timeout; completion treats + a timeout as "no candidates", so a cold, slow endpoint just yields no model suggestions + that keystroke. +- [Ordering with `retire-model-families`] → this change is additive (a new capability) and + does not modify the same requirements, but its narrative assumes the static lists are + gone. It should be reviewed/merged after `retire-model-families`. + +## Migration Plan + +Additive; no data migration. Ship the `internal/discovery` package plus the `--models` +flag and completion wiring together. Rollback is a straight revert — nothing else depends +on discovery. + +## Open Questions + +- Resolved during implementation: no `discovery:` field or protocol switch is needed — + every discoverable provider answers `GET {baseURL}/models`, and discoverability reduces + to whether a base URL resolves. +- Should `outfit list` (no flag) opportunistically show discovered models for *local* + providers only (llama.cpp/Ollama/vLLM on localhost), where the call is cheap and offline + is the norm? Left out for now to keep plain `list` network-free; `--models` is explicit. diff --git a/openspec/changes/live-model-discovery/proposal.md b/openspec/changes/live-model-discovery/proposal.md new file mode 100644 index 00000000..54560458 --- /dev/null +++ b/openspec/changes/live-model-discovery/proposal.md @@ -0,0 +1,61 @@ +## Why + +With the static model lists removed from the catalogue (see the +`retire-model-families` change), `outfit list` no longer suggests any models — the +catalogue is provider plumbing only. But most of these providers already expose their +*own* authoritative, always-current model list over HTTP. Querying that on demand gives +users a browsable, self-updating set of models to pick from with **zero curation** — the +right fix for "models change all the time", and a far better source than a general weights +hub like Hugging Face, which does not know what a given provider actually serves. + +## What Changes + +- Add an optional live model-discovery layer that, for a selected provider, fetches the + models the provider currently serves from the provider's own endpoint: + - OpenAI-compatible `GET {baseURL}/models` for every discoverable provider — + OpenRouter (`https://openrouter.ai/api/v1/models`), vLLM, llama.cpp, the generic + `openai-compatible` endpoint, and Ollama (whose compatibility layer serves + `/v1/models`). + - (Amazon Bedrock `ListFoundationModels` is out of scope for this change.) +- Surface discovered models through `outfit list --models ` (and enrich the + per-provider block of `outfit list` when a discovery source is available). +- Source shell model completion (`outfit __complete … --model `) from discovery when + the provider supports it, bounded and cached so completion stays instant and quiet. +- Degrade gracefully: discovery is best-effort. A network failure, timeout, missing key, + or unparseable response yields "no models" — never an error, and `outfit list` still + prints the provider's plumbing. Results are cached for a short TTL so repeated calls in + one session do not re-hit the network. +- Never write secrets: discovery reuses the same resolved API key the provider already + uses, read from the environment/`.env`; it is only sent as a request header, never + persisted. + +## Capabilities + +### New Capabilities + +- `model-discovery`: fetching a provider's currently-served models from the provider's own + HTTP endpoint (per-provider protocol, auth, caching, and graceful-failure rules) and + surfacing them in `outfit list` and shell completion. + +### Modified Capabilities + +_None._ Discovery is additive: it introduces a new capability rather than changing the +`provider-catalog`, `outfit-files`, or `shell-completion` requirements. It builds on the +`retire-model-families` change (which removes the static lists it replaces) and should +land after it. + +## Impact + +- Code: a new `internal/discovery` package (HTTP client, per-provider protocol adapters, + in-memory TTL cache), reusing the client pattern in `internal/remote/remote.go`. Wiring + in `cmd/outfit/main.go` (`cmdList` + a `--models` flag) and `cmd/outfit/complete.go` + (model candidates). +- Catalogue: providers gain a small hint of which discovery protocol applies. The existing + `pi.api` already distinguishes `openai-completions` vs others; discovery can key off the + provider id / base URL, or an explicit `discovery:` field may be added to + `providers.yaml` if the mapping is not derivable. Decide in design. +- Dependencies: standard library `net/http` only; no new module dependency. +- Tests: unit tests for each protocol adapter against recorded fixtures, cache behaviour, + and offline/error fallback; a `--models` command test using a stub server. +- Docs: `docs/commands/list.md` (the `--models` flag), and a note in `docs/outfit-file.md` + pointing users at `outfit list --models` to discover model ids. diff --git a/openspec/changes/live-model-discovery/specs/model-discovery/spec.md b/openspec/changes/live-model-discovery/specs/model-discovery/spec.md new file mode 100644 index 00000000..2f8b625b --- /dev/null +++ b/openspec/changes/live-model-discovery/specs/model-discovery/spec.md @@ -0,0 +1,91 @@ +## ADDED Requirements + +### Requirement: Per-provider model discovery + +The system SHALL be able to fetch the set of models a provider currently serves from that +provider's own OpenAI-compatible endpoint with `GET {baseURL}/models`, reading model ids +from the returned `data[].id` list and returning them in stable order. This covers every +discoverable provider — OpenRouter, vLLM, llama.cpp, the generic `openai-compatible` +endpoint, and Ollama (whose compatibility layer serves `/v1/models`). + +The base URL SHALL be resolved with the same precedence a selection uses (`--base-url`, +then `OUTFIT_BASE_URL`, then the provider's catalogue value, then its Pi endpoint). A +provider with no resolvable base URL (for example AWS Bedrock) is not discoverable. When +the provider declares an API key variable and it resolves to a value, that value SHALL be +sent as the request's `Authorization` header; a resolved key SHALL NOT be written to disk +or logged. + +#### Scenario: A provider lists the models it serves + +- **WHEN** discovery runs for a provider whose endpoint answers `GET {baseURL}/models` + with a `data` array of objects carrying `id` +- **THEN** those ids are returned, in stable order, as the provider's discovered models + +#### Scenario: A provider with no endpoint is not discoverable + +- **WHEN** discovery runs for a provider with no resolvable base URL (such as AWS Bedrock) +- **THEN** discovery reports the provider is not discoverable and returns no models + +#### Scenario: The resolved key is only sent, never stored + +- **WHEN** discovery queries a provider whose key resolves from the environment +- **THEN** the key is sent as an `Authorization` header and never written to any file or log + +### Requirement: Discovery is best-effort and quiet + +Model discovery SHALL be best-effort. A network failure, a non-success HTTP status, a +timeout, a missing required key, or an unparseable response SHALL yield an empty model set +rather than an error, and SHALL NOT prevent the surrounding command from succeeding. +Discovery SHALL apply a bounded request timeout so a slow or unreachable endpoint cannot +hang a command. + +#### Scenario: Offline discovery does not fail the command + +- **WHEN** `outfit list --models ` runs and the provider's endpoint is + unreachable +- **THEN** the command still prints the provider's plumbing, reports no models were found, + and exits successfully + +#### Scenario: A slow endpoint cannot hang the command + +- **WHEN** a provider's endpoint does not respond within the discovery timeout +- **THEN** discovery abandons the request and returns no models + +#### Scenario: Completion stays silent on discovery failure + +- **WHEN** model completion sources from discovery and the endpoint errors +- **THEN** `__complete` offers no model candidates, exits zero, and writes nothing to + stderr + +### Requirement: Discovery result caching + +Within a single process, discovery SHALL cache a provider's result for a short time-to-live +so that repeated lookups (for example, listing then completing) do not re-hit the network. +A cache entry SHALL be keyed by the resolved provider endpoint. + +#### Scenario: Repeated lookups reuse the cached result + +- **WHEN** discovery for the same provider endpoint is requested twice within the TTL +- **THEN** the second lookup returns the cached models without a second network request + +### Requirement: Surfacing discovered models + +`outfit list --models ` SHALL print the provider's discovered models beneath its +plumbing. Without `--models`, `outfit list` SHALL behave as before (plumbing only) and +SHALL NOT perform any network request. Shell model completion SHALL offer discovered models +for a provider that supports discovery, scoped to the `--provider` already on the line. + +#### Scenario: Listing a provider's live models + +- **WHEN** the user runs `outfit list --models openrouter` and discovery succeeds +- **THEN** the provider's currently-served model ids are printed under its entry + +#### Scenario: Plain list makes no network call + +- **WHEN** the user runs `outfit list` with no `--models` flag +- **THEN** no discovery request is made and only provider plumbing is printed + +#### Scenario: Model completion offers discovered ids + +- **WHEN** the user completes `outfit add -p openrouter -m ` and discovery succeeds +- **THEN** the provider's discovered model ids are offered as candidates diff --git a/openspec/changes/live-model-discovery/tasks.md b/openspec/changes/live-model-discovery/tasks.md new file mode 100644 index 00000000..9a12a89e --- /dev/null +++ b/openspec/changes/live-model-discovery/tasks.md @@ -0,0 +1,36 @@ +## 1. Discovery package + +- [x] 1.1 Create `internal/discovery` with a short-timeout `*http.Client` (3s, not the remote package's 10 minutes) and a `Models(provider, baseURLOverride, resolve) ([]string, error)` entry point. +- [x] 1.2 Implement the uniform OpenAI-compatible query: `GET {baseURL}/models`, parse `data[].id`, return sorted; send `Authorization: Bearer ` only when a key resolves. (Ollama and llama.cpp both serve `/v1/models`, so one path covers every discoverable provider — no separate Ollama `/api/tags` adapter needed.) +- [x] 1.3 `ResolveBaseURL` / `Discoverable`: a provider with no resolvable base URL (AWS Bedrock) is not discoverable and returns `ErrNotDiscoverable`. +- [x] 1.4 Resolve base URL with selection precedence (`--base-url`, `OUTFIT_BASE_URL`, optionsFromEnv, options.baseURL, then the Pi endpoint) via the same `resolve` closure the apply path uses; never write the key anywhere. + +## 2. Caching and failure semantics + +- [x] 2.1 In-process TTL cache (60s) keyed by resolved endpoint; a second lookup within the TTL returns cached models with no network call. +- [x] 2.2 Every failure path (network error, non-2xx, timeout, missing endpoint, bad JSON) returns an error the callers treat as "no models", never a fatal error. + +## 3. Surfacing + +- [x] 3.1 Add a `--models` flag to `outfit list` (plus an optional positional provider filter); when set, call discovery per listed provider and print discovered ids, with a `(none found)` note on empty/failure. +- [x] 3.2 Keep plain `outfit list` (no `--models`) network-free — it resolves no keys and makes no request. +- [x] 3.3 Source model completion in `cmd/outfit/complete.go` from discovery for the typed `--provider`; emit nothing on failure. + +## 4. Tests + +- [x] 4.1 Discovery package tests against httptest stubs: sorting, auth-header (sent only when resolved), base-URL precedence. +- [x] 4.2 Cache test: three lookups, one served endpoint hit (counting stub server). +- [x] 4.3 Failure tests: unreachable endpoint, 500 status, malformed body, not-discoverable — each yields an error/no models. +- [x] 4.4 `outfit list --models` command test against a stub; assert models printed, plain `list` makes no request, unknown provider filter errors. +- [x] 4.5 Completion test: discovered ids offered from a stub; non-discoverable provider offers nothing. Coverage: discovery 96%, cmd/outfit 90%. + +## 5. Docs + +- [x] 5.1 Document `outfit list --models` (and the provider positional) in `docs/commands/list.md`. +- [x] 5.2 Note in `docs/outfit-file.md` that `outfit list --models ` shows model ids for `MODEL`. + +## 6. Verify + +- [x] 6.1 `go build ./...`, `gofmt -l` clean, `go test ./... -cover` green. +- [x] 6.2 Manual check against a local stub: `outfit list --models stub` prints live models; `--models` on a non-discoverable provider prints `(none found)` and exits 0; plain `list` stays offline. +- [x] 6.3 `openspec validate live-model-discovery` passes.