Merge pull request #812 from docker/docker-model-gateway

Add model-cli gateway

Author: ericcurtin

.github/workflows/e2e-test.yml

@@ -34,5 +34,12 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         uses: docker/setup-docker-action@1a6edb0ba9ac496f6850236981f15d8f9a82254d
 
+      - name: Install build dependencies (Linux)
+        if: matrix.os == 'ubuntu-latest'
+        run: sudo apt-get update && sudo apt-get install -y libssl-dev pkg-config
+
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@stable
+
       - name: Run e2e tests
         run: make e2e

cmd/cli/.gitignore

@@ -1,3 +1,4 @@
 model-cli
 .idea/
-dist/
+dist/
+commands/gateway_lib/

cmd/cli/Makefile

@@ -1,4 +1,4 @@
-.PHONY: all build clean link mock unit-tests docs
+.PHONY: all build build-gateway clean link mock unit-tests docs
 
 BINARY_NAME=model-cli
 
@@ -10,11 +10,22 @@ VERSION ?=
 MACOS_MIN_VERSION := 14.0
 MACOS_MIN_VERSION_LDFLAG := -mmacosx-version-min=$(MACOS_MIN_VERSION)
 
+GATEWAY_RUST_DIR := ../../model-cli
+# The static library produced by cargo and consumed by CGo.
+GATEWAY_LIB_DIR := commands/gateway_lib
+
 all: build
 
-build:
+build-gateway:
+	@echo "Building gateway static library (Rust)..."
+	@mkdir -p $(GATEWAY_LIB_DIR)
+	cargo build --release --manifest-path $(GATEWAY_RUST_DIR)/Cargo.toml
+	@cp $(GATEWAY_RUST_DIR)/target/release/libmodel_cli_gateway.a $(GATEWAY_LIB_DIR)/libgateway.a
+	@echo "Gateway library staged at $(GATEWAY_LIB_DIR)/libgateway.a"
+
+build: build-gateway
 	@echo "Building $(BINARY_NAME)..."
-	go build -ldflags="-s -w -X github.com/docker/model-runner/cmd/cli/desktop.Version=$(shell git describe --tags --always --dirty --match 'v*')" -o $(BINARY_NAME) .
+	CGO_ENABLED=1 go build -tags gateway -ldflags="-s -w -X github.com/docker/model-runner/cmd/cli/desktop.Version=$(shell git describe --tags --always --dirty --match 'v*')" -o $(BINARY_NAME) .
 
 link:
 	@if [ ! -f $(BINARY_NAME) ]; then \
@@ -69,6 +80,7 @@ unit-tests:
 clean:
 	@echo "Cleaning up..."
 	@rm -f $(BINARY_NAME)
+	@rm -rf $(GATEWAY_LIB_DIR)
 	@echo "Cleaned!"
 
 docs:

cmd/cli/commands/gateway.go

@@ -0,0 +1,54 @@
+//go:build gateway
+
+package commands
+
+/*
+#cgo LDFLAGS: -L${SRCDIR}/gateway_lib -lgateway -lm
+#cgo darwin LDFLAGS: -framework CoreFoundation -framework Security -framework SystemConfiguration
+#cgo linux LDFLAGS: -lpthread -ldl -lssl -lcrypto
+
+#include <stdlib.h>
+
+extern int run_gateway(int argc, const char **argv);
+*/
+import "C"
+
+import (
+	"fmt"
+	"unsafe"
+
+	"github.com/spf13/cobra"
+)
+
+func newGatewayCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "gateway",
+		Short: "Run an OpenAI-compatible LLM gateway",
+		Long: `Run an OpenAI-compatible LLM gateway that routes requests to configured providers.
+
+Supported providers include Docker Model Runner, Ollama, OpenAI, Anthropic,
+Groq, Mistral, Azure OpenAI, and many more OpenAI-compatible endpoints.`,
+		// Pass all flags straight through to the Rust arg parser.
+		DisableFlagParsing: true,
+		SilenceUsage:       true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			// Build a C argv: ["model-cli"] + args
+			cArgs := make([]*C.char, 0, len(args)+1)
+			cArgs = append(cArgs, C.CString("model-cli"))
+			for _, a := range args {
+				cArgs = append(cArgs, C.CString(a))
+			}
+			defer func() {
+				for _, p := range cArgs {
+					C.free(unsafe.Pointer(p))
+				}
+			}()
+
+			rc := C.run_gateway(C.int(len(cArgs)), (**C.char)(unsafe.Pointer(&cArgs[0])))
+			if rc != 0 {
+				return fmt.Errorf("gateway exited with code %d", rc)
+			}
+			return nil
+		},
+	}
+}

cmd/cli/commands/gateway_stub.go

@@ -0,0 +1,43 @@
+//go:build !gateway
+
+package commands
+
+import (
+	"errors"
+
+	"github.com/spf13/cobra"
+)
+
+// newGatewayCmd returns a metadata-only stub used by the docs generator and
+// any build that was compiled without -tags gateway (i.e. without the Rust
+// static library).  The command is fully described so that 'make docs' can
+// generate correct reference documentation, but it exits with an error if
+// actually invoked.
+func newGatewayCmd() *cobra.Command {
+	var (
+		config  string
+		host    string
+		port    uint16
+		verbose bool
+	)
+
+	c := &cobra.Command{
+		Use:   "gateway",
+		Short: "Run an OpenAI-compatible LLM gateway",
+		Long: `Run an OpenAI-compatible LLM gateway that routes requests to configured providers.
+
+Supported providers include Docker Model Runner, Ollama, OpenAI, Anthropic,
+Groq, Mistral, Azure OpenAI, and many more OpenAI-compatible endpoints.`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return errors.New("gateway is not available in this build; rebuild with 'make build-cli'")
+		},
+	}
+
+	c.Flags().StringVarP(&config, "config", "c", "", "Path to the YAML configuration file")
+	c.Flags().StringVar(&host, "host", "0.0.0.0", "Host address to bind to")
+	c.Flags().Uint16VarP(&port, "port", "p", 4000, "Port to listen on")
+	c.Flags().BoolVarP(&verbose, "verbose", "v", false, "Enable verbose (debug) logging")
+	_ = c.MarkFlagRequired("config")
+
+	return c
+}

cmd/cli/commands/root.go

@@ -96,6 +96,7 @@ func NewRootCmd(cli *command.DockerCli) *cobra.Command {
 		newSearchCmd(),
 		newSkillsCmd(),
 	)
+	rootCmd.AddCommand(newGatewayCmd())
 
 	// Commands that require a running model runner. These are wrapped to ensure the standalone runner is available.
 	for _, cmd := range []*cobra.Command{

cmd/cli/docs/reference/docker_model.yaml

@@ -8,6 +8,7 @@ plink: docker.yaml
 cname:
     - docker model bench
     - docker model df
+    - docker model gateway
     - docker model inspect
     - docker model install-runner
     - docker model launch
@@ -36,6 +37,7 @@ cname:
 clink:
     - docker_model_bench.yaml
     - docker_model_df.yaml
+    - docker_model_gateway.yaml
     - docker_model_inspect.yaml
     - docker_model_install-runner.yaml
     - docker_model_launch.yaml

cmd/cli/docs/reference/docker_model_gateway.yaml

@@ -0,0 +1,182 @@
+command: docker model gateway
+short: Run an OpenAI-compatible LLM gateway
+long: |-
+    `docker model gateway` starts a local OpenAI-compatible HTTP gateway that routes
+    requests to one or more configured LLM providers. It supports Docker Model Runner
+    as a first-class provider, alongside Ollama, OpenAI, Anthropic, Groq, Mistral,
+    Azure OpenAI, and many other OpenAI-compatible endpoints.
+
+    The gateway is configured through a YAML file that declares the model list,
+    provider routing, load-balancing, retries, and fallbacks.
+
+    ### Configuration file format
+
+    ```yaml
+    model_list:
+      - model_name: <alias exposed to clients>
+        params:
+          model: <provider>/<upstream-model-name>
+          api_base: <optional base URL override>
+          api_key: <optional key or os.environ/VAR_NAME>
+
+    general_settings:
+      master_key: <optional API key required by clients>
+      num_retries: <optional integer, default 0>
+      fallbacks:
+        - <primary-alias>: [<fallback-alias>, ...]
+    ```
+
+    The `model` field under `params` uses the format `provider/model-name`.
+    Supported provider prefixes include: `docker_model_runner`, `openai`,
+    `anthropic`, `ollama`, `groq`, `mistral`, `together_ai`, `deepseek`,
+    `fireworks_ai`, `openrouter`, `perplexity`, `xai`, `nvidia_nim`,
+    `cerebras`, `sambanova`, `deepinfra`, `azure`, `azure_ai`, `vllm`,
+    `lm_studio`, `huggingface`.
+
+    API keys can be supplied inline, as `os.environ/VAR_NAME` references, or as
+    `${VAR_NAME}` references. The gateway resolves well-known environment variables
+    automatically (for example, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`).
+usage: docker model gateway
+pname: docker model
+plink: docker_model.yaml
+options:
+    - option: config
+      shorthand: c
+      value_type: string
+      description: Path to the YAML configuration file
+      deprecated: false
+      hidden: false
+      experimental: false
+      experimentalcli: false
+      kubernetes: false
+      swarm: false
+    - option: host
+      value_type: string
+      default_value: 0.0.0.0
+      description: Host address to bind to
+      deprecated: false
+      hidden: false
+      experimental: false
+      experimentalcli: false
+      kubernetes: false
+      swarm: false
+    - option: port
+      shorthand: p
+      value_type: uint16
+      default_value: "4000"
+      description: Port to listen on
+      deprecated: false
+      hidden: false
+      experimental: false
+      experimentalcli: false
+      kubernetes: false
+      swarm: false
+    - option: verbose
+      shorthand: v
+      value_type: bool
+      default_value: "false"
+      description: Enable verbose (debug) logging
+      deprecated: false
+      hidden: false
+      experimental: false
+      experimentalcli: false
+      kubernetes: false
+      swarm: false
+examples: |-
+    ### Route requests to Docker Model Runner
+
+    ```yaml
+    model_list:
+      - model_name: smollm2
+        params:
+          model: docker_model_runner/ai/smollm2
+          api_base: http://localhost:12434/engines/llama.cpp/v1
+    ```
+
+    ```console
+    $ docker model gateway --config config.yaml
+    ```
+
+    The gateway starts on `http://0.0.0.0:4000`. Send requests using any
+    OpenAI-compatible client:
+
+    ```console
+    $ curl http://localhost:4000/v1/chat/completions \
+        -H "Content-Type: application/json" \
+        -d '{
+          "model": "smollm2",
+          "messages": [{"role": "user", "content": "Hello"}]
+        }'
+    ```
+
+    ### Route requests to multiple providers with fallback
+
+    ```yaml
+    model_list:
+      - model_name: fast
+        params:
+          model: groq/llama-3.1-8b-instant
+          api_key: os.environ/GROQ_API_KEY
+      - model_name: smart
+        params:
+          model: openai/gpt-4o
+          api_key: os.environ/OPENAI_API_KEY
+      - model_name: local
+        params:
+          model: docker_model_runner/ai/smollm2
+          api_base: http://localhost:12434/engines/llama.cpp/v1
+
+    general_settings:
+      num_retries: 2
+      fallbacks:
+        - fast: [local]
+        - smart: [fast, local]
+    ```
+
+    ```console
+    $ docker model gateway --config config.yaml --port 8080
+    ```
+
+    ### Secure the gateway with an API key
+
+    ```yaml
+    model_list:
+      - model_name: smollm2
+        params:
+          model: docker_model_runner/ai/smollm2
+          api_base: http://localhost:12434/engines/llama.cpp/v1
+
+    general_settings:
+      master_key: os.environ/GATEWAY_API_KEY
+    ```
+
+    ```console
+    $ GATEWAY_API_KEY=my-secret docker model gateway --config config.yaml
+    ```
+
+    Clients must then pass the key as a Bearer token or via the `x-api-key` header:
+
+    ```console
+    $ curl http://localhost:4000/v1/chat/completions \
+        -H "Content-Type: application/json" \
+        -d '{"model": "smollm2", "messages": [{"role": "user", "content": "Hi"}]}'
+    ```
+
+    ### Use a custom host and port
+
+    ```console
+    $ docker model gateway --config config.yaml --host 127.0.0.1 --port 9000
+    ```
+
+    ### Enable debug logging
+
+    ```console
+    $ docker model gateway --config config.yaml --verbose
+    ```
+deprecated: false
+hidden: false
+experimental: false
+experimentalcli: false
+kubernetes: false
+swarm: false
+

cmd/cli/docs/reference/model.md

@@ -9,6 +9,7 @@ Docker Model Runner
 |:------------------------------------------------|:-----------------------------------------------------------------------|
 | [`bench`](model_bench.md)                       | Benchmark a model's performance at different concurrency levels        |
 | [`df`](model_df.md)                             | Show Docker Model Runner disk usage                                    |
+| [`gateway`](model_gateway.md)                   | Run an OpenAI-compatible LLM gateway                                   |
 | [`inspect`](model_inspect.md)                   | Display detailed information on one model                              |
 | [`install-runner`](model_install-runner.md)     | Install Docker Model Runner (Docker Engine only)                       |
 | [`launch`](model_launch.md)                     | Launch an app configured to use Docker Model Runner                    |