Add audio, moderations, and tokenize/detokenize endpoint support

Register previously missing OpenAI-compatible routes so they are no
longer rejected with 404:
  /v1/audio/transcriptions, /v1/audio/translations (multipart/form-data)
  /v1/audio/speech (JSON)
  /v1/moderations
  /tokenize and /detokenize (vLLM extension)

Add BackendModeAudio and handleAudioInference which extracts the model
field from multipart form data. vLLM passes audio requests through
natively; llama.cpp returns a descriptive error directing users to the
chat completions input_audio content-part instead. Address review
feedback: extract shared scheduleInference helper to restore parity
(auto-install progress, preload-only, recorder, origin tracking) between
handleOpenAIInference and handleAudioInference; fix multipart temp-file
leak (defer MultipartForm.RemoveAll); tighten /v1/audio/ path matching
to explicit HasSuffix checks; make Content-Type check case-insensitive.

Replace the Go routing layer with a Rust reverse proxy (axum + tokio)
compiled as a CGo-linked staticlib (router/). The Rust router owns:
  - All route registration and path aliasing (/v1/ -> /engines/, etc.)
  - CORS middleware matching the Go CorsMiddleware semantics
  - Path normalisation (NormalizePathLayer)
  - Static routes: GET / and GET /version

The deleted Go files (pkg/routing/router.go, pkg/routing/routing.go,
pkg/middleware/alias.go) are fully replaced. pkg/routing/service.go is
trimmed; main.go registers Ollama, Anthropic, and Responses handlers
directly on the backend mux.

The in-process CGo callback path (pkg/router/handler.go) replaces the
network proxy hop: Rust calls Go's http.Handler directly via a streaming
protocol — dmr_write_chunk()/dmr_close_stream() push response chunks
into a tokio::sync::mpsc channel as Go writes them, so streaming
endpoints like POST /models/create deliver progress in real time without
buffering the full response.

Rust shared utilities are extracted into a dmr-common workspace crate
(init_tracing, unix_now_secs). model-cli Rust code is deduplicated:
shared send_and_check free function, request_timeout helper,
apply_azure_version helper, build_app/run_gateway_async extracted to
handlers.rs.

Build system:
  - Cargo workspace root (Cargo.toml) unifies all Rust crates
  - make build-router-lib compiles the Rust staticlib before go build
  - Dockerfile installs Rust and builds libdmr_router.a in the builder
    stage
  - CI test job installs Rust toolchain and builds the library so
    go test -race works with CGo enabled
  - Platform-split CGo LDFLAGS: Darwin keeps -framework flags, Linux
    uses plain -lpthread/-ldl/-lm
  - pkg/router/router_stub.go provides a no-op implementation for
    CGO_ENABLED=0 builds (lint, cross-compilation)

Signed-off-by: Eric Curtin <eric.curtin@docker.com>

Author: ericcurtin

.github/workflows/ci.yml

@@ -53,6 +53,22 @@ jobs:
           go mod tidy
           git diff --exit-code go.mod go.sum
 
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache Rust build artifacts
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+
+      - name: Build Rust router library
+        run: make build-router-lib
+
       - name: Run tests with race detection
         run: go test -race ./...
 

.gitignore

@@ -15,3 +15,7 @@ llamacpp/install
 vllm-metal-macos-arm64-*.tar.gz
 
 .DS_Store
+
+# Cargo workspace build output
+/target/
+/Cargo.lock

Cargo.toml

@@ -0,0 +1,10 @@
+[workspace]
+members = [
+    "dmr-common",
+    "model-cli",
+    "router",
+]
+resolver = "2"
+
+[profile.release]
+lto = true

Dockerfile

@@ -15,25 +15,31 @@ FROM docker.io/library/golang:${GO_VERSION}-bookworm AS builder
 
 ARG VERSION
 
-# Install git for go mod download if needed
-RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/*
+# Install git and the Rust toolchain (needed to build libdmr_router.a via CGo).
+RUN apt-get update && apt-get install -y --no-install-recommends git curl && rm -rf /var/lib/apt/lists/*
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
+ENV PATH="/root/.cargo/bin:${PATH}"
 
 WORKDIR /app
 
 # Copy go mod/sum first for better caching
 COPY --link go.mod go.sum ./
 
-# Download dependencies (with cache mounts)
+# Download Go dependencies (with cache mounts)
 RUN --mount=type=cache,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache/go-build \
     go mod download
 
 # Copy the rest of the source code
 COPY --link . .
 
-# Build the Go binary (static build)
-RUN --mount=type=cache,target=/go/pkg/mod \
+# Build the Rust dmr-router static library, then the Go binary.
+# Both steps share a single RUN so the Rust output is available to the linker.
+RUN --mount=type=cache,target=/root/.cargo/registry \
+    --mount=type=cache,target=/root/.cargo/git \
     --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    cargo build --release --manifest-path router/Cargo.toml && \
     CGO_ENABLED=1 GOOS=linux go build -ldflags="-s -w -X main.Version=${VERSION}" -o model-runner .
 
 # Build the Go binary for SGLang (without vLLM)

Makefile

@@ -23,19 +23,24 @@ DOCKER_BUILD_ARGS := \
 	-t $(DOCKER_IMAGE)
 
 # Phony targets grouped by category
-.PHONY: build build-cli build-dmr build-llamacpp install-cli run clean test integration-tests e2e
+.PHONY: build build-cli build-dmr build-llamacpp build-router-lib install-cli run clean test integration-tests e2e
 .PHONY: validate validate-all lint help
 .PHONY: docker-build docker-build-multiplatform docker-run docker-run-impl
 .PHONY: docker-build-vllm docker-run-vllm docker-build-sglang docker-run-sglang
 .PHONY: test-docker-ce-installation
 .PHONY: vllm-metal-build vllm-metal-install vllm-metal-dev vllm-metal-clean
 .PHONY: diffusers-build diffusers-install diffusers-dev diffusers-clean
+
 # Default target: build server, CLI plugin, and dmr convenience wrapper
 .DEFAULT_GOAL := build
 
 build: build-server build-cli build-dmr
 
-build-server:
+# Build the Rust dmr-router static library.
+build-router-lib:
+	cargo build --release --manifest-path router/Cargo.toml
+
+build-server: build-router-lib
 	CGO_ENABLED=1 go build -ldflags="-s -w -X main.Version=$(shell git describe --tags --always --dirty --match 'v*')" -o $(APP_NAME) .
 
 build-cli:
@@ -68,6 +73,7 @@ clean:
 	rm -f $(APP_NAME)
 	rm -f dmr
 	rm -f model-runner.sock
+	cargo clean --manifest-path router/Cargo.toml
 
 # Run tests
 test:

cmd/cli/Makefile

@@ -20,7 +20,7 @@ build-gateway:
 	@echo "Building gateway static library (Rust)..."
 	@mkdir -p $(GATEWAY_LIB_DIR)
 	cargo build --release --manifest-path $(GATEWAY_RUST_DIR)/Cargo.toml
-	@cp $(GATEWAY_RUST_DIR)/target/release/libmodel_cli_gateway.a $(GATEWAY_LIB_DIR)/libgateway.a
+	@cp ../../target/release/libmodel_cli_gateway.a $(GATEWAY_LIB_DIR)/libgateway.a
 	@echo "Gateway library staged at $(GATEWAY_LIB_DIR)/libgateway.a"
 
 build: build-gateway

dmr-common/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "dmr-common"
+version = "0.1.0"
+edition = "2021"
+description = "Shared utilities for Docker Model Runner Rust crates"
+
+[dependencies]
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }

dmr-common/src/lib.rs

@@ -0,0 +1,30 @@
+//! Shared utilities for Docker Model Runner Rust crates.
+
+/// Return the current time as seconds since the Unix epoch.
+///
+/// Used when constructing OpenAI-format response objects that require a
+/// `created` timestamp.  Returns 0 on the (extremely unlikely) event that
+/// the system clock predates 1970.
+pub fn unix_now_secs() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs()
+}
+
+/// Initialise the global tracing subscriber.
+///
+/// Reads `RUST_LOG` from the environment; falls back to `fallback` if unset
+/// or invalid.  Silently ignores subsequent calls (e.g. when called from both
+/// a library entry point and a binary entry point in the same process).
+///
+/// # Arguments
+/// * `fallback` – default filter string, e.g. `"info"` or `"myapp=debug"`.
+pub fn init_tracing(fallback: &str) {
+    let _ = tracing_subscriber::fmt()
+        .with_env_filter(
+            tracing_subscriber::EnvFilter::try_from_default_env()
+                .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(fallback)),
+        )
+        .try_init();
+}