Add audio, moderations, and tokenize support

Register previously missing OpenAI-compatible routes so they are
proxied to backends instead of being rejected with 404:
- /v1/audio/transcriptions and /v1/audio/translations (multipart/form-data)
- /v1/audio/speech (JSON)
- /v1/moderations
- /tokenize and /detokenize (vLLM extension)

Add BackendModeAudio and a dedicated handleAudioInference handler that
parses the model field from multipart form data rather than a JSON body.
vLLM (Linux and Metal) passes audio requests through natively; llama.cpp
returns a descriptive error directing users to use chat completions with
an input_audio content part instead.

Signed-off-by: Eric Curtin <eric.curtin@docker.com>

Author: ericcurtin

.github/workflows/ci.yml

@@ -53,6 +53,22 @@ jobs:
           go mod tidy
           git diff --exit-code go.mod go.sum
 
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache Rust build artifacts
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+
+      - name: Build Rust router library
+        run: make build-router-lib
+
       - name: Run tests with race detection
         run: go test -race ./...
 

.gitignore

@@ -15,3 +15,7 @@ llamacpp/install
 vllm-metal-macos-arm64-*.tar.gz
 
 .DS_Store
+
+# Cargo workspace build output
+/target/
+/Cargo.lock

Cargo.toml

@@ -0,0 +1,10 @@
+[workspace]
+members = [
+    "dmr-common",
+    "model-cli",
+    "router",
+]
+resolver = "2"
+
+[profile.release]
+lto = true

Dockerfile

@@ -15,25 +15,31 @@ FROM docker.io/library/golang:${GO_VERSION}-bookworm AS builder
 
 ARG VERSION
 
-# Install git for go mod download if needed
-RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/*
+# Install git and the Rust toolchain (needed to build libdmr_router.a via CGo).
+RUN apt-get update && apt-get install -y --no-install-recommends git curl && rm -rf /var/lib/apt/lists/*
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
+ENV PATH="/root/.cargo/bin:${PATH}"
 
 WORKDIR /app
 
 # Copy go mod/sum first for better caching
 COPY --link go.mod go.sum ./
 
-# Download dependencies (with cache mounts)
+# Download Go dependencies (with cache mounts)
 RUN --mount=type=cache,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache/go-build \
     go mod download
 
 # Copy the rest of the source code
 COPY --link . .
 
-# Build the Go binary (static build)
-RUN --mount=type=cache,target=/go/pkg/mod \
+# Build the Rust dmr-router static library, then the Go binary.
+# Both steps share a single RUN so the Rust output is available to the linker.
+RUN --mount=type=cache,target=/root/.cargo/registry \
+    --mount=type=cache,target=/root/.cargo/git \
     --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    cargo build --release --manifest-path router/Cargo.toml && \
     CGO_ENABLED=1 GOOS=linux go build -ldflags="-s -w -X main.Version=${VERSION}" -o model-runner .
 
 # Build the Go binary for SGLang (without vLLM)

Makefile

@@ -23,19 +23,24 @@ DOCKER_BUILD_ARGS := \
 	-t $(DOCKER_IMAGE)
 
 # Phony targets grouped by category
-.PHONY: build build-cli build-dmr build-llamacpp install-cli run clean test integration-tests e2e
+.PHONY: build build-cli build-dmr build-llamacpp build-router-lib install-cli run clean test integration-tests e2e
 .PHONY: validate validate-all lint help
 .PHONY: docker-build docker-build-multiplatform docker-run docker-run-impl
 .PHONY: docker-build-vllm docker-run-vllm docker-build-sglang docker-run-sglang
 .PHONY: test-docker-ce-installation
 .PHONY: vllm-metal-build vllm-metal-install vllm-metal-dev vllm-metal-clean
 .PHONY: diffusers-build diffusers-install diffusers-dev diffusers-clean
+
 # Default target: build server, CLI plugin, and dmr convenience wrapper
 .DEFAULT_GOAL := build
 
 build: build-server build-cli build-dmr
 
-build-server:
+# Build the Rust dmr-router static library.
+build-router-lib:
+	cargo build --release --manifest-path router/Cargo.toml
+
+build-server: build-router-lib
 	CGO_ENABLED=1 go build -ldflags="-s -w -X main.Version=$(shell git describe --tags --always --dirty --match 'v*')" -o $(APP_NAME) .
 
 build-cli:
@@ -68,6 +73,7 @@ clean:
 	rm -f $(APP_NAME)
 	rm -f dmr
 	rm -f model-runner.sock
+	cargo clean --manifest-path router/Cargo.toml
 
 # Run tests
 test:

cmd/cli/Makefile

@@ -20,7 +20,7 @@ build-gateway:
 	@echo "Building gateway static library (Rust)..."
 	@mkdir -p $(GATEWAY_LIB_DIR)
 	cargo build --release --manifest-path $(GATEWAY_RUST_DIR)/Cargo.toml
-	@cp $(GATEWAY_RUST_DIR)/target/release/libmodel_cli_gateway.a $(GATEWAY_LIB_DIR)/libgateway.a
+	@cp ../../target/release/libmodel_cli_gateway.a $(GATEWAY_LIB_DIR)/libgateway.a
 	@echo "Gateway library staged at $(GATEWAY_LIB_DIR)/libgateway.a"
 
 build: build-gateway

dmr-common/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "dmr-common"
+version = "0.1.0"
+edition = "2021"
+description = "Shared utilities for Docker Model Runner Rust crates"
+
+[dependencies]
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }

dmr-common/src/lib.rs

@@ -0,0 +1,30 @@
+//! Shared utilities for Docker Model Runner Rust crates.
+
+/// Return the current time as seconds since the Unix epoch.
+///
+/// Used when constructing OpenAI-format response objects that require a
+/// `created` timestamp.  Returns 0 on the (extremely unlikely) event that
+/// the system clock predates 1970.
+pub fn unix_now_secs() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_secs()
+}
+
+/// Initialise the global tracing subscriber.
+///
+/// Reads `RUST_LOG` from the environment; falls back to `fallback` if unset
+/// or invalid.  Silently ignores subsequent calls (e.g. when called from both
+/// a library entry point and a binary entry point in the same process).
+///
+/// # Arguments
+/// * `fallback` – default filter string, e.g. `"info"` or `"myapp=debug"`.
+pub fn init_tracing(fallback: &str) {
+    let _ = tracing_subscriber::fmt()
+        .with_env_filter(
+            tracing_subscriber::EnvFilter::try_from_default_env()
+                .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(fallback)),
+        )
+        .try_init();
+}