Merge pull request #621 from docker/add-default-runner-config

feat: add default runner configuration if none exists

Author: ericcurtin

pkg/distribution/internal/store/bundles.go

@@ -30,12 +30,13 @@ func (s *LocalStore) BundleForModel(ref string) (types.ModelBundle, error) {
 		return nil, fmt.Errorf("get model ID: %w", err)
 	}
 	path := s.bundlePath(dgst)
-	if bdl, err := bundle.Parse(path); err != nil {
+	bdl, err := bundle.Parse(path)
+	if err != nil {
 		// create for first time or replace bad/corrupted bundle
 		return s.createBundle(path, mdl)
-	} else {
-		return bdl, nil
 	}
+
+	return bdl, nil
 }
 
 // createBundle unpacks the bundle to path, replacing existing bundle if one is found

pkg/inference/scheduling/loader.go

@@ -436,6 +436,22 @@ func (l *loader) load(ctx context.Context, backendName, modelID, modelRef string
 		}
 	}
 
+	// If no explicit config exists, create a default one with the model's context size
+	// so that the OpenAI recorder can report the actual configuration being used.
+	if runnerConfig == nil {
+		defaultConfig := inference.BackendConfiguration{}
+		if l.modelManager != nil {
+			if bundle, err := l.modelManager.GetBundle(modelID); err != nil {
+				l.log.Warnf("Failed to get bundle for model %s to determine default context size: %v", modelID, err)
+			} else if runtimeConfig := bundle.RuntimeConfig(); runtimeConfig != nil {
+				if ctxSize := runtimeConfig.GetContextSize(); ctxSize != nil {
+					defaultConfig.ContextSize = ctxSize
+				}
+			}
+		}
+		runnerConfig = &defaultConfig
+	}
+
 	l.log.Infof("Loading %s backend runner with model %s in %s mode", backendName, modelID, mode)
 
 	// Acquire the loader lock and defer its release.