diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d11aaf..a5cc3c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,17 @@ the fact. ## [Unreleased] +### Fixed + +- Profiles for Qwen3 hybrid models got `--reasoning-parser deepseek_r1` from + the card's `vllm serve` line. With thinking turned off + (`enable_thinking: false`), a non-streaming reply then came back with + `content` empty and the answer in `reasoning`. When the card's SGLang + command names a different parser and the installed vLLM has one by that + name (Qwen3's cards name `qwen3`), that parser is used, and `inspect` and + `new` say so. Existing profiles keep their parser; change one with + `llm-helper set`. + ## [0.6.0] - 2026-11-04 ### Added diff --git a/docs/models.md b/docs/models.md index e0dae7f..842ffe2 100644 --- a/docs/models.md +++ b/docs/models.md @@ -12,6 +12,8 @@ llm-helper reads a model's HuggingFace card before anything is saved: it checks - `llm-helper inspect ` shows one repo's license, its weight size against this machine's budget, and the vLLM flags its card suggests — without pulling anything. `search`, `inspect`, and `new` pre-fill known vLLM serving flags from the model card: tool call parsers, reasoning parsers, and context length. +When the card's SGLang command names a different reasoning parser than its `vllm serve` line, and the installed vLLM has a parser by that exact name, that one is used and `inspect` says so. +Qwen3's cards are the case this covers: their `vllm serve` line names `deepseek_r1`, which puts the whole answer in `reasoning` when a request turns thinking off. `search` marks what is too big for this machine and what cannot load, and preselects the biggest quant that fits. `new` refuses a quant that cannot load rather than serving it — activating a profile restarts the service, so a model that cannot load would take the running one down with it. diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go index 2405840..1d56d1a 100644 --- a/internal/catalog/catalog.go +++ b/internal/catalog/catalog.go @@ -102,6 +102,7 @@ type Detail struct { // vLLM won't take, so it's visible before pulling rather than as a // crash loop afterwards. DroppedFlags []string `json:"dropped_flags,omitempty"` + FlagNotes []string `json:"flag_notes,omitempty"` // KVBytesPerToken is what one token of context costs in KV cache, and // MaxContext the longest the model was trained for. Both 0 when the // config could not be read. @@ -230,12 +231,27 @@ func Inspect(ctx context.Context, m hf.Model, budget detect.WeightsBudget, caps if flags, err := hf.DetectFlags(ctx, m.ID); err == nil { // Validate here too: seeing that a card's parser won't work on this // machine is most useful before pulling tens of gigabytes. - d.DetectedFlags, d.DroppedFlags = vllm.Check(flags.ExtraArgs, caps) + args, note := cardFlags(flags, caps) + if note != "" { + d.FlagNotes = append(d.FlagNotes, note) + } + d.DetectedFlags, d.DroppedFlags = vllm.Check(args, caps) d.MaxModelLen = flags.MaxModelLen } return d } +// cardFlags returns the card's vllm serve flags, rebuilt around the parser its +// SGLang command names when this vLLM registers that name. +func cardFlags(flags hf.DetectedFlags, caps vllm.Caps) (args, note string) { + if flags.SGLangExtraArgs == "" || !caps.Known() || !caps.HasReasoningParser(flags.SGLangReasoningParser) { + return flags.ExtraArgs, "" + } + return flags.SGLangExtraArgs, fmt.Sprintf( + "using --reasoning-parser %s instead of the card's %s: the card's SGLang command names %s, and vLLM %s has it", + flags.SGLangReasoningParser, flags.ReasoningParser, flags.SGLangReasoningParser, caps.Version) +} + // ForProvider reshapes what Inspect found for the server that will load the // model, since Inspect judges GGUF by vLLM's plugin. A server that picks its // own GGUF quant has none of the plugin's limits, so split and subfolder @@ -255,6 +271,7 @@ func ForProvider(d Detail, prov provider.Provider) Detail { d.ArchBlocked = "" d.DetectedFlags = "" d.DroppedFlags = nil + d.FlagNotes = nil if !caps.GGUFQuant && d.NeedsGGUF { d.ArchBlocked = "only published as GGUF here, which " + prov.Name() + " does not load" } @@ -357,7 +374,11 @@ func PrepareProfile(ctx context.Context, dirs config.Dirs, repo string, choice * // author prescribed, the other is what this tool added. Merging them // before reporting attributed our own --tokenizer to the card, and said // "flags auto-detected from README" about a repo whose README had none. - cardArgs, dropped := vllm.Check(detected.ExtraArgs, caps) + cardArgs, note := cardFlags(detected, caps) + if note != "" { + p.Notes = append(p.Notes, note) + } + cardArgs, dropped := vllm.Check(cardArgs, caps) p.Notes = append(p.Notes, dropped...) if patched, needsPatch, err := hf.DetectChatTemplatePatch(ctx, repo); err != nil { diff --git a/internal/catalog/catalog_internal_test.go b/internal/catalog/catalog_internal_test.go index 129382f..4e059cb 100644 --- a/internal/catalog/catalog_internal_test.go +++ b/internal/catalog/catalog_internal_test.go @@ -763,3 +763,40 @@ func TestAPullWithoutAQuantChoiceStillMeasuresTheWeights(t *testing.T) { t.Error("the pull planned a KV cache without ever measuring the model it has to fit around") } } + +func TestCardFlagsPrefersARegisteredSGLangParser(t *testing.T) { + t.Parallel() + qwen := hf.DetectedFlags{ + ExtraArgs: "--reasoning-parser deepseek_r1", + ReasoningParser: "deepseek_r1", + SGLangReasoningParser: "qwen3", + SGLangExtraArgs: "--reasoning-parser qwen3", + } + unregistered := qwen + unregistered.SGLangReasoningParser = "deepseek-r1" + unregistered.SGLangExtraArgs = "--reasoning-parser deepseek-r1" + tests := []struct { + name string + flags hf.DetectedFlags + caps vllm.Caps + want string + changed bool + }{ + {"registered", qwen, spark, "--reasoning-parser qwen3", true}, + {"not registered", unregistered, spark, "--reasoning-parser deepseek_r1", false}, + {"caps unknown", qwen, vllm.Caps{}, "--reasoning-parser deepseek_r1", false}, + {"parsers agree", hf.DetectedFlags{ExtraArgs: "--reasoning-parser qwen3", SGLangReasoningParser: "qwen3"}, spark, "--reasoning-parser qwen3", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, note := cardFlags(tt.flags, tt.caps) + if got != tt.want { + t.Errorf("args = %q, want %q", got, tt.want) + } + if tt.changed != (note != "") { + t.Errorf("note = %q, want changed=%v", note, tt.changed) + } + }) + } +} diff --git a/internal/cli/search.go b/internal/cli/search.go index 0284e6f..6804f3b 100644 --- a/internal/cli/search.go +++ b/internal/cli/search.go @@ -260,6 +260,9 @@ func printModelDetail(d catalog.Detail, cfg *config.Config) { } else { fmt.Println(" vllm flags: no vllm serve snippet in README -- set flags by hand after pulling") } + for _, n := range d.FlagNotes { + fmt.Printf(" changed: %s\n", n) + } for _, w := range d.DroppedFlags { fmt.Printf(" dropped: %s\n", w) } diff --git a/internal/hf/hf.go b/internal/hf/hf.go index 6644572..a5f897f 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -690,6 +690,14 @@ func nextPageURL(header, want string) string { type DetectedFlags struct { ExtraArgs string // e.g. "--enable-auto-tool-choice --tool-call-parser qwen3_coder" MaxModelLen string // from --max-model-len in the snippet, "" if absent + // SGLangReasoningParser is the --reasoning-parser the card's SGLang + // command names, "" if none or if its SGLang commands disagree. + SGLangReasoningParser string + // ReasoningParser is the --reasoning-parser the vllm serve line names. + ReasoningParser string + // SGLangExtraArgs is ExtraArgs rebuilt with SGLangReasoningParser, set + // only when the two parsers differ. + SGLangExtraArgs string } // DetectFlags fetches a model's README and pulls known vLLM serving flags @@ -717,8 +725,12 @@ func DetectFlags(ctx context.Context, repo string) (DetectedFlags, error) { // context clamp below -- those models are exactly the ones whose real // context is far under the generic default. snippetRe := regexp.MustCompile(`vllm serve(?:.*\\\n)*.*`) + d.SGLangReasoningParser = sglangReasoningParser(string(body)) if snippet := snippetRe.FindString(string(body)); snippet != "" { - d.ExtraArgs, d.MaxModelLen = flagsFromSnippet(snippet) + d.ExtraArgs, d.MaxModelLen, d.ReasoningParser = parseSnippet(snippet, "") + if d.ReasoningParser != "" && d.SGLangReasoningParser != "" && d.ReasoningParser != d.SGLangReasoningParser { + d.SGLangExtraArgs, _, _ = parseSnippet(snippet, d.SGLangReasoningParser) + } } // Model cards routinely paste a snippet from a bigger sibling model, so diff --git a/internal/hf/hf_internal_test.go b/internal/hf/hf_internal_test.go index 148f985..a5bc2da 100644 --- a/internal/hf/hf_internal_test.go +++ b/internal/hf/hf_internal_test.go @@ -1138,3 +1138,25 @@ func TestGGUFServable(t *testing.T) { t.Error("an unreadable config was treated as a refusal") } } + +func TestDetectFlagsReadsTheSGLangParser(t *testing.T) { + serveTestdata(t, map[string]string{ + "/" + repo + "/raw/main/README.md": "README_qwen3_hybrid.md", + }) + d, err := DetectFlags(context.Background(), repo) + if err != nil { + t.Fatalf("DetectFlags: %v", err) + } + if d.ExtraArgs != "--enable-reasoning --reasoning-parser deepseek_r1" { + t.Errorf("ExtraArgs = %q", d.ExtraArgs) + } + if d.SGLangReasoningParser != "qwen3" { + t.Errorf("SGLangReasoningParser = %q, want qwen3", d.SGLangReasoningParser) + } + if d.ReasoningParser != "deepseek_r1" { + t.Errorf("ReasoningParser = %q, want deepseek_r1", d.ReasoningParser) + } + if d.SGLangExtraArgs != "--enable-reasoning --reasoning-parser qwen3" { + t.Errorf("SGLangExtraArgs = %q", d.SGLangExtraArgs) + } +} diff --git a/internal/hf/snippet.go b/internal/hf/snippet.go index 1e32764..ea41475 100644 --- a/internal/hf/snippet.go +++ b/internal/hf/snippet.go @@ -53,6 +53,13 @@ var thinkingParsers = regexp.MustCompile(`^(deepseek_v3|poolside_v1)$`) // name is real depends on the vLLM installed on this machine, which is not // something a HuggingFace client should know about. func flagsFromSnippet(snippet string) (extraArgs, maxModelLen string) { + extraArgs, maxModelLen, _ = parseSnippet(snippet, "") + return extraArgs, maxModelLen +} + +// parseSnippet is flagsFromSnippet that also returns the --reasoning-parser it +// kept, replaced by parser when that is set. +func parseSnippet(snippet, parser string) (extraArgs, maxModelLen, reasoningParser string) { // Fold shell line continuations, then flatten what is left. The backslash // and the newline are separate concerns: a card can wrap without one, and // a bare newline left in place gets read as the previous flag's value. @@ -62,7 +69,7 @@ func flagsFromSnippet(snippet string) (extraArgs, maxModelLen string) { if err != nil { // A card whose snippet isn't valid shell (prose wrapped mid-line, // an unbalanced quote) yields nothing rather than half of it. - return "", "" + return "", "", "" } var kept []string @@ -92,6 +99,9 @@ func flagsFromSnippet(snippet string) (extraArgs, maxModelLen string) { continue } if flag == "--reasoning-parser" { + if parser != "" { + value = parser + } sawParser = value } if value == "" { @@ -104,7 +114,42 @@ func flagsFromSnippet(snippet string) (extraArgs, maxModelLen string) { if thinkingParsers.MatchString(sawParser) { kept = append(kept, `--default-chat-template-kwargs {"thinking":true}`) } - return strings.Join(kept, " "), maxModelLen + return strings.Join(kept, " "), maxModelLen, sawParser +} + +var sglangSnippetRe = regexp.MustCompile(`sglang[._](?:launch_server|serve)(?:.*\\\n)*.*`) + +func sglangReasoningParser(card string) string { + found := "" + for _, snippet := range sglangSnippetRe.FindAllString(card, -1) { + name := reasoningParserIn(snippet) + if name == "" { + continue + } + if found != "" && found != name { + return "" + } + found = name + } + return found +} + +func reasoningParserIn(snippet string) string { + flat := strings.ReplaceAll(snippet, "\\\n", " ") + flat = strings.NewReplacer("\r", " ", "\n", " ", "\t", " ").Replace(flat) + args, err := config.SplitArgs(flat) + if err != nil { + return "" + } + for i, a := range args { + if v, ok := strings.CutPrefix(a, "--reasoning-parser="); ok { + return v + } + if a == "--reasoning-parser" && i+1 < len(args) && !isFlagToken(args[i+1]) { + return args[i+1] + } + } + return "" } // looksSecret keeps a credential pasted into a card out of a profile file. diff --git a/internal/hf/snippet_internal_test.go b/internal/hf/snippet_internal_test.go index de15e59..286f803 100644 --- a/internal/hf/snippet_internal_test.go +++ b/internal/hf/snippet_internal_test.go @@ -93,3 +93,45 @@ func TestFlagsFromSnippetUnparseable(t *testing.T) { t.Errorf("got %q, want nothing from an unparseable snippet", got) } } + +func TestParseSnippetReplacesTheParser(t *testing.T) { + t.Parallel() + tests := []struct { + name, snippet, parser, want string + }{ + {"into deepseek_v3", `vllm serve org/m --reasoning-parser deepseek_r1`, "deepseek_v3", `--reasoning-parser deepseek_v3 --default-chat-template-kwargs {"thinking":true}`}, + {"away from deepseek_v3", `vllm serve org/m --reasoning-parser deepseek_v3`, "qwen3", "--reasoning-parser qwen3"}, + {"equals form", `vllm serve org/m --reasoning-parser=deepseek_r1 --enforce-eager`, "qwen3", "--reasoning-parser qwen3 --enforce-eager"}, + {"no parser in the snippet", `vllm serve org/m --enforce-eager`, "qwen3", "--enforce-eager"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, _, _ := parseSnippet(tt.snippet, tt.parser) + if got != tt.want { + t.Errorf("parseSnippet() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestSGLangReasoningParser(t *testing.T) { + t.Parallel() + tests := []struct { + name, card, want string + }{ + {"qwen3 hybrid card", "- SGLang:\n ```shell\n python -m sglang.launch_server --model-path Qwen/Qwen3-0.6B --reasoning-parser qwen3\n ```\n- vLLM:\n ```shell\n vllm serve Qwen/Qwen3-0.6B --enable-reasoning --reasoning-parser deepseek_r1\n ```\n", "qwen3"}, + {"continued lines", "python3 -m sglang.launch_server \\\n --model-path ./m \\\n --reasoning-parser=deepseek-r1\n", "deepseek-r1"}, + {"commands disagree", "sglang.launch_server --reasoning-parser qwen3\nsglang.launch_server --reasoning-parser deepseek-r1\n", ""}, + {"no parser", "python -m sglang.launch_server --model-path m\n", ""}, + {"no sglang command", "vllm serve m --reasoning-parser deepseek_r1\n", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := sglangReasoningParser(tt.card); got != tt.want { + t.Errorf("sglangReasoningParser() = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/internal/hf/testdata/README_qwen3_hybrid.md b/internal/hf/testdata/README_qwen3_hybrid.md new file mode 100644 index 0000000..244bc7b --- /dev/null +++ b/internal/hf/testdata/README_qwen3_hybrid.md @@ -0,0 +1,9 @@ +For deployment, you can use `sglang>=0.4.6.post1` or `vllm>=0.8.5` or to create an OpenAI-compatible API endpoint: +- SGLang: + ```shell + python -m sglang.launch_server --model-path Qwen/Qwen3-0.6B --reasoning-parser qwen3 + ``` +- vLLM: + ```shell + vllm serve Qwen/Qwen3-0.6B --enable-reasoning --reasoning-parser deepseek_r1 + ``` diff --git a/internal/web/static/models.js b/internal/web/static/models.js index 600b8fa..117a6a2 100644 --- a/internal/web/static/models.js +++ b/internal/web/static/models.js @@ -460,7 +460,10 @@ async function inspect(repo) { el.flags.hidden = !d.detected_flags; if (d.detected_flags) { - el.flags.textContent = `Serving flags found on the model card and accepted by your vLLM: ${d.detected_flags}`; + el.flags.textContent = [ + `Serving flags found on the model card and accepted by your vLLM: ${d.detected_flags}`, + ...(d.flag_notes ?? []), + ].join("\n"); } el.dropped.hidden = !d.dropped_flags?.length; if (d.dropped_flags?.length) {