Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,17 @@ the fact.

## [Unreleased]

### Fixed

- Profiles for Qwen3 hybrid models got `--reasoning-parser deepseek_r1` from
the card's `vllm serve` line. With thinking turned off
(`enable_thinking: false`), a non-streaming reply then came back with
`content` empty and the answer in `reasoning`. When the card's SGLang
command names a different parser and the installed vLLM has one by that
name (Qwen3's cards name `qwen3`), that parser is used, and `inspect` and
`new` say so. Existing profiles keep their parser; change one with
`llm-helper set`.

## [0.6.0] - 2026-11-04

### Added
Expand Down
2 changes: 2 additions & 0 deletions docs/models.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ llm-helper reads a model's HuggingFace card before anything is saved: it checks
- `llm-helper inspect <org/model>` shows one repo's license, its weight size against this machine's budget, and the vLLM flags its card suggests — without pulling anything.

`search`, `inspect`, and `new` pre-fill known vLLM serving flags from the model card: tool call parsers, reasoning parsers, and context length.
When the card's SGLang command names a different reasoning parser than its `vllm serve` line, and the installed vLLM has a parser by that exact name, that one is used and `inspect` says so.
Qwen3's cards are the case this covers: their `vllm serve` line names `deepseek_r1`, which puts the whole answer in `reasoning` when a request turns thinking off.
`search` marks what is too big for this machine and what cannot load, and preselects the biggest quant that fits.
`new` refuses a quant that cannot load rather than serving it — activating a profile restarts the service, so a model that cannot load would take the running one down with it.

Expand Down
25 changes: 23 additions & 2 deletions internal/catalog/catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ type Detail struct {
// vLLM won't take, so it's visible before pulling rather than as a
// crash loop afterwards.
DroppedFlags []string `json:"dropped_flags,omitempty"`
FlagNotes []string `json:"flag_notes,omitempty"`
// KVBytesPerToken is what one token of context costs in KV cache, and
// MaxContext the longest the model was trained for. Both 0 when the
// config could not be read.
Expand Down Expand Up @@ -230,12 +231,27 @@ func Inspect(ctx context.Context, m hf.Model, budget detect.WeightsBudget, caps
if flags, err := hf.DetectFlags(ctx, m.ID); err == nil {
// Validate here too: seeing that a card's parser won't work on this
// machine is most useful before pulling tens of gigabytes.
d.DetectedFlags, d.DroppedFlags = vllm.Check(flags.ExtraArgs, caps)
args, note := cardFlags(flags, caps)
if note != "" {
d.FlagNotes = append(d.FlagNotes, note)
}
d.DetectedFlags, d.DroppedFlags = vllm.Check(args, caps)
d.MaxModelLen = flags.MaxModelLen
}
return d
}

// cardFlags returns the card's vllm serve flags, rebuilt around the parser its
// SGLang command names when this vLLM registers that name.
func cardFlags(flags hf.DetectedFlags, caps vllm.Caps) (args, note string) {
if flags.SGLangExtraArgs == "" || !caps.Known() || !caps.HasReasoningParser(flags.SGLangReasoningParser) {
return flags.ExtraArgs, ""
}
return flags.SGLangExtraArgs, fmt.Sprintf(
"using --reasoning-parser %s instead of the card's %s: the card's SGLang command names %s, and vLLM %s has it",
flags.SGLangReasoningParser, flags.ReasoningParser, flags.SGLangReasoningParser, caps.Version)
}

// ForProvider reshapes what Inspect found for the server that will load the
// model, since Inspect judges GGUF by vLLM's plugin. A server that picks its
// own GGUF quant has none of the plugin's limits, so split and subfolder
Expand All @@ -255,6 +271,7 @@ func ForProvider(d Detail, prov provider.Provider) Detail {
d.ArchBlocked = ""
d.DetectedFlags = ""
d.DroppedFlags = nil
d.FlagNotes = nil
if !caps.GGUFQuant && d.NeedsGGUF {
d.ArchBlocked = "only published as GGUF here, which " + prov.Name() + " does not load"
}
Expand Down Expand Up @@ -357,7 +374,11 @@ func PrepareProfile(ctx context.Context, dirs config.Dirs, repo string, choice *
// author prescribed, the other is what this tool added. Merging them
// before reporting attributed our own --tokenizer to the card, and said
// "flags auto-detected from README" about a repo whose README had none.
cardArgs, dropped := vllm.Check(detected.ExtraArgs, caps)
cardArgs, note := cardFlags(detected, caps)
if note != "" {
p.Notes = append(p.Notes, note)
}
cardArgs, dropped := vllm.Check(cardArgs, caps)
p.Notes = append(p.Notes, dropped...)

if patched, needsPatch, err := hf.DetectChatTemplatePatch(ctx, repo); err != nil {
Expand Down
37 changes: 37 additions & 0 deletions internal/catalog/catalog_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -763,3 +763,40 @@ func TestAPullWithoutAQuantChoiceStillMeasuresTheWeights(t *testing.T) {
t.Error("the pull planned a KV cache without ever measuring the model it has to fit around")
}
}

func TestCardFlagsPrefersARegisteredSGLangParser(t *testing.T) {
t.Parallel()
qwen := hf.DetectedFlags{
ExtraArgs: "--reasoning-parser deepseek_r1",
ReasoningParser: "deepseek_r1",
SGLangReasoningParser: "qwen3",
SGLangExtraArgs: "--reasoning-parser qwen3",
}
unregistered := qwen
unregistered.SGLangReasoningParser = "deepseek-r1"
unregistered.SGLangExtraArgs = "--reasoning-parser deepseek-r1"
tests := []struct {
name string
flags hf.DetectedFlags
caps vllm.Caps
want string
changed bool
}{
{"registered", qwen, spark, "--reasoning-parser qwen3", true},
{"not registered", unregistered, spark, "--reasoning-parser deepseek_r1", false},
{"caps unknown", qwen, vllm.Caps{}, "--reasoning-parser deepseek_r1", false},
{"parsers agree", hf.DetectedFlags{ExtraArgs: "--reasoning-parser qwen3", SGLangReasoningParser: "qwen3"}, spark, "--reasoning-parser qwen3", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got, note := cardFlags(tt.flags, tt.caps)
if got != tt.want {
t.Errorf("args = %q, want %q", got, tt.want)
}
if tt.changed != (note != "") {
t.Errorf("note = %q, want changed=%v", note, tt.changed)
}
})
}
}
3 changes: 3 additions & 0 deletions internal/cli/search.go
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,9 @@ func printModelDetail(d catalog.Detail, cfg *config.Config) {
} else {
fmt.Println(" vllm flags: no vllm serve snippet in README -- set flags by hand after pulling")
}
for _, n := range d.FlagNotes {
fmt.Printf(" changed: %s\n", n)
}
for _, w := range d.DroppedFlags {
fmt.Printf(" dropped: %s\n", w)
}
Expand Down
14 changes: 13 additions & 1 deletion internal/hf/hf.go
Original file line number Diff line number Diff line change
Expand Up @@ -690,6 +690,14 @@ func nextPageURL(header, want string) string {
type DetectedFlags struct {
ExtraArgs string // e.g. "--enable-auto-tool-choice --tool-call-parser qwen3_coder"
MaxModelLen string // from --max-model-len in the snippet, "" if absent
// SGLangReasoningParser is the --reasoning-parser the card's SGLang
// command names, "" if none or if its SGLang commands disagree.
SGLangReasoningParser string
// ReasoningParser is the --reasoning-parser the vllm serve line names.
ReasoningParser string
// SGLangExtraArgs is ExtraArgs rebuilt with SGLangReasoningParser, set
// only when the two parsers differ.
SGLangExtraArgs string
}

// DetectFlags fetches a model's README and pulls known vLLM serving flags
Expand Down Expand Up @@ -717,8 +725,12 @@ func DetectFlags(ctx context.Context, repo string) (DetectedFlags, error) {
// context clamp below -- those models are exactly the ones whose real
// context is far under the generic default.
snippetRe := regexp.MustCompile(`vllm serve(?:.*\\\n)*.*`)
d.SGLangReasoningParser = sglangReasoningParser(string(body))
if snippet := snippetRe.FindString(string(body)); snippet != "" {
d.ExtraArgs, d.MaxModelLen = flagsFromSnippet(snippet)
d.ExtraArgs, d.MaxModelLen, d.ReasoningParser = parseSnippet(snippet, "")
if d.ReasoningParser != "" && d.SGLangReasoningParser != "" && d.ReasoningParser != d.SGLangReasoningParser {
d.SGLangExtraArgs, _, _ = parseSnippet(snippet, d.SGLangReasoningParser)
}
}

// Model cards routinely paste a snippet from a bigger sibling model, so
Expand Down
22 changes: 22 additions & 0 deletions internal/hf/hf_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1138,3 +1138,25 @@ func TestGGUFServable(t *testing.T) {
t.Error("an unreadable config was treated as a refusal")
}
}

func TestDetectFlagsReadsTheSGLangParser(t *testing.T) {
serveTestdata(t, map[string]string{
"/" + repo + "/raw/main/README.md": "README_qwen3_hybrid.md",
})
d, err := DetectFlags(context.Background(), repo)
if err != nil {
t.Fatalf("DetectFlags: %v", err)
}
if d.ExtraArgs != "--enable-reasoning --reasoning-parser deepseek_r1" {
t.Errorf("ExtraArgs = %q", d.ExtraArgs)
}
if d.SGLangReasoningParser != "qwen3" {
t.Errorf("SGLangReasoningParser = %q, want qwen3", d.SGLangReasoningParser)
}
if d.ReasoningParser != "deepseek_r1" {
t.Errorf("ReasoningParser = %q, want deepseek_r1", d.ReasoningParser)
}
if d.SGLangExtraArgs != "--enable-reasoning --reasoning-parser qwen3" {
t.Errorf("SGLangExtraArgs = %q", d.SGLangExtraArgs)
}
}
49 changes: 47 additions & 2 deletions internal/hf/snippet.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,13 @@ var thinkingParsers = regexp.MustCompile(`^(deepseek_v3|poolside_v1)$`)
// name is real depends on the vLLM installed on this machine, which is not
// something a HuggingFace client should know about.
func flagsFromSnippet(snippet string) (extraArgs, maxModelLen string) {
extraArgs, maxModelLen, _ = parseSnippet(snippet, "")
return extraArgs, maxModelLen
}

// parseSnippet is flagsFromSnippet that also returns the --reasoning-parser it
// kept, replaced by parser when that is set.
func parseSnippet(snippet, parser string) (extraArgs, maxModelLen, reasoningParser string) {
// Fold shell line continuations, then flatten what is left. The backslash
// and the newline are separate concerns: a card can wrap without one, and
// a bare newline left in place gets read as the previous flag's value.
Expand All @@ -62,7 +69,7 @@ func flagsFromSnippet(snippet string) (extraArgs, maxModelLen string) {
if err != nil {
// A card whose snippet isn't valid shell (prose wrapped mid-line,
// an unbalanced quote) yields nothing rather than half of it.
return "", ""
return "", "", ""
}

var kept []string
Expand Down Expand Up @@ -92,6 +99,9 @@ func flagsFromSnippet(snippet string) (extraArgs, maxModelLen string) {
continue
}
if flag == "--reasoning-parser" {
if parser != "" {
value = parser
}
sawParser = value
}
if value == "" {
Expand All @@ -104,7 +114,42 @@ func flagsFromSnippet(snippet string) (extraArgs, maxModelLen string) {
if thinkingParsers.MatchString(sawParser) {
kept = append(kept, `--default-chat-template-kwargs {"thinking":true}`)
}
return strings.Join(kept, " "), maxModelLen
return strings.Join(kept, " "), maxModelLen, sawParser
}

var sglangSnippetRe = regexp.MustCompile(`sglang[._](?:launch_server|serve)(?:.*\\\n)*.*`)

func sglangReasoningParser(card string) string {
found := ""
for _, snippet := range sglangSnippetRe.FindAllString(card, -1) {
name := reasoningParserIn(snippet)
if name == "" {
continue
}
if found != "" && found != name {
return ""
}
found = name
}
return found
}

func reasoningParserIn(snippet string) string {
flat := strings.ReplaceAll(snippet, "\\\n", " ")
flat = strings.NewReplacer("\r", " ", "\n", " ", "\t", " ").Replace(flat)
args, err := config.SplitArgs(flat)
if err != nil {
return ""
}
for i, a := range args {
if v, ok := strings.CutPrefix(a, "--reasoning-parser="); ok {
return v
}
if a == "--reasoning-parser" && i+1 < len(args) && !isFlagToken(args[i+1]) {
return args[i+1]
}
}
return ""
}

// looksSecret keeps a credential pasted into a card out of a profile file.
Expand Down
42 changes: 42 additions & 0 deletions internal/hf/snippet_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,3 +93,45 @@ func TestFlagsFromSnippetUnparseable(t *testing.T) {
t.Errorf("got %q, want nothing from an unparseable snippet", got)
}
}

func TestParseSnippetReplacesTheParser(t *testing.T) {
t.Parallel()
tests := []struct {
name, snippet, parser, want string
}{
{"into deepseek_v3", `vllm serve org/m --reasoning-parser deepseek_r1`, "deepseek_v3", `--reasoning-parser deepseek_v3 --default-chat-template-kwargs {"thinking":true}`},
{"away from deepseek_v3", `vllm serve org/m --reasoning-parser deepseek_v3`, "qwen3", "--reasoning-parser qwen3"},
{"equals form", `vllm serve org/m --reasoning-parser=deepseek_r1 --enforce-eager`, "qwen3", "--reasoning-parser qwen3 --enforce-eager"},
{"no parser in the snippet", `vllm serve org/m --enforce-eager`, "qwen3", "--enforce-eager"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got, _, _ := parseSnippet(tt.snippet, tt.parser)
if got != tt.want {
t.Errorf("parseSnippet() = %q, want %q", got, tt.want)
}
})
}
}

func TestSGLangReasoningParser(t *testing.T) {
t.Parallel()
tests := []struct {
name, card, want string
}{
{"qwen3 hybrid card", "- SGLang:\n ```shell\n python -m sglang.launch_server --model-path Qwen/Qwen3-0.6B --reasoning-parser qwen3\n ```\n- vLLM:\n ```shell\n vllm serve Qwen/Qwen3-0.6B --enable-reasoning --reasoning-parser deepseek_r1\n ```\n", "qwen3"},
{"continued lines", "python3 -m sglang.launch_server \\\n --model-path ./m \\\n --reasoning-parser=deepseek-r1\n", "deepseek-r1"},
{"commands disagree", "sglang.launch_server --reasoning-parser qwen3\nsglang.launch_server --reasoning-parser deepseek-r1\n", ""},
{"no parser", "python -m sglang.launch_server --model-path m\n", ""},
{"no sglang command", "vllm serve m --reasoning-parser deepseek_r1\n", ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := sglangReasoningParser(tt.card); got != tt.want {
t.Errorf("sglangReasoningParser() = %q, want %q", got, tt.want)
}
})
}
}
9 changes: 9 additions & 0 deletions internal/hf/testdata/README_qwen3_hybrid.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
For deployment, you can use `sglang>=0.4.6.post1` or `vllm>=0.8.5` or to create an OpenAI-compatible API endpoint:
- SGLang:
```shell
python -m sglang.launch_server --model-path Qwen/Qwen3-0.6B --reasoning-parser qwen3
```
- vLLM:
```shell
vllm serve Qwen/Qwen3-0.6B --enable-reasoning --reasoning-parser deepseek_r1
```
5 changes: 4 additions & 1 deletion internal/web/static/models.js
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,10 @@ async function inspect(repo) {

el.flags.hidden = !d.detected_flags;
if (d.detected_flags) {
el.flags.textContent = `Serving flags found on the model card and accepted by your vLLM: ${d.detected_flags}`;
el.flags.textContent = [
`Serving flags found on the model card and accepted by your vLLM: ${d.detected_flags}`,
...(d.flag_notes ?? []),
].join("\n");
}
el.dropped.hidden = !d.dropped_flags?.length;
if (d.dropped_flags?.length) {
Expand Down
Loading