Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,8 @@ Start the whole fleet with `spinloop up` or just one with `spinloop fleet start

> In this screenshot, five nodes are configured and four are up, each serving the same
> `Qwen3.8-27B` model through llama.cpp. Three of them — `dev-2`, `dev-3`,
> `dev-4` — are mid-request: one slot running apiece, GPU util between 91% and
> 98%. `dev-1` finished about a minute ago, so its GPU util has dropped to 0%
> `dev-4` — are mid-request: one slot running apiece, GPU between 91% and
> 98%. `dev-1` finished about a minute ago, so its GPU has dropped to 0%
> while GPU memory stays at 89% — the weights are still loaded, and the next
> request it takes starts generating without a reload.
>
Expand Down
6 changes: 3 additions & 3 deletions cmd/spinloop/cloud_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -897,14 +897,14 @@ func TestCloudMetrics_BarFormat(t *testing.T) {
if !strings.Contains(out, "RAM") {
t.Errorf("bar output missing RAM bar:\n%s", out)
}
if !strings.Contains(out, "GPU util") {
t.Errorf("bar output missing GPU util bar:\n%s", out)
if !strings.Contains(out, " GPU ") {
t.Errorf("bar output missing GPU bar:\n%s", out)
}
if !strings.Contains(out, "GPU mem") {
t.Errorf("bar output missing GPU mem bar:\n%s", out)
}
if !strings.Contains(out, "85%") {
t.Errorf("bar output missing GPU utilization percentage:\n%s", out)
t.Errorf("bar output missing GPU utilisation percentage:\n%s", out)
}
if !strings.Contains(out, "running:") {
t.Errorf("bar output missing token stats:\n%s", out)
Expand Down
6 changes: 3 additions & 3 deletions cmd/spinloop/dashboard_render.go
Original file line number Diff line number Diff line change
Expand Up @@ -465,12 +465,12 @@ func dashTileReportBody(w io.Writer, m metrics.Stats, resources bool, gauge bool
// after it — one line, from the same read, whatever the engine's state. A
// read without either — a local node, an unkept or lapsed environment —
// draws nothing.
renderActiveIndented(w, m.LastActiveAt, m.IdleSeconds, m.RetainUntil, now)
renderActiveIndented(w, m.LastActiveAt, m.IdleSeconds, m.RetainUntil, now, statLabelWidth(m.GPUs, m.History))
if resources {
if gauge {
renderStatGauges(w, m.CPU, m.Memory, m.GPUs)
renderStatGauges(w, m.CPU, m.Memory, m.GPUs, m.Speed, m.History)
} else {
renderStatBars(w, m.CPU, m.Memory, m.GPUs, m.History, lineW)
renderStatBars(w, m.CPU, m.Memory, m.GPUs, m.Speed, m.History, lineW)
}
renderTokenLines(w, m.Tokens)
}
Expand Down
7 changes: 4 additions & 3 deletions cmd/spinloop/fleet.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,27 +160,28 @@ func renderFleetMetrics(w io.Writer, results []fleet.NodeResult, format string)
if format == "table" {
renderActiveKeyValue(w, stats.LastActiveAt, stats.IdleSeconds, retainUntilOf(r), now)
} else {
renderActiveIndented(w, stats.LastActiveAt, stats.IdleSeconds, retainUntilOf(r), now)
renderActiveIndented(w, stats.LastActiveAt, stats.IdleSeconds, retainUntilOf(r), now, statLabelWidth(stats.GPUs, stats.History))
}
switch format {
case "bar":
// No state gate, for the same reason the cloud bar format has
// none: a stopped node's retained history says what its engine
// was doing until it stopped, and a stopped node's current
// reading carries no figures for it to fall back on.
renderStatBars(w, stats.CPU, stats.Memory, stats.GPUs, stats.History, barLineW)
renderStatBars(w, stats.CPU, stats.Memory, stats.GPUs, stats.Speed, stats.History, barLineW)
renderTokenLines(w, stats.Tokens)
case "gauge":
if stats.State != "running" {
continue
}
renderStatGauges(w, stats.CPU, stats.Memory, stats.GPUs)
renderStatGauges(w, stats.CPU, stats.Memory, stats.GPUs, stats.Speed, stats.History)
renderTokenLines(w, stats.Tokens)
default:
if stats.State != "running" {
continue
}
renderTokenLines(w, stats.Tokens)
renderSpeedTable(w, stats.Speed, stats.History)
renderGPUTable(w, stats.GPUs)
renderCPUMemTable(w, stats.CPU, stats.Memory)
}
Expand Down
14 changes: 7 additions & 7 deletions cmd/spinloop/fleet_dashboard_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ func dashBar(label string, pct float64) string {
colour = "\033[33m"
}
filled := int(pct / 100 * width)
return fmt.Sprintf(" %-9s ", label) + colour +
return fmt.Sprintf(" %-7s ", label) + colour +
strings.Repeat("█", filled) + "\033[0m" +
strings.Repeat("░", width-filled) + fmt.Sprintf(" %.0f%%", pct)
}
Expand Down Expand Up @@ -355,10 +355,10 @@ func TestDashTileRunningByteStable(t *testing.T) {
want := dashTileExpected([]string{
dashExpectedHeader("up running (up 2h 0m 0s)", dashHealthy),
"llamacpp org/qwen:q4",
" active 12s ago",
" active 12s ago",
dashBar("CPU", 42),
dashBar("RAM", 30),
dashBar("GPU util", 61),
dashBar("GPU", 61),
dashBar("GPU mem", 50),
"",
" running: 2",
Expand Down Expand Up @@ -390,10 +390,10 @@ func TestDashTileRunningNoRequestCount(t *testing.T) {
want := dashTileExpected([]string{
dashExpectedHeader("up running (up 2h 0m 0s)", dashHealthy),
"llamacpp org/qwen:q4",
" active 12s ago",
" active 12s ago",
dashBar("CPU", 42),
dashBar("RAM", 30),
dashBar("GPU util", 61),
dashBar("GPU", 61),
dashBar("GPU mem", 50),
"",
" running: 2",
Expand Down Expand Up @@ -551,7 +551,7 @@ func TestDashTileActionInFlightWithReport(t *testing.T) {
"vllm org/qwen3:32b",
dashBar("CPU", 12),
dashBar("RAM", 48),
dashBar("GPU util", 35),
dashBar("GPU", 35),
dashBar("GPU mem", 45),
"", "", "", "",
})
Expand Down Expand Up @@ -3558,7 +3558,7 @@ func TestDashModelFormatToggle(t *testing.T) {
// blank columns, ending on the last retained sample — the current reading
// is the next point, not in the history yet. Both samples sit below the
// thresholds, so neither takes a colour.
cpuLine := " CPU " + strings.Repeat(" ", 23) + "▁▂" + " 20%"
cpuLine := " CPU " + strings.Repeat(" ", dashBarLineW-2) + "▁▂" + " 20%"
if !strings.Contains(bar, cpuLine) {
t.Errorf("bar tile did not draw the history, want %q in:\n%s", cpuLine, bar)
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/spinloop/fleet_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ func TestCmdFleetMetricsDrawsHistory(t *testing.T) {
t.Errorf("up's CPU gauge drew history or missed the current reading:\n%s", out)
}
halted := out[strings.Index(out, "halted"):]
if strings.Contains(halted, "GPU util") || strings.Contains(halted, " 60%") {
if strings.Contains(halted, "GPU") || strings.Contains(halted, " 60%") {
t.Errorf("a stopped node drew series in the gauge format:\n%s", halted)
}

Expand All @@ -291,7 +291,7 @@ func TestCmdFleetMetricsDrawsHistory(t *testing.T) {
if !strings.Contains(out, "░") {
t.Errorf("up's RAM did not fall back to the gauge:\n%s", out)
}
if !strings.Contains(out, "halted stopped org/qwen") || !strings.Contains(out, "GPU util") {
if !strings.Contains(out, "halted stopped org/qwen") || !strings.Contains(out, " GPU ") {
t.Errorf("halted's history not drawn:\n%s", out)
}
}
Expand Down
10 changes: 5 additions & 5 deletions cmd/spinloop/last_active_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func TestCloudMetricsBarShowsLastActive(t *testing.T) {
t.Fatalf("cmdMetrics: %v", err)
}
})
if !strings.Contains(out, "active 2m 5s ago") {
if !strings.Contains(out, "active 2m 5s ago") {
t.Errorf("bar format missing the active line:\n%s", out)
}
// It belongs between the header and the bars: a fact about the endpoint,
Expand Down Expand Up @@ -127,7 +127,7 @@ func TestCloudMetricsStoppedStillShowsLastActive(t *testing.T) {
// The bar format indents the line to the bar-label column; the table
// format makes it a key-value row. Same fact, each format's own idiom.
for format, want := range map[string]string{
"bar": "active 1h 0m 0s ago",
"bar": "active 1h 0m 0s ago",
"table": "active: 1h 0m 0s ago",
} {
t.Run(format, func(t *testing.T) {
Expand Down Expand Up @@ -157,7 +157,7 @@ func TestLastActiveZeroIdleStillRenders(t *testing.T) {
}`)

for format, want := range map[string]string{
"bar": "active 0s ago",
"bar": "active 0s ago",
"table": "active: 0s ago",
} {
t.Run(format, func(t *testing.T) {
Expand Down Expand Up @@ -305,7 +305,7 @@ func TestFleetMetricsShowsLastActive(t *testing.T) {
})
// bar indents its active line under the header; table spells it
// as a key-value, like the facts around it.
want := "active 2m 5s ago"
want := "active 2m 5s ago"
if format == "table" {
want = "active: 2m 5s ago"
}
Expand Down Expand Up @@ -379,7 +379,7 @@ func TestFleetMetricsStoppedNodeStillShowsLastActive(t *testing.T) {
t.Fatalf("cmdFleet metrics: %v", err)
}
})
if !strings.Contains(out, "active 10m 0s ago") {
if !strings.Contains(out, "active 10m 0s ago") {
t.Errorf("a stopped node dropped its active figure:\n%s", out)
}
}
Loading
Loading