From 60608c11eabc1922dde512b6b0110a536f0c3667 Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:43:01 +0200 Subject: [PATCH 01/19] Add host hardware discovery profile --- internal/hostprofile/profile.go | 371 ++++++++++++++++++++++++++++++++ 1 file changed, 371 insertions(+) create mode 100644 internal/hostprofile/profile.go diff --git a/internal/hostprofile/profile.go b/internal/hostprofile/profile.go new file mode 100644 index 0000000..fcbff1c --- /dev/null +++ b/internal/hostprofile/profile.go @@ -0,0 +1,371 @@ +package hostprofile + +import ( + "bufio" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "time" +) + +const SchemaVersion = "quantum.runtime/host-profile/v1alpha1" + +type Profile struct { + SchemaVersion string `json:"schema_version"` + CollectedAt time.Time `json:"collected_at"` + OS string `json:"os"` + Arch string `json:"arch"` + CPU CPUProfile `json:"cpu"` + NUMA NUMAProfile `json:"numa"` + Memory MemoryProfile `json:"memory"` + Storage []StorageProfile `json:"storage"` + Accelerators []AcceleratorProfile `json:"accelerators"` + Warnings []string `json:"warnings,omitempty"` +} + +type CPUProfile struct { + VendorID string `json:"vendor_id,omitempty"` + ModelName string `json:"model_name,omitempty"` + PhysicalCores int `json:"physical_cores"` + LogicalCores int `json:"logical_cores"` + ThreadsPerCore int `json:"threads_per_core"` + Features []string `json:"features,omitempty"` +} + +type NUMAProfile struct { + Count int `json:"count"` + Nodes []NUMANode `json:"nodes,omitempty"` +} + +type NUMANode struct { + ID int `json:"id"` + CPUList string `json:"cpu_list,omitempty"` + MemoryBytes uint64 `json:"memory_bytes,omitempty"` + FreeBytes uint64 `json:"free_bytes,omitempty"` +} + +type MemoryProfile struct { + TotalBytes uint64 `json:"total_bytes"` + AvailableBytes uint64 `json:"available_bytes"` + HugePageSizeBytes uint64 `json:"huge_page_size_bytes,omitempty"` + HugePagesTotal uint64 `json:"huge_pages_total,omitempty"` + HugePagesFree uint64 `json:"huge_pages_free,omitempty"` +} + +type StorageProfile struct { + Device string `json:"device"` + Kind string `json:"kind"` + SizeBytes uint64 `json:"size_bytes,omitempty"` + Rotational bool `json:"rotational"` + NUMANode *int `json:"numa_node,omitempty"` +} + +type AcceleratorProfile struct { + Kind string `json:"kind"` + Vendor string `json:"vendor"` + Device string `json:"device"` + VRAMBytes uint64 `json:"vram_bytes,omitempty"` + NUMANode *int `json:"numa_node,omitempty"` +} + +func Discover() Profile { + p := Profile{ + SchemaVersion: SchemaVersion, + CollectedAt: time.Now().UTC(), + OS: runtime.GOOS, + Arch: runtime.GOARCH, + CPU: CPUProfile{ + LogicalCores: runtime.NumCPU(), + }, + } + if err := discoverCPU(&p); err != nil { + p.Warnings = append(p.Warnings, "cpu: "+err.Error()) + } + if err := discoverMemory(&p); err != nil { + p.Warnings = append(p.Warnings, "memory: "+err.Error()) + } + if err := discoverNUMA(&p); err != nil { + p.Warnings = append(p.Warnings, "numa: "+err.Error()) + } + if err := discoverStorage(&p); err != nil { + p.Warnings = append(p.Warnings, "storage: "+err.Error()) + } + if err := discoverAccelerators(&p); err != nil { + p.Warnings = append(p.Warnings, "accelerators: "+err.Error()) + } + if p.CPU.PhysicalCores <= 0 { + p.CPU.PhysicalCores = p.CPU.LogicalCores + } + if p.CPU.PhysicalCores > 0 { + p.CPU.ThreadsPerCore = max(1, p.CPU.LogicalCores/p.CPU.PhysicalCores) + } + sort.Strings(p.CPU.Features) + return p +} + +func discoverCPU(p *Profile) error { + f, err := os.Open("/proc/cpuinfo") + if err != nil { + return err + } + defer f.Close() + + type coreKey struct{ socket, core string } + cores := map[coreKey]struct{}{} + var physicalID, coreID string + featureSet := map[string]struct{}{} + flushCore := func() { + if coreID != "" { + cores[coreKey{socket: physicalID, core: coreID}] = struct{}{} + } + physicalID, coreID = "", "" + } + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + flushCore() + continue + } + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + switch key { + case "vendor_id", "CPU implementer": + if p.CPU.VendorID == "" { + p.CPU.VendorID = value + } + case "model name", "Processor", "Hardware": + if p.CPU.ModelName == "" { + p.CPU.ModelName = value + } + case "physical id": + physicalID = value + case "core id": + coreID = value + case "flags", "Features": + for _, feature := range strings.Fields(value) { + featureSet[strings.ToLower(feature)] = struct{}{} + } + } + } + flushCore() + if err := scanner.Err(); err != nil { + return err + } + if len(cores) > 0 { + p.CPU.PhysicalCores = len(cores) + } + for feature := range featureSet { + p.CPU.Features = append(p.CPU.Features, feature) + } + return nil +} + +func discoverMemory(p *Profile) error { + values, err := parseMemInfo("/proc/meminfo") + if err != nil { + return err + } + p.Memory.TotalBytes = values["MemTotal"] * 1024 + p.Memory.AvailableBytes = values["MemAvailable"] * 1024 + if p.Memory.AvailableBytes == 0 { + p.Memory.AvailableBytes = (values["MemFree"] + values["Buffers"] + values["Cached"]) * 1024 + } + p.Memory.HugePageSizeBytes = values["Hugepagesize"] * 1024 + p.Memory.HugePagesTotal = values["HugePages_Total"] + p.Memory.HugePagesFree = values["HugePages_Free"] + return nil +} + +func parseMemInfo(path string) (map[string]uint64, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + values := map[string]uint64{} + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + key, rest, ok := strings.Cut(line, ":") + if !ok { + continue + } + fields := strings.Fields(rest) + if len(fields) == 0 { + continue + } + value, err := strconv.ParseUint(fields[0], 10, 64) + if err == nil { + values[key] = value + } + } + return values, scanner.Err() +} + +func discoverNUMA(p *Profile) error { + paths, err := filepath.Glob("/sys/devices/system/node/node[0-9]*") + if err != nil { + return err + } + for _, path := range paths { + base := filepath.Base(path) + id, err := strconv.Atoi(strings.TrimPrefix(base, "node")) + if err != nil { + continue + } + node := NUMANode{ID: id} + if data, err := os.ReadFile(filepath.Join(path, "cpulist")); err == nil { + node.CPUList = strings.TrimSpace(string(data)) + } + if values, err := parseNodeMemInfo(filepath.Join(path, "meminfo")); err == nil { + node.MemoryBytes = values["MemTotal"] * 1024 + node.FreeBytes = values["MemFree"] * 1024 + } + p.NUMA.Nodes = append(p.NUMA.Nodes, node) + } + sort.Slice(p.NUMA.Nodes, func(i, j int) bool { return p.NUMA.Nodes[i].ID < p.NUMA.Nodes[j].ID }) + p.NUMA.Count = len(p.NUMA.Nodes) + return nil +} + +func parseNodeMemInfo(path string) (map[string]uint64, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + values := map[string]uint64{} + scanner := bufio.NewScanner(f) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 4 { + continue + } + key := strings.TrimSuffix(fields[2], ":") + value, err := strconv.ParseUint(fields[3], 10, 64) + if err == nil { + values[key] = value + } + } + return values, scanner.Err() +} + +func discoverStorage(p *Profile) error { + devices, err := filepath.Glob("/sys/block/*") + if err != nil { + return err + } + for _, path := range devices { + name := filepath.Base(path) + if strings.HasPrefix(name, "loop") || strings.HasPrefix(name, "ram") || strings.HasPrefix(name, "fd") { + continue + } + storage := StorageProfile{Device: name, Kind: "block"} + if strings.HasPrefix(name, "nvme") { + storage.Kind = "nvme" + } + if data, err := os.ReadFile(filepath.Join(path, "size")); err == nil { + if sectors, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64); err == nil { + storage.SizeBytes = sectors * 512 + } + } + if data, err := os.ReadFile(filepath.Join(path, "queue", "rotational")); err == nil { + storage.Rotational = strings.TrimSpace(string(data)) == "1" + } + if data, err := os.ReadFile(filepath.Join(path, "device", "numa_node")); err == nil { + if value, err := strconv.Atoi(strings.TrimSpace(string(data))); err == nil && value >= 0 { + storage.NUMANode = &value + } + } + p.Storage = append(p.Storage, storage) + } + sort.Slice(p.Storage, func(i, j int) bool { return p.Storage[i].Device < p.Storage[j].Device }) + return nil +} + +func discoverAccelerators(p *Profile) error { + cards, err := filepath.Glob("/sys/class/drm/card[0-9]*") + if err != nil { + return err + } + seen := map[string]struct{}{} + for _, card := range cards { + devicePath := filepath.Join(card, "device") + resolved, err := filepath.EvalSymlinks(devicePath) + if err != nil { + continue + } + if _, ok := seen[resolved]; ok { + continue + } + seen[resolved] = struct{}{} + vendorBytes, err := os.ReadFile(filepath.Join(devicePath, "vendor")) + if err != nil { + continue + } + vendorID := strings.ToLower(strings.TrimSpace(string(vendorBytes))) + vendor, kind := acceleratorVendor(vendorID) + if vendor == "" { + continue + } + acc := AcceleratorProfile{Kind: kind, Vendor: vendor, Device: filepath.Base(card)} + if data, err := os.ReadFile(filepath.Join(devicePath, "mem_info_vram_total")); err == nil { + if value, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64); err == nil { + acc.VRAMBytes = value + } + } + if data, err := os.ReadFile(filepath.Join(devicePath, "numa_node")); err == nil { + if value, err := strconv.Atoi(strings.TrimSpace(string(data))); err == nil && value >= 0 { + acc.NUMANode = &value + } + } + p.Accelerators = append(p.Accelerators, acc) + } + return nil +} + +func acceleratorVendor(id string) (string, string) { + switch id { + case "0x1002": + return "AMD", "gpu" + case "0x10de": + return "NVIDIA", "gpu" + case "0x8086": + return "Intel", "gpu" + default: + return "", "" + } +} + +func (p Profile) Validate() error { + if p.SchemaVersion != SchemaVersion { + return fmt.Errorf("unexpected host profile schema %q", p.SchemaVersion) + } + if p.CPU.LogicalCores <= 0 { + return errors.New("logical CPU count must be positive") + } + if p.CPU.PhysicalCores <= 0 || p.CPU.PhysicalCores > p.CPU.LogicalCores { + return errors.New("physical CPU count is invalid") + } + if p.Memory.TotalBytes > 0 && p.Memory.AvailableBytes > p.Memory.TotalBytes { + return errors.New("available memory exceeds total memory") + } + return nil +} + +func HasFeature(p Profile, feature string) bool { + feature = strings.ToLower(strings.TrimSpace(feature)) + i := sort.SearchStrings(p.CPU.Features, feature) + return i < len(p.CPU.Features) && p.CPU.Features[i] == feature +} From 1e6f524641370ed4437152de7d4de5bf16e5563a Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:43:09 +0200 Subject: [PATCH 02/19] Test host profile discovery --- internal/hostprofile/profile_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 internal/hostprofile/profile_test.go diff --git a/internal/hostprofile/profile_test.go b/internal/hostprofile/profile_test.go new file mode 100644 index 0000000..0af2fe3 --- /dev/null +++ b/internal/hostprofile/profile_test.go @@ -0,0 +1,26 @@ +package hostprofile + +import "testing" + +func TestDiscoverReturnsValidProfile(t *testing.T) { + p := Discover() + if err := p.Validate(); err != nil { + t.Fatalf("profile invalid: %v", err) + } + if p.SchemaVersion != SchemaVersion { + t.Fatalf("schema = %q", p.SchemaVersion) + } + if p.CPU.LogicalCores < 1 { + t.Fatalf("logical cores = %d", p.CPU.LogicalCores) + } +} + +func TestHasFeatureUsesSortedFeatures(t *testing.T) { + p := Profile{CPU: CPUProfile{Features: []string{"avx2", "avx512f"}}} + if !HasFeature(p, "AVX2") { + t.Fatal("expected avx2") + } + if HasFeature(p, "sve") { + t.Fatal("unexpected sve") + } +} From 6be87f97830927a4deaa3ea090f7b934bd486c50 Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:43:25 +0200 Subject: [PATCH 03/19] Add bounded host calibration --- internal/calibration/calibration.go | 221 ++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 internal/calibration/calibration.go diff --git a/internal/calibration/calibration.go b/internal/calibration/calibration.go new file mode 100644 index 0000000..3918d4a --- /dev/null +++ b/internal/calibration/calibration.go @@ -0,0 +1,221 @@ +package calibration + +import ( + "context" + "runtime" + "sort" + "sync" + "time" + + "github.com/Starlight-Unit-Studio/Quantum-Runtime/internal/hostprofile" +) + +const SchemaVersion = "quantum.runtime/host-calibration/v1alpha1" + +type Sample struct { + Workers int `json:"workers"` + MemoryCopyBytesPerSec uint64 `json:"memory_copy_bytes_per_sec"` + MemoryReadBytesPerSec uint64 `json:"memory_read_bytes_per_sec"` +} + +type Result struct { + SchemaVersion string `json:"schema_version"` + MeasuredAt time.Time `json:"measured_at"` + DurationMillis int64 `json:"duration_ms"` + BestWorkers int `json:"best_workers"` + MemoryCopyBytesPerSec uint64 `json:"memory_copy_bytes_per_sec"` + MemoryReadBytesPerSec uint64 `json:"memory_read_bytes_per_sec"` + MemoryBandwidthClass string `json:"memory_bandwidth_class"` + Samples []Sample `json:"samples"` + Checksum uint64 `json:"checksum"` +} + +func Run(ctx context.Context, host hostprofile.Profile, budget time.Duration) Result { + if budget <= 0 { + budget = 250 * time.Millisecond + } + if budget > 2*time.Second { + budget = 2 * time.Second + } + physical := host.CPU.PhysicalCores + if physical <= 0 { + physical = runtime.NumCPU() + } + logical := host.CPU.LogicalCores + if logical <= 0 { + logical = runtime.NumCPU() + } + candidates := workerCandidates(physical, logical) + perSample := budget / time.Duration(len(candidates)) + if perSample < 20*time.Millisecond { + perSample = 20 * time.Millisecond + } + + started := time.Now() + result := Result{SchemaVersion: SchemaVersion, MeasuredAt: time.Now().UTC()} + var bestScore uint64 + for _, workers := range candidates { + copyRate := measureParallelCopy(ctx, workers, perSample/2) + readRate, checksum := measureParallelRead(ctx, workers, perSample/2) + result.Checksum += checksum + result.Samples = append(result.Samples, Sample{ + Workers: workers, + MemoryCopyBytesPerSec: copyRate, + MemoryReadBytesPerSec: readRate, + }) + score := min(copyRate, readRate) + if score > bestScore { + bestScore = score + result.BestWorkers = workers + result.MemoryCopyBytesPerSec = copyRate + result.MemoryReadBytesPerSec = readRate + } + if ctx.Err() != nil { + break + } + } + result.DurationMillis = time.Since(started).Milliseconds() + result.MemoryBandwidthClass = classifyBandwidth(max(result.MemoryCopyBytesPerSec, result.MemoryReadBytesPerSec)) + return result +} + +func workerCandidates(physical, logical int) []int { + if physical < 1 { + physical = 1 + } + if logical < physical { + logical = physical + } + set := map[int]struct{}{1: {}} + for _, n := range []int{2, 4, 8, 16, 32, 64} { + if n <= physical { + set[n] = struct{}{} + } + } + set[physical] = struct{}{} + if logical > physical { + set[logical] = struct{}{} + } + out := make([]int, 0, len(set)) + for n := range set { + out = append(out, n) + } + sort.Ints(out) + return out +} + +func measureParallelCopy(ctx context.Context, workers int, budget time.Duration) uint64 { + if budget <= 0 || workers < 1 { + return 0 + } + chunkSize := (32 << 20) / workers + if chunkSize < 256<<10 { + chunkSize = 256 << 10 + } + deadline := time.Now().Add(budget) + started := time.Now() + counts := make(chan uint64, workers) + var wg sync.WaitGroup + for worker := 0; worker < workers; worker++ { + wg.Add(1) + go func(seed byte) { + defer wg.Done() + src := make([]byte, chunkSize) + dst := make([]byte, chunkSize) + for i := range src { + src[i] = byte(i) ^ seed + } + var total uint64 + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + counts <- total + return + default: + } + copy(dst, src) + total += uint64(len(src)) + } + runtime.KeepAlive(dst) + counts <- total + }(byte(worker)) + } + wg.Wait() + close(counts) + var total uint64 + for n := range counts { + total += n + } + return bytesPerSecond(total, time.Since(started)) +} + +func measureParallelRead(ctx context.Context, workers int, budget time.Duration) (uint64, uint64) { + if budget <= 0 || workers < 1 { + return 0, 0 + } + chunkSize := (32 << 20) / workers + if chunkSize < 256<<10 { + chunkSize = 256 << 10 + } + deadline := time.Now().Add(budget) + started := time.Now() + type measurement struct{ bytes, sum uint64 } + counts := make(chan measurement, workers) + var wg sync.WaitGroup + for worker := 0; worker < workers; worker++ { + wg.Add(1) + go func(seed byte) { + defer wg.Done() + buf := make([]byte, chunkSize) + for i := range buf { + buf[i] = byte(i*31+7) ^ seed + } + var total, sum uint64 + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + counts <- measurement{bytes: total, sum: sum} + return + default: + } + for i := 0; i < len(buf); i += 64 { + sum += uint64(buf[i]) + } + total += uint64(len(buf)) + } + runtime.KeepAlive(sum) + counts <- measurement{bytes: total, sum: sum} + }(byte(worker)) + } + wg.Wait() + close(counts) + var total, sum uint64 + for n := range counts { + total += n.bytes + sum += n.sum + } + return bytesPerSecond(total, time.Since(started)), sum +} + +func bytesPerSecond(bytes uint64, elapsed time.Duration) uint64 { + if elapsed <= 0 { + return 0 + } + return uint64(float64(bytes) / elapsed.Seconds()) +} + +func classifyBandwidth(bytesPerSecond uint64) string { + const gib = uint64(1 << 30) + switch { + case bytesPerSecond >= 100*gib: + return "very_high" + case bytesPerSecond >= 50*gib: + return "high" + case bytesPerSecond >= 20*gib: + return "medium" + case bytesPerSecond > 0: + return "low" + default: + return "unknown" + } +} From 2bfeb362da1532ebdce65ee4c6ad70f1d035e879 Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:43:32 +0200 Subject: [PATCH 04/19] Test bounded calibration --- internal/calibration/calibration_test.go | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 internal/calibration/calibration_test.go diff --git a/internal/calibration/calibration_test.go b/internal/calibration/calibration_test.go new file mode 100644 index 0000000..da59071 --- /dev/null +++ b/internal/calibration/calibration_test.go @@ -0,0 +1,34 @@ +package calibration + +import ( + "context" + "testing" + "time" + + "github.com/Starlight-Unit-Studio/Quantum-Runtime/internal/hostprofile" +) + +func TestRunIsBoundedAndReturnsRates(t *testing.T) { + host := hostprofile.Profile{CPU: hostprofile.CPUProfile{PhysicalCores: 8, LogicalCores: 16}, Memory: hostprofile.MemoryProfile{AvailableBytes: 8 << 30}} + started := time.Now() + result := Run(context.Background(), host, 100*time.Millisecond) + if time.Since(started) > 2*time.Second { + t.Fatal("calibration exceeded bound") + } + if result.SchemaVersion != SchemaVersion { + t.Fatalf("schema = %q", result.SchemaVersion) + } + if result.MemoryCopyBytesPerSec == 0 || result.MemoryReadBytesPerSec == 0 { + t.Fatalf("rates missing: %+v", result) + } + if result.BestWorkers < 1 || len(result.Samples) < 1 { + t.Fatalf("worker calibration missing: %+v", result) + } +} + +func TestWorkerCandidatesIncludeSMTThreadCount(t *testing.T) { + got := workerCandidates(96, 192) + if got[len(got)-1] != 192 { + t.Fatalf("last candidate = %d, want 192", got[len(got)-1]) + } +} From c3a40ac504759e1b04853a0d45967727b4eb40d1 Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:43:51 +0200 Subject: [PATCH 05/19] Add CPU-first placement planner --- internal/placement/placement.go | 242 ++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 internal/placement/placement.go diff --git a/internal/placement/placement.go b/internal/placement/placement.go new file mode 100644 index 0000000..38e0e1c --- /dev/null +++ b/internal/placement/placement.go @@ -0,0 +1,242 @@ +package placement + +import ( + "errors" + "fmt" + + "github.com/Starlight-Unit-Studio/Quantum-Runtime/internal/calibration" + "github.com/Starlight-Unit-Studio/Quantum-Runtime/internal/hostprofile" +) + +const SchemaVersion = "quantum.runtime/placement-plan/v1alpha1" + +type Request struct { + ModelBytes uint64 `json:"model_bytes"` + MoEExpertBytes uint64 `json:"moe_expert_bytes,omitempty"` + KVCacheBytes uint64 `json:"kv_cache_bytes,omitempty"` + PrefixCacheBytes uint64 `json:"prefix_cache_bytes,omitempty"` + ProjectorBytes uint64 `json:"projector_bytes,omitempty"` + WorkspaceBytes uint64 `json:"workspace_bytes,omitempty"` + ColdBytes uint64 `json:"cold_bytes,omitempty"` + AllowAcceleration bool `json:"allow_acceleration,omitempty"` + AllowNVMeColdTier bool `json:"allow_nvme_cold_tier,omitempty"` + NVMeBudgetBytes uint64 `json:"nvme_budget_bytes,omitempty"` + ReserveRAMBytes uint64 `json:"reserve_ram_bytes,omitempty"` +} + +type Plan struct { + SchemaVersion string `json:"schema_version"` + Decision string `json:"decision"` + Reason string `json:"reason"` + CPUFirst bool `json:"cpu_first"` + RAM TierAllocation `json:"ram"` + VRAM TierAllocation `json:"vram"` + NVMe TierAllocation `json:"nvme"` + Classes []ClassPlacement `json:"classes"` + Requirements []string `json:"requires_backend_capabilities,omitempty"` + Warnings []string `json:"warnings,omitempty"` +} + +type TierAllocation struct { + AvailableBytes uint64 `json:"available_bytes"` + AssignedBytes uint64 `json:"assigned_bytes"` +} + +type ClassPlacement struct { + Class string `json:"class"` + Tier string `json:"tier"` + Bytes uint64 `json:"bytes"` +} + +func Build(host hostprofile.Profile, calibrationResult *calibration.Result, req Request) (Plan, error) { + if err := validateRequest(req); err != nil { + return Plan{}, err + } + plan := Plan{SchemaVersion: SchemaVersion, CPUFirst: true} + availableRAM := host.Memory.AvailableBytes + if availableRAM == 0 { + availableRAM = host.Memory.TotalBytes + } + reserve := req.ReserveRAMBytes + if reserve == 0 { + reserve = defaultReserve(host.Memory.TotalBytes) + } + usableRAM := uint64(0) + if availableRAM > reserve { + usableRAM = availableRAM - reserve + } + plan.RAM.AvailableBytes = usableRAM + + var bestVRAM uint64 + for _, acc := range host.Accelerators { + if acc.VRAMBytes > bestVRAM { + bestVRAM = acc.VRAMBytes + } + } + plan.VRAM.AvailableBytes = bestVRAM + if req.AllowNVMeColdTier { + plan.NVMe.AvailableBytes = req.NVMeBudgetBytes + } + + hot := req.ModelBytes + req.MoEExpertBytes + req.KVCacheBytes + req.PrefixCacheBytes + req.ProjectorBytes + req.WorkspaceBytes + if hot <= usableRAM { + plan.Decision = "cpu_only" + plan.Reason = "CPU-first hot working set fits in usable host RAM." + assignHotToRAM(&plan, req) + assignCold(&plan, req) + if calibrationResult != nil && calibrationResult.MemoryBandwidthClass == "low" { + plan.Warnings = append(plan.Warnings, "Host calibration reports low memory bandwidth; CPU-only capacity fit does not guarantee requested throughput.") + } + return plan, nil + } + + if req.AllowAcceleration && bestVRAM > 0 { + plan.Decision = "hybrid_candidate" + plan.Reason = "CPU-first working set exceeds usable RAM; an accelerator tier is available for a pre-activation hybrid candidate." + remainingVRAM := bestVRAM + remainingRAM := usableRAM + unplaced := false + + allocateWhole(&plan, "kv_cache", req.KVCacheBytes, &remainingVRAM, &remainingRAM, &unplaced) + allocateWhole(&plan, "projector", req.ProjectorBytes, &remainingVRAM, &remainingRAM, &unplaced) + allocateSplit(&plan, "model_weights", req.ModelBytes, &remainingVRAM, &remainingRAM, &unplaced) + allocateSplit(&plan, "moe_experts", req.MoEExpertBytes, &remainingVRAM, &remainingRAM, &unplaced) + allocateWhole(&plan, "prefix_cache", req.PrefixCacheBytes, &remainingVRAM, &remainingRAM, &unplaced) + allocateWhole(&plan, "workspace", req.WorkspaceBytes, &remainingVRAM, &remainingRAM, &unplaced) + + if unplaced { + plan.Decision = "capacity_exceeded" + plan.Reason = "The requested hot working set does not fit in known RAM/VRAM tiers without unsafe active-state disk spilling." + } + plan.Requirements = append(plan.Requirements, "hybrid_pre_activation_placement") + if req.MoEExpertBytes > 0 { + plan.Requirements = append(plan.Requirements, "moe_expert_offload") + } + if req.KVCacheBytes > 0 && plan.VRAM.AssignedBytes > 0 { + plan.Requirements = append(plan.Requirements, "kv_cache_placement") + } + assignCold(&plan, req) + return plan, nil + } + + plan.Decision = "capacity_exceeded" + plan.Reason = "CPU-first working set exceeds usable RAM and no compatible accelerator capacity was requested/observed." + assignUnplaced(&plan, req) + assignCold(&plan, req) + return plan, nil +} + +func validateRequest(req Request) error { + if req.ModelBytes == 0 { + return errors.New("model_bytes must be greater than zero") + } + if req.AllowNVMeColdTier && req.ColdBytes > 0 && req.NVMeBudgetBytes == 0 { + return errors.New("nvme_budget_bytes is required when the NVMe cold tier is enabled") + } + if req.ColdBytes > req.NVMeBudgetBytes && req.AllowNVMeColdTier { + return fmt.Errorf("cold_bytes (%d) exceed nvme_budget_bytes (%d)", req.ColdBytes, req.NVMeBudgetBytes) + } + return nil +} + +func defaultReserve(total uint64) uint64 { + const twoGiB = uint64(2 << 30) + if total == 0 { + return twoGiB + } + reserve := total / 5 + if reserve < twoGiB { + reserve = twoGiB + } + return reserve +} + +func allocateWhole(plan *Plan, class string, bytes uint64, vram, ram *uint64, unplaced *bool) { + if bytes == 0 { + return + } + if bytes <= *vram { + *vram -= bytes + plan.VRAM.AssignedBytes += bytes + plan.Classes = append(plan.Classes, ClassPlacement{Class: class, Tier: "vram", Bytes: bytes}) + return + } + if bytes <= *ram { + *ram -= bytes + plan.RAM.AssignedBytes += bytes + plan.Classes = append(plan.Classes, ClassPlacement{Class: class, Tier: "ram", Bytes: bytes}) + return + } + *unplaced = true + plan.Classes = append(plan.Classes, ClassPlacement{Class: class, Tier: "unplaced", Bytes: bytes}) +} + +func allocateSplit(plan *Plan, class string, bytes uint64, vram, ram *uint64, unplaced *bool) { + if bytes == 0 { + return + } + toVRAM := min(bytes, *vram) + if toVRAM > 0 { + *vram -= toVRAM + bytes -= toVRAM + plan.VRAM.AssignedBytes += toVRAM + plan.Classes = append(plan.Classes, ClassPlacement{Class: class, Tier: "vram", Bytes: toVRAM}) + } + toRAM := min(bytes, *ram) + if toRAM > 0 { + *ram -= toRAM + bytes -= toRAM + plan.RAM.AssignedBytes += toRAM + plan.Classes = append(plan.Classes, ClassPlacement{Class: class, Tier: "ram", Bytes: toRAM}) + } + if bytes > 0 { + *unplaced = true + plan.Classes = append(plan.Classes, ClassPlacement{Class: class, Tier: "unplaced", Bytes: bytes}) + } +} + +func assignHotToRAM(plan *Plan, req Request) { + for _, c := range []ClassPlacement{ + {Class: "model_weights", Tier: "ram", Bytes: req.ModelBytes}, + {Class: "moe_experts", Tier: "ram", Bytes: req.MoEExpertBytes}, + {Class: "kv_cache", Tier: "ram", Bytes: req.KVCacheBytes}, + {Class: "prefix_cache", Tier: "ram", Bytes: req.PrefixCacheBytes}, + {Class: "projector", Tier: "ram", Bytes: req.ProjectorBytes}, + {Class: "workspace", Tier: "ram", Bytes: req.WorkspaceBytes}, + } { + if c.Bytes == 0 { + continue + } + plan.Classes = append(plan.Classes, c) + plan.RAM.AssignedBytes += c.Bytes + } +} + +func assignUnplaced(plan *Plan, req Request) { + for _, c := range []ClassPlacement{ + {Class: "model_weights", Tier: "unplaced", Bytes: req.ModelBytes}, + {Class: "moe_experts", Tier: "unplaced", Bytes: req.MoEExpertBytes}, + {Class: "kv_cache", Tier: "unplaced", Bytes: req.KVCacheBytes}, + {Class: "prefix_cache", Tier: "unplaced", Bytes: req.PrefixCacheBytes}, + {Class: "projector", Tier: "unplaced", Bytes: req.ProjectorBytes}, + {Class: "workspace", Tier: "unplaced", Bytes: req.WorkspaceBytes}, + } { + if c.Bytes > 0 { + plan.Classes = append(plan.Classes, c) + } + } +} + +func assignCold(plan *Plan, req Request) { + if req.ColdBytes == 0 { + return + } + if req.AllowNVMeColdTier && req.ColdBytes <= req.NVMeBudgetBytes { + plan.Classes = append(plan.Classes, ClassPlacement{Class: "cold_weights_or_cache", Tier: "nvme", Bytes: req.ColdBytes}) + plan.NVMe.AssignedBytes += req.ColdBytes + plan.Requirements = append(plan.Requirements, "explicit_nvme_cold_tier") + return + } + plan.Classes = append(plan.Classes, ClassPlacement{Class: "cold_weights_or_cache", Tier: "unplaced", Bytes: req.ColdBytes}) + plan.Warnings = append(plan.Warnings, "Cold bytes were requested but the explicit NVMe tier is disabled or insufficient.") +} From c2ead9091fedee39e1e0045547828b40f7f1fe41 Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:44:03 +0200 Subject: [PATCH 06/19] Test CPU-first placement policy --- internal/placement/placement_test.go | 60 ++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 internal/placement/placement_test.go diff --git a/internal/placement/placement_test.go b/internal/placement/placement_test.go new file mode 100644 index 0000000..b3f8277 --- /dev/null +++ b/internal/placement/placement_test.go @@ -0,0 +1,60 @@ +package placement + +import ( + "testing" + + "github.com/Starlight-Unit-Studio/Quantum-Runtime/internal/hostprofile" +) + +func TestCPUOnlyPreferredWhenWorkingSetFits(t *testing.T) { + host := hostprofile.Profile{Memory: hostprofile.MemoryProfile{TotalBytes: 64 << 30, AvailableBytes: 60 << 30}, Accelerators: []hostprofile.AcceleratorProfile{{Kind: "gpu", Vendor: "AMD", VRAMBytes: 24 << 30}}} + plan, err := Build(host, nil, Request{ModelBytes: 24 << 30, KVCacheBytes: 2 << 30, AllowAcceleration: true}) + if err != nil { + t.Fatal(err) + } + if plan.Decision != "cpu_only" { + t.Fatalf("decision = %q", plan.Decision) + } + if plan.VRAM.AssignedBytes != 0 { + t.Fatalf("unexpected VRAM assignment = %d", plan.VRAM.AssignedBytes) + } +} + +func TestHybridCandidateOnlyWhenCPUDoesNotFit(t *testing.T) { + host := hostprofile.Profile{Memory: hostprofile.MemoryProfile{TotalBytes: 32 << 30, AvailableBytes: 28 << 30}, Accelerators: []hostprofile.AcceleratorProfile{{Kind: "gpu", Vendor: "NVIDIA", VRAMBytes: 24 << 30}}} + plan, err := Build(host, nil, Request{ModelBytes: 30 << 30, KVCacheBytes: 2 << 30, AllowAcceleration: true}) + if err != nil { + t.Fatal(err) + } + if plan.Decision != "hybrid_candidate" { + t.Fatalf("decision = %q, reason=%s", plan.Decision, plan.Reason) + } + if plan.VRAM.AssignedBytes == 0 || plan.RAM.AssignedBytes == 0 { + t.Fatalf("expected split placement: %+v", plan) + } +} + +func TestNVMeOnlyAcceptsExplicitColdTier(t *testing.T) { + host := hostprofile.Profile{Memory: hostprofile.MemoryProfile{TotalBytes: 64 << 30, AvailableBytes: 60 << 30}} + plan, err := Build(host, nil, Request{ModelBytes: 8 << 30, ColdBytes: 16 << 30, AllowNVMeColdTier: true, NVMeBudgetBytes: 32 << 30}) + if err != nil { + t.Fatal(err) + } + if plan.NVMe.AssignedBytes != 16<<30 { + t.Fatalf("nvme = %d", plan.NVMe.AssignedBytes) + } +} + +func TestHotStateIsNeverSpilledToDisk(t *testing.T) { + host := hostprofile.Profile{Memory: hostprofile.MemoryProfile{TotalBytes: 16 << 30, AvailableBytes: 12 << 30}} + plan, err := Build(host, nil, Request{ModelBytes: 20 << 30, AllowNVMeColdTier: true, NVMeBudgetBytes: 100 << 30}) + if err != nil { + t.Fatal(err) + } + if plan.Decision != "capacity_exceeded" { + t.Fatalf("decision = %q", plan.Decision) + } + if plan.NVMe.AssignedBytes != 0 { + t.Fatalf("hot state spilled to NVMe: %+v", plan) + } +} From 7411b1e4d54bacd404d45f4e19cd8124bd144326 Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:44:11 +0200 Subject: [PATCH 07/19] Add resource control manager --- internal/resourcecontrol/manager.go | 57 +++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 internal/resourcecontrol/manager.go diff --git a/internal/resourcecontrol/manager.go b/internal/resourcecontrol/manager.go new file mode 100644 index 0000000..56c686b --- /dev/null +++ b/internal/resourcecontrol/manager.go @@ -0,0 +1,57 @@ +package resourcecontrol + +import ( + "context" + "sync" + "time" + + "github.com/Starlight-Unit-Studio/Quantum-Runtime/internal/calibration" + "github.com/Starlight-Unit-Studio/Quantum-Runtime/internal/hostprofile" + "github.com/Starlight-Unit-Studio/Quantum-Runtime/internal/placement" +) + +type Manager struct { + calibrationMu sync.Mutex + stateMu sync.RWMutex + last *calibration.Result +} + +func New() *Manager { + return &Manager{} +} + +func (m *Manager) Host() hostprofile.Profile { + return hostprofile.Discover() +} + +func (m *Manager) Calibrate(ctx context.Context, budget time.Duration) (hostprofile.Profile, calibration.Result) { + m.calibrationMu.Lock() + defer m.calibrationMu.Unlock() + host := hostprofile.Discover() + result := calibration.Run(ctx, host, budget) + m.stateMu.Lock() + copy := result + m.last = © + m.stateMu.Unlock() + return host, result +} + +func (m *Manager) LastCalibration() (calibration.Result, bool) { + m.stateMu.RLock() + defer m.stateMu.RUnlock() + if m.last == nil { + return calibration.Result{}, false + } + return *m.last, true +} + +func (m *Manager) Plan(req placement.Request) (hostprofile.Profile, *calibration.Result, placement.Plan, error) { + host := hostprofile.Discover() + var last *calibration.Result + if result, ok := m.LastCalibration(); ok { + copy := result + last = © + } + plan, err := placement.Build(host, last, req) + return host, last, plan, err +} From 7a2da7b8d2e103ac6e6723611f2e64a66e87200b Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:44:18 +0200 Subject: [PATCH 08/19] Test resource control manager --- internal/resourcecontrol/manager_test.go | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 internal/resourcecontrol/manager_test.go diff --git a/internal/resourcecontrol/manager_test.go b/internal/resourcecontrol/manager_test.go new file mode 100644 index 0000000..cc5d737 --- /dev/null +++ b/internal/resourcecontrol/manager_test.go @@ -0,0 +1,29 @@ +package resourcecontrol + +import ( + "context" + "testing" + "time" + + "github.com/Starlight-Unit-Studio/Quantum-Runtime/internal/placement" +) + +func TestCalibrationIsStored(t *testing.T) { + m := New() + _, result := m.Calibrate(context.Background(), 50*time.Millisecond) + got, ok := m.LastCalibration() + if !ok { + t.Fatal("calibration not stored") + } + if got.MeasuredAt != result.MeasuredAt { + t.Fatal("stored calibration differs") + } +} + +func TestPlanRejectsMissingModelSize(t *testing.T) { + m := New() + _, _, _, err := m.Plan(placement.Request{}) + if err == nil { + t.Fatal("expected validation error") + } +} From 1cbbbd75dbc6a488b0995d88a7403421b718add3 Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:44:30 +0200 Subject: [PATCH 09/19] Expose host calibration and placement APIs --- internal/httpapi/resource_routes.go | 132 ++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 internal/httpapi/resource_routes.go diff --git a/internal/httpapi/resource_routes.go b/internal/httpapi/resource_routes.go new file mode 100644 index 0000000..bf054b5 --- /dev/null +++ b/internal/httpapi/resource_routes.go @@ -0,0 +1,132 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "time" + + "github.com/Starlight-Unit-Studio/Quantum-Runtime/internal/calibration" + "github.com/Starlight-Unit-Studio/Quantum-Runtime/internal/hostprofile" + "github.com/Starlight-Unit-Studio/Quantum-Runtime/internal/placement" + "github.com/Starlight-Unit-Studio/Quantum-Runtime/internal/resourcecontrol" +) + +const hostCalibrationBudget = 400 * time.Millisecond + +var runtimeResources = resourcecontrol.New() + +func (s *Server) withResourceRoutes(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/host": + s.handleHostProfile(w, r) + return + case "/v1/host/calibrate": + s.handleHostCalibration(w, r) + return + case "/v1/placement": + s.handlePlacement(w, r) + return + default: + next.ServeHTTP(w, r) + } + }) +} + +func (s *Server) handleHostProfile(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + s.writeMethodNotAllowed(w, r, http.MethodGet) + return + } + if !s.authorized(r) { + s.writeUnauthorized(w) + return + } + profile := runtimeResources.Host() + if err := profile.Validate(); err != nil { + s.logger.Error("invalid host profile", "error", err) + s.writeError(w, http.StatusInternalServerError, "invalid_host_profile", "The discovered host profile is invalid.") + return + } + last, calibrated := runtimeResources.LastCalibration() + s.writeJSON(w, http.StatusOK, map[string]any{ + "api_version": "v1alpha1", + "host_schema": hostprofile.SchemaVersion, + "calibration_schema": calibration.SchemaVersion, + "host": profile, + "calibrated": calibrated, + "last_calibration": calibrationOrNil(last, calibrated), + }) +} + +func (s *Server) handleHostCalibration(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + s.writeMethodNotAllowed(w, r, http.MethodPost) + return + } + if !s.authorized(r) { + s.writeUnauthorized(w) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second) + defer cancel() + host, result := runtimeResources.Calibrate(ctx, hostCalibrationBudget) + if err := host.Validate(); err != nil { + s.logger.Error("invalid host profile after calibration", "error", err) + s.writeError(w, http.StatusInternalServerError, "invalid_host_profile", "The discovered host profile is invalid.") + return + } + s.writeJSON(w, http.StatusOK, map[string]any{ + "api_version": "v1alpha1", + "host": host, + "calibration": result, + }) +} + +func (s *Server) handlePlacement(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + s.writeMethodNotAllowed(w, r, http.MethodPost) + return + } + if !s.authorized(r) { + s.writeUnauthorized(w) + return + } + if r.ContentLength > s.config.RequestBodyLimit { + s.writeError(w, http.StatusRequestEntityTooLarge, "request_too_large", "The request body exceeds the configured limit.") + return + } + r.Body = http.MaxBytesReader(w, r.Body, s.config.RequestBodyLimit) + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + var request placement.Request + if err := decoder.Decode(&request); err != nil { + s.writeError(w, http.StatusBadRequest, "invalid_placement_request", "The placement request must be valid JSON using the v1alpha1 placement contract.") + return + } + host, last, plan, err := runtimeResources.Plan(request) + if err != nil { + s.writeError(w, http.StatusBadRequest, "invalid_placement_request", err.Error()) + return + } + if err := host.Validate(); err != nil { + s.logger.Error("invalid host profile during placement", "error", err) + s.writeError(w, http.StatusInternalServerError, "invalid_host_profile", "The discovered host profile is invalid.") + return + } + s.writeJSON(w, http.StatusOK, map[string]any{ + "api_version": "v1alpha1", + "placement_schema": placement.SchemaVersion, + "host": host, + "calibration": last, + "plan": plan, + }) +} + +func calibrationOrNil(value calibration.Result, ok bool) any { + if !ok { + return nil + } + return value +} From 079ccf9e554aae1a9aa744c90590979814b0d299 Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:44:43 +0200 Subject: [PATCH 10/19] Wire resource routes into native API --- internal/httpapi/registry_routes.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/httpapi/registry_routes.go b/internal/httpapi/registry_routes.go index bd4c40f..a543793 100644 --- a/internal/httpapi/registry_routes.go +++ b/internal/httpapi/registry_routes.go @@ -11,7 +11,7 @@ import ( var builtinModelRegistry = modelregistry.MustBuiltin() func (s *Server) withRegistryRoutes(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + registry := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.URL.Path == "/v1/models": s.handleModelList(w, r) @@ -23,6 +23,7 @@ func (s *Server) withRegistryRoutes(next http.Handler) http.Handler { next.ServeHTTP(w, r) } }) + return s.withResourceRoutes(registry) } func (s *Server) handleModelList(w http.ResponseWriter, r *http.Request) { From 8cbc1b0940997ca74aca0c0c28808814b2a3bddc Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:44:59 +0200 Subject: [PATCH 11/19] Test resource APIs --- internal/httpapi/resource_routes_test.go | 76 ++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 internal/httpapi/resource_routes_test.go diff --git a/internal/httpapi/resource_routes_test.go b/internal/httpapi/resource_routes_test.go new file mode 100644 index 0000000..a6c4af1 --- /dev/null +++ b/internal/httpapi/resource_routes_test.go @@ -0,0 +1,76 @@ +package httpapi + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Starlight-Unit-Studio/Quantum-Runtime/internal/resourcecontrol" +) + +func TestHostProfileEndpoint(t *testing.T) { + runtimeResources = resourcecontrol.New() + server := New(testConfig(t), &fakeUpstream{}, testBuild(), discardLogger()) + request := httptest.NewRequest(http.MethodGet, "/v1/host", nil) + response := httptest.NewRecorder() + + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("unexpected status: %d body=%s", response.Code, response.Body.String()) + } + var payload map[string]any + if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil { + t.Fatal(err) + } + if payload["host_schema"] != "quantum.runtime/host-profile/v1alpha1" { + t.Fatalf("unexpected host schema: %#v", payload["host_schema"]) + } +} + +func TestHostCalibrationEndpoint(t *testing.T) { + runtimeResources = resourcecontrol.New() + server := New(testConfig(t), &fakeUpstream{}, testBuild(), discardLogger()) + request := httptest.NewRequest(http.MethodPost, "/v1/host/calibrate", nil) + response := httptest.NewRecorder() + + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("unexpected status: %d body=%s", response.Code, response.Body.String()) + } + if !strings.Contains(response.Body.String(), "memory_bandwidth_class") || !strings.Contains(response.Body.String(), "best_workers") { + t.Fatalf("calibration missing: %s", response.Body.String()) + } +} + +func TestPlacementEndpointPrefersCPUFirst(t *testing.T) { + runtimeResources = resourcecontrol.New() + server := New(testConfig(t), &fakeUpstream{}, testBuild(), discardLogger()) + request := httptest.NewRequest(http.MethodPost, "/v1/placement", strings.NewReader(`{"model_bytes":1048576,"allow_acceleration":true}`)) + response := httptest.NewRecorder() + + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("unexpected status: %d body=%s", response.Code, response.Body.String()) + } + if !strings.Contains(response.Body.String(), `"cpu_first":true`) || !strings.Contains(response.Body.String(), `"decision":"cpu_only"`) { + t.Fatalf("CPU-first placement missing: %s", response.Body.String()) + } +} + +func TestPlacementEndpointRejectsUnknownFields(t *testing.T) { + runtimeResources = resourcecontrol.New() + server := New(testConfig(t), &fakeUpstream{}, testBuild(), discardLogger()) + request := httptest.NewRequest(http.MethodPost, "/v1/placement", strings.NewReader(`{"model_bytes":1048576,"magic_gpu":true}`)) + response := httptest.NewRecorder() + + server.Handler().ServeHTTP(response, request) + + if response.Code != http.StatusBadRequest { + t.Fatalf("unexpected status: %d body=%s", response.Code, response.Body.String()) + } +} From 359196b71c8e7c7bccf7132e5c7aa866a80c0761 Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:45:34 +0200 Subject: [PATCH 12/19] Bump Runtime to 0.3.0-alpha.3 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6906e9e..72e5e28 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0-alpha.2 +0.3.0-alpha.3 From 4299d45d719dca17cd19c1c278aa9197b81ab67e Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:45:45 +0200 Subject: [PATCH 13/19] Align build metadata with alpha.3 --- internal/buildinfo/buildinfo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/buildinfo/buildinfo.go b/internal/buildinfo/buildinfo.go index db16d96..4483c6c 100644 --- a/internal/buildinfo/buildinfo.go +++ b/internal/buildinfo/buildinfo.go @@ -2,7 +2,7 @@ package buildinfo // Values may be replaced with -ldflags during release builds. var ( - Version = "0.3.0-alpha.2" + Version = "0.3.0-alpha.3" Commit = "development" BuildDate = "unknown" ) From f728c5b3643db3397750a5428b2574c51a2cee67 Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:46:02 +0200 Subject: [PATCH 14/19] Document CPU-first hardware placement foundation --- docs/HARDWARE-PLACEMENT.md | 107 +++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 docs/HARDWARE-PLACEMENT.md diff --git a/docs/HARDWARE-PLACEMENT.md b/docs/HARDWARE-PLACEMENT.md new file mode 100644 index 0000000..2a49821 --- /dev/null +++ b/docs/HARDWARE-PLACEMENT.md @@ -0,0 +1,107 @@ +# Hardware discovery, calibration and CPU-first placement + +Status: `v1alpha1` foundation introduced in `0.3.0-alpha.3`. + +Quantum Runtime treats CPU/RAM as the mandatory baseline and GPUs as optional accelerators. Placement is decided before model activation; this layer does not migrate arbitrary live inference state between tiers. + +## Host profile + +`GET /v1/host` reports the current host using `quantum.runtime/host-profile/v1alpha1`. + +The Linux discovery path currently records: + +- OS and architecture +- CPU vendor/model, physical and logical CPU counts, threads per core and OS-visible instruction features +- NUMA node count, CPU lists and node memory where exposed by sysfs +- total/available RAM and huge-page metadata +- block devices, NVMe classification, rotational flag, capacity and NUMA locality where exposed +- DRM-visible AMD/NVIDIA/Intel accelerator devices, optional VRAM total and NUMA locality where exposed + +Discovery is fail-soft: unavailable optional kernel/sysfs data becomes a warning instead of being guessed. CPU and memory invariants still validate fail-closed before the profile is returned as usable. + +No hardware name alone selects an optimized kernel. A Turin-class EPYC host is useful evidence, but Runtime must confirm OS-visible features and later real-model conformance results. + +## Bounded calibration + +`POST /v1/host/calibrate` runs a short bounded memory copy/read calibration and records a worker-count sweep across representative CPU counts, including physical-core and SMT/logical-CPU candidates where present. + +The initial calibration is intentionally lightweight. It provides: + +- per-worker memory copy throughput +- per-worker memory read throughput +- the best observed worker count for the synthetic memory test +- a coarse memory-bandwidth class + +This is not yet a model tokens-per-second benchmark and must not be presented as one. It is a host-resource signal used to avoid naive assumptions such as "all logical CPUs are always faster". Real prefill/decode calibration remains tied to backend/model/quantization/context profiles in later slices. + +Calibration is explicit rather than automatic at daemon startup so a bounded synthetic benchmark does not unexpectedly consume host resources during service activation. + +## Placement plan + +`POST /v1/placement` accepts an explicit capacity request using separate memory classes: + +- `model_bytes` +- `moe_expert_bytes` +- `kv_cache_bytes` +- `prefix_cache_bytes` +- `projector_bytes` +- `workspace_bytes` +- `cold_bytes` + +The first planner follows these rules: + +1. Reserve host RAM for the OS/operator workload. +2. Build a CPU-only candidate first. +3. If the hot working set fits usable RAM, choose `cpu_only` even when a GPU exists. +4. Only when CPU-only capacity does not fit and acceleration is explicitly allowed may the planner build a `hybrid_candidate`. +5. Model weights and MoE expert weights may be split across VRAM/RAM as a candidate; active caches/workspaces remain whole-tier allocations in this slice. +6. NVMe is an explicit secondary tier only for operator-declared cold bytes. Hot model execution state is never silently spilled to disk. +7. If known RAM/VRAM capacity cannot hold hot state, return `capacity_exceeded` instead of relying on uncontrolled OS swap. + +A `hybrid_candidate` is not yet a claim that GPU execution is faster. Runtime must still require backend capability support and measured performance before such a placement can become the preferred production plan. + +## API examples + +Inspect the host: + +```bash +curl -s http://127.0.0.1:11450/v1/host +``` + +Run bounded calibration: + +```bash +curl -s -X POST http://127.0.0.1:11450/v1/host/calibrate +``` + +Ask for a CPU-first capacity plan: + +```bash +curl -s -X POST http://127.0.0.1:11450/v1/placement \ + -H 'Content-Type: application/json' \ + -d '{ + "model_bytes": 25769803776, + "moe_expert_bytes": 8589934592, + "kv_cache_bytes": 2147483648, + "allow_acceleration": true + }' +``` + +## Current boundaries + +`0.3.0-alpha.3` does not yet: + +- launch or reconfigure llama-server from a placement plan +- set NUMA affinity or thread pinning automatically +- estimate model/KV sizes from GGUF metadata automatically +- benchmark real model prefill/decode throughput +- persist host calibration across daemon restarts +- measure NVMe throughput or enforce a disk cache directory/quota/journal +- query vendor APIs for dynamic free VRAM +- guarantee hybrid execution is faster than CPU-only + +Those boundaries are deliberate. The purpose of this slice is to establish truthful host/resource contracts before backend-specific activation and real-model auto-tuning are layered on top. + +## Turin reference track + +AMD EPYC Turin remains the Tier-1 CPU-only reference target from issue #12. The generic API contains no AMD-only fields. Turin-specific quality comes from feeding its OS-visible CPU features, NUMA topology, memory capacity/bandwidth and later real-model benchmark results through the same generic host/calibration/placement contracts. From cf48055197ca4d0ac95d0cfa4a84dfd39f62102f Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:48:20 +0200 Subject: [PATCH 15/19] Bound calibration allocation under race instrumentation --- internal/calibration/calibration.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/calibration/calibration.go b/internal/calibration/calibration.go index 3918d4a..c3b8b38 100644 --- a/internal/calibration/calibration.go +++ b/internal/calibration/calibration.go @@ -108,9 +108,9 @@ func measureParallelCopy(ctx context.Context, workers int, budget time.Duration) if budget <= 0 || workers < 1 { return 0 } - chunkSize := (32 << 20) / workers - if chunkSize < 256<<10 { - chunkSize = 256 << 10 + chunkSize := (8 << 20) / workers + if chunkSize < 64<<10 { + chunkSize = 64 << 10 } deadline := time.Now().Add(budget) started := time.Now() @@ -153,9 +153,9 @@ func measureParallelRead(ctx context.Context, workers int, budget time.Duration) if budget <= 0 || workers < 1 { return 0, 0 } - chunkSize := (32 << 20) / workers - if chunkSize < 256<<10 { - chunkSize = 256 << 10 + chunkSize := (8 << 20) / workers + if chunkSize < 64<<10 { + chunkSize = 64 << 10 } deadline := time.Now().Add(budget) started := time.Now() From 5a3beddb25eb15140d749ee6c6250075958da89b Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:48:57 +0200 Subject: [PATCH 16/19] Make calibration bound race-CI safe --- internal/calibration/calibration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/calibration/calibration_test.go b/internal/calibration/calibration_test.go index da59071..34acc02 100644 --- a/internal/calibration/calibration_test.go +++ b/internal/calibration/calibration_test.go @@ -12,8 +12,8 @@ func TestRunIsBoundedAndReturnsRates(t *testing.T) { host := hostprofile.Profile{CPU: hostprofile.CPUProfile{PhysicalCores: 8, LogicalCores: 16}, Memory: hostprofile.MemoryProfile{AvailableBytes: 8 << 30}} started := time.Now() result := Run(context.Background(), host, 100*time.Millisecond) - if time.Since(started) > 2*time.Second { - t.Fatal("calibration exceeded bound") + if time.Since(started) > 5*time.Second { + t.Fatal("calibration exceeded CI-safe bound") } if result.SchemaVersion != SchemaVersion { t.Fatalf("schema = %q", result.SchemaVersion) From 295af92ae87ec6d68d03cdda6d1f5559669a8a48 Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:50:46 +0200 Subject: [PATCH 17/19] Start calibration timing after buffer setup --- internal/calibration/calibration.go | 43 +++++++++++++++++++---------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/internal/calibration/calibration.go b/internal/calibration/calibration.go index c3b8b38..934a3ce 100644 --- a/internal/calibration/calibration.go +++ b/internal/calibration/calibration.go @@ -112,19 +112,26 @@ func measureParallelCopy(ctx context.Context, workers int, budget time.Duration) if chunkSize < 64<<10 { chunkSize = 64 << 10 } - deadline := time.Now().Add(budget) + + sources := make([][]byte, workers) + destinations := make([][]byte, workers) + for worker := 0; worker < workers; worker++ { + sources[worker] = make([]byte, chunkSize) + destinations[worker] = make([]byte, chunkSize) + seed := byte(worker) + for i := range sources[worker] { + sources[worker][i] = byte(i) ^ seed + } + } + started := time.Now() + deadline := started.Add(budget) counts := make(chan uint64, workers) var wg sync.WaitGroup for worker := 0; worker < workers; worker++ { wg.Add(1) - go func(seed byte) { + go func(src, dst []byte) { defer wg.Done() - src := make([]byte, chunkSize) - dst := make([]byte, chunkSize) - for i := range src { - src[i] = byte(i) ^ seed - } var total uint64 for time.Now().Before(deadline) { select { @@ -138,7 +145,7 @@ func measureParallelCopy(ctx context.Context, workers int, budget time.Duration) } runtime.KeepAlive(dst) counts <- total - }(byte(worker)) + }(sources[worker], destinations[worker]) } wg.Wait() close(counts) @@ -157,19 +164,25 @@ func measureParallelRead(ctx context.Context, workers int, budget time.Duration) if chunkSize < 64<<10 { chunkSize = 64 << 10 } - deadline := time.Now().Add(budget) + + buffers := make([][]byte, workers) + for worker := 0; worker < workers; worker++ { + buffers[worker] = make([]byte, chunkSize) + seed := byte(worker) + for i := range buffers[worker] { + buffers[worker][i] = byte(i*31+7) ^ seed + } + } + started := time.Now() + deadline := started.Add(budget) type measurement struct{ bytes, sum uint64 } counts := make(chan measurement, workers) var wg sync.WaitGroup for worker := 0; worker < workers; worker++ { wg.Add(1) - go func(seed byte) { + go func(buf []byte) { defer wg.Done() - buf := make([]byte, chunkSize) - for i := range buf { - buf[i] = byte(i*31+7) ^ seed - } var total, sum uint64 for time.Now().Before(deadline) { select { @@ -185,7 +198,7 @@ func measureParallelRead(ctx context.Context, workers int, budget time.Duration) } runtime.KeepAlive(sum) counts <- measurement{bytes: total, sum: sum} - }(byte(worker)) + }(buffers[worker]) } wg.Wait() close(counts) From 32ecdef65bea9f421af035f89e07a0774fbeb127 Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:51:46 +0200 Subject: [PATCH 18/19] Document Runtime 0.3.0-alpha.3 resource foundation --- README.md | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4779ecc..5ea1d82 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Quantum Runtime is the independent AI runtime and model-service project of Starlight Unit Studios. -Current version: `0.3.0-alpha.2` +Current version: `0.3.0-alpha.3` ## Current alpha scope @@ -13,6 +13,7 @@ Quantum Runtime 0.3 adds the first model- and engine-neutral backend capability 1. the inference service endpoint used by Ember CoreUI, STΛRLIGHT UNIT The Game and later Quantum CoreOS 2. the versioned model identity and read-only registry contract 3. a non-destructive Linux service package that can be installed beside an existing Ollama service +4. host-resource discovery, bounded calibration and a CPU-first placement contract for RAM/VRAM/NVMe candidates Quantum Runtime now has two local execution paths. Ollama remains the default adoption/fallback mode, while a direct llama.cpp path can talk to `llama-server` without an Ollama daemon in the inference request path: @@ -41,6 +42,10 @@ The initial llama.cpp adapter deliberately uses the external `llama-server` proc - first direct llama.cpp/ggml backend path using `llama-server`, with Ollama absent from the request path when selected - Ollama chat/generate/embed compatibility translation to llama.cpp OpenAI-style endpoints - fail-closed handling for llama.cpp features not yet normalized: vision, tools, reasoning control and structured output +- Linux host discovery for CPU topology/features, NUMA, RAM/huge pages, block/NVMe devices and visible accelerator/VRAM metadata +- explicit bounded memory-bandwidth/worker-count calibration rather than automatic startup benchmarking +- versioned CPU-first placement plans that prefer RAM/CPU when the hot set fits and only consider hybrid acceleration after CPU capacity fails +- explicit NVMe cold-tier placement; hot execution state is never silently spilled to disk - streamed forwarding/translation, cancellation, body limits and backend timeouts - loopback-only default on `127.0.0.1:11450` - bearer-token requirement for any non-loopback bind @@ -62,7 +67,7 @@ The initial llama.cpp adapter deliberately uses the external `llama-server` proc go run ./cmd/quantum-runtime ``` -Check the service and registry: +Check the service, registry and host-resource contracts: ```bash curl -s http://127.0.0.1:11450/healthz @@ -72,6 +77,8 @@ curl -s http://127.0.0.1:11450/v1/models curl -s http://127.0.0.1:11450/v1/backends curl -s http://127.0.0.1:11450/v1/model-policies curl -s http://127.0.0.1:11450/v1/upstreams +curl -s http://127.0.0.1:11450/v1/host +curl -s -X POST http://127.0.0.1:11450/v1/host/calibrate ``` Run all verification: @@ -115,7 +122,7 @@ The manifest separates canonical model identity from aliases, source revision, b ## Direct llama.cpp backend -`0.3.0-alpha.2` can use an already running `llama-server` directly. The Runtime does not download, replace or manage the llama.cpp binary or GGUF files in this slice. Configure the server with an API-visible alias that matches the model identifier used by the client, for example: +`0.3.0-alpha.2` and later can use an already running `llama-server` directly. The Runtime does not download, replace or manage the llama.cpp binary or GGUF files in this slice. Configure the server with an API-visible alias that matches the model identifier used by the client, for example: ```bash llama-server -m /path/model.gguf --host 127.0.0.1 --port 8080 --alias ember-coreui:latest @@ -132,6 +139,14 @@ go run ./cmd/quantum-runtime If the llama.cpp server uses `--api-key`, set `QUANTUM_RUNTIME_LLAMA_CPP_API_KEY` locally. Runtime bearer credentials are never reused as llama.cpp credentials. The initial bridge supports text chat, text generation, embeddings and model-read compatibility. Unsupported modalities/capabilities fail closed instead of being silently dropped. +## CPU-first host placement + +`0.3.0-alpha.3` adds the first generic hardware/resource contract. `GET /v1/host` reports OS-visible CPU, NUMA, RAM, storage and accelerator metadata. `POST /v1/host/calibrate` runs an explicit bounded synthetic memory/worker sweep. `POST /v1/placement` creates a pre-activation capacity plan for model weights, MoE experts, caches, projectors, workspace and an optional explicit cold NVMe tier. + +CPU/RAM remains the baseline. A visible GPU does not automatically win: if the hot working set fits usable RAM, the first planner returns `cpu_only`. Hybrid placement is only considered after CPU-only capacity fails and acceleration was explicitly allowed. NVMe is never used as an implicit escape hatch for hot state. + +This is a truthful capacity/host foundation, not yet a claim of measured model throughput. Real backend/model prefill/decode calibration, NUMA affinity, thread pinning, GGUF size estimation and backend activation from placement plans remain later Runtime 0.3 slices. See `docs/HARDWARE-PLACEMENT.md`. + ## Ember CoreUI adoption Ember CoreUI remains an independent Repack. Quantum Runtime is optional and Quantum CoreOS is not required. @@ -186,7 +201,8 @@ The process rejects a network-wide bind when no bearer token is configured. ```text Quantum Runtime - model identity, lifecycle, inference APIs, streaming, backend adapters + model identity, lifecycle, inference APIs, streaming, backend adapters, + host discovery, calibration and placement policy Quantum Control server, hosting, service, database, backup and update management @@ -201,6 +217,7 @@ Quantum CoreOS consumes released Quantum Runtime packages rather than maintainin - `docs/ARCHITECTURE.md` - `docs/API.md` +- `docs/HARDWARE-PLACEMENT.md` - `docs/INSTALLATION.md` - `docs/ROADMAP.md` - `docs/SECURITY.md` @@ -213,4 +230,4 @@ Quantum Runtime project-owned code is licensed under the **Starlight Unit Studio Private/internal use, internal commercial use and Integrated Application Use are permitted under the license conditions. Quantum Runtime itself may not be sold, white-labeled or offered as a paid standalone general-purpose Runtime service. Third-party inference engines, models, model weights, datasets and tools retain their own terms. -The legally controlling German text is `LICENSE.de.md`; `LICENSE.md` is an English convenience translation. This is a custom Source Available license and is not an OSI-approved open-source license. +The legally controlling German text is `LICENSE.de.md`; `LICENSE.md` is an English convenience translation. This is a custom Source Available license and is not an OSI-approved open-source license. \ No newline at end of file From 717e023c41ecb5b690a996af28d37736e9acb21c Mon Sep 17 00:00:00 2001 From: Starlight Unit Studios Date: Sat, 5 Sep 2026 23:52:14 +0200 Subject: [PATCH 19/19] Add 0.3.0-alpha.3 changelog --- CHANGELOG.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4570331..d6ea829 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 0.3.0-alpha.3 - 2026-09-05 + +- Added `quantum.runtime/host-profile/v1alpha1` Linux discovery for CPU vendor/model/features, physical/logical CPUs, SMT ratio, NUMA nodes, RAM/huge-page metadata, block/NVMe devices and visible AMD/NVIDIA/Intel accelerator metadata. +- Added explicit bounded host calibration with representative physical-core and logical/SMT worker candidates, recording synthetic memory copy/read throughput and the best observed worker count without benchmarking automatically at daemon startup. +- Added `quantum.runtime/placement-plan/v1alpha1` with separate model-weight, MoE-expert, KV-cache, prefix-cache, projector, workspace and cold-tier memory classes. +- Implemented the first CPU-first placement policy: RAM/CPU is evaluated first and remains selected when the hot working set fits, even when an accelerator is visible. +- Added pre-activation hybrid candidates only after CPU-only capacity fails and acceleration is explicitly allowed; model/MoE weights may split across RAM/VRAM while cache/workspace classes remain whole-tier in this slice. +- Added an explicit NVMe cold tier. Hot execution state is never silently spilled to disk; insufficient RAM/VRAM capacity returns `capacity_exceeded` instead of relying on uncontrolled swap. +- Added `/v1/host`, `/v1/host/calibrate` and `/v1/placement` plus an in-memory resource-control boundary for the last bounded calibration. +- Added `docs/HARDWARE-PLACEMENT.md` documenting the generic CPU-first contract, current scope limits and the AMD EPYC Turin Tier-1 reference track. +- Kept real-model prefill/decode benchmarking, NUMA/thread affinity, GGUF/KV size estimation, persistent calibration and automatic backend activation out of this slice until they can be validated independently. + ## 0.3.0-alpha.2 - 2026-09-05 - Added the first direct llama.cpp/ggml execution adapter using the external `llama-server` process boundary; selecting it removes Ollama from the inference request path without vendoring upstream code. @@ -12,7 +24,6 @@ - Added optional separate llama-server API-key configuration. Runtime bearer credentials are never forwarded as llama.cpp credentials. - Updated the upstream ledger from planned to implemented-but-unpinned protocol evidence. No llama.cpp tag/commit is promoted to latest-known-good until real model/hardware conformance passes. - ## 0.3.0-alpha.1 - 2026-09-05 - Added `quantum.runtime/backend/v1alpha1` with explicit supported, unsupported, conditional and unknown capability states instead of optimistic booleans. @@ -129,4 +140,4 @@ Initial Quantum Runtime foundation. - Quantum native chat API - OpenAI-compatible API - GPU scheduling -- Quantum CoreOS TCI runtime integration +- Quantum CoreOS TCI runtime integration \ No newline at end of file