From 4e8976c754d621f063d385ef6c0ae89342ce2f6b Mon Sep 17 00:00:00 2001 From: Graffioh Date: Tue, 13 Oct 2026 08:06:48 +0000 Subject: [PATCH 1/5] fix(ggml-cuda): take the transpose copy only into a contiguous destination ggml_cuda_cpy chose the tiled transpose kernel from the source layout alone. That kernel writes the destination densely and ignores its strides, so a contiguous [1, n, m] source (ne00 == 1 also gives nb01 == element size) copied into a strided view landed in the wrong elements and left the right ones unwritten. The PFlash Qwen3.5 scorer hits exactly this when its chunked logits copy ends on a one-key-wide chunk (S % key_chunk == 1, e.g. a 65,537-token prompt): the unwritten memory held NaN and the request failed with non-finite scores. Require a contiguous src1 for the transpose path, as upstream llama.cpp does (134b2bb756, ggml-org/llama.cpp#27663). test_cpy_strided_dst runs the scorer's chunked-copy pattern and the bare [1, n, m] copy on the GPU backend against a host reference and the CPU backend, for key counts with remainders 0, 1, 2 and chunk-1, and keeps the legitimate transpose fast path covered. Co-Authored-By: Claude Opus 5.5 --- server/CMakeLists.txt | 6 + .../deps/llama.cpp/ggml/src/ggml-cuda/cpy.cu | 7 +- server/test/unit/test_cpy_strided_dst.cpp | 200 ++++++++++++++++++ 3 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 server/test/unit/test_cpy_strided_dst.cpp diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index d79af1092..f323d3fba 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -1606,6 +1606,12 @@ if(LUCE_TESTS) SKIP_RETURN_CODE 77 ENVIRONMENT "GGML_CUDA_DISABLE_COPY_BATCH=1") endif() + # GPU F32 copy into a strided view (the PFlash scorer's chunked logits + # copy; a one-key-wide last chunk used to take the dense transpose + # kernel). CUDA and HIP; exits 77 without a device. + luce_add_ggml_gpu_executable(test_cpy_strided_dst test/unit/test_cpy_strided_dst.cpp) + target_link_libraries(test_cpy_strided_dst PRIVATE ggml-cpu) + list(APPEND _raw_unit_test_targets test_cpy_strided_dst) if(LUCE_GPU_BACKEND STREQUAL "hip") add_executable(test_ggml_host_mailbox_cuda test/unit/test_ggml_host_mailbox_cuda.cpp) set_source_files_properties(test/unit/test_ggml_host_mailbox_cuda.cpp PROPERTIES LANGUAGE HIP) diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/cpy.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/cpy.cu index e3586763b..c0dccc97f 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/cpy.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/cpy.cu @@ -625,8 +625,13 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg char * src1_ddc = (char *) src1->data; const bool contiguous_srcs = ggml_is_contiguous(src0) && ggml_is_contiguous(src1); + // The tiled transpose kernel writes dst densely (dst[imat*n + row*ne00 + col]) and ignores + // src1's strides, so it is only valid for a contiguous destination. Without this guard a + // contiguous [1, n, m] source (ne00 == 1 also gives nb01 == element size) copied into a + // strided view lands in the wrong elements and leaves the right ones unwritten. const bool can_be_transposed = nb01 == (int64_t)ggml_element_size(src0) && - src0->ne[3] == 1 && nb02 == ne00 * ne01 * (int64_t)ggml_element_size(src0); + src0->ne[3] == 1 && nb02 == ne00 * ne01 * (int64_t)ggml_element_size(src0) && + ggml_is_contiguous(src1); if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q8_1) { // The q8_1 rows MMVQ reads, from the kernel it quantizes its own diff --git a/server/test/unit/test_cpy_strided_dst.cpp b/server/test/unit/test_cpy_strided_dst.cpp new file mode 100644 index 000000000..1b532693f --- /dev/null +++ b/server/test/unit/test_cpy_strided_dst.cpp @@ -0,0 +1,200 @@ +// ggml-cuda/HIP F32 copy into a strided destination view. +// +// The PFlash Qwen3.5 scorer writes key logits chunk by chunk into a +// [S, n, H] buffer: ggml_cpy(scores[nk, n, H], view(logits, nk, n, H, offset b)). +// When S % key_chunk == 1 the last chunk is one key wide. A contiguous +// [1, n, H] source has nb01 == element size, so ggml-cuda took the tiled +// transpose kernel, which writes dst densely and ignores the view's strides: +// column S-1 stayed unwritten for rows 1.. and the values spilled into row 1. +// On the R9700 the unwritten memory held NaN and the request failed with +// "non-finite Qwen3.5 scoring-head scores" (65537 = 8 * 8192 + 1). +// +// This test runs the scorer's chunked-copy pattern and the bare [1, n, m] +// copy on the GPU backend and checks them bit for bit against a host +// reference and the CPU backend, for key counts with remainders 0, 1, 2 and +// chunk-1. It also keeps the legitimate transpose fast path covered (a +// transposed source into a contiguous destination). +#include "ggml-backend.h" +#include "ggml-cpu.h" +#include "ggml-cuda.h" +#include "ggml.h" + +#include +#include +#include +#include +#include +#include + +namespace { + +uint32_t lcg_state = 0x2468ace1u; +float lcg_uniform() { + lcg_state = lcg_state * 1664525u + 1013904223u; + return (float) ((lcg_state >> 8) & 0xFFFFFF) / (float) 0x1000000 * 2.0f - 1.0f; +} + +size_t bit_mismatches(const std::vector & a, const std::vector & b) { + if (a.size() != b.size()) return a.size() + b.size(); + size_t mm = 0; + for (size_t i = 0; i < a.size(); ++i) { + if (std::memcmp(&a[i], &b[i], sizeof(float)) != 0) ++mm; + } + return mm; +} + +// The scorer pattern: logits [S, n, H] prefilled with a sentinel, then one +// contiguous [nk, n, H] source per key chunk copied into the logits view at +// key offset b. Returns the whole logits buffer. +bool run_chunked(ggml_backend_t backend, int S, int n, int H, int chunk, + const std::vector> & parts, float sentinel, + std::vector & out) { + ggml_init_params params{}; + params.mem_size = 16 * 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + if (!ctx) return false; + ggml_tensor * logits = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, n, H); + std::vector srcs; + ggml_cgraph * graph = ggml_new_graph(ctx); + for (int b = 0, c = 0; b < S; b += chunk, ++c) { + const int nk = std::min(chunk, S - b); + ggml_tensor * src = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, nk, n, H); + srcs.push_back(src); + ggml_tensor * dst = ggml_view_3d(ctx, logits, nk, n, H, logits->nb[1], logits->nb[2], + (size_t) b * logits->nb[0]); + ggml_build_forward_expand(graph, ggml_cpy(ctx, src, dst)); + } + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend); + if (!buf) { + ggml_free(ctx); + return false; + } + std::vector fill((size_t) S * n * H, sentinel); + ggml_backend_tensor_set(logits, fill.data(), 0, fill.size() * sizeof(float)); + for (size_t c = 0; c < srcs.size(); ++c) { + ggml_backend_tensor_set(srcs[c], parts[c].data(), 0, parts[c].size() * sizeof(float)); + } + const bool ok = ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS; + ggml_backend_synchronize(backend); + if (ok) { + out.resize(fill.size()); + ggml_backend_tensor_get(logits, out.data(), 0, out.size() * sizeof(float)); + } + ggml_backend_buffer_free(buf); + ggml_free(ctx); + return ok; +} + +// A transposed source (view of a [rows, cols] tensor) into a fresh +// contiguous tensor: the transpose fast path must stay exact. +bool run_transpose(ggml_backend_t backend, int rows, int cols, const std::vector & in, + std::vector & out) { + ggml_init_params params{}; + params.mem_size = 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + if (!ctx) return false; + ggml_tensor * src = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, rows, cols); + ggml_tensor * t = ggml_cont(ctx, ggml_transpose(ctx, src)); // [cols, rows] + ggml_set_output(t); + ggml_cgraph * graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, t); + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend); + if (!buf) { + ggml_free(ctx); + return false; + } + ggml_backend_tensor_set(src, in.data(), 0, in.size() * sizeof(float)); + const bool ok = ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS; + ggml_backend_synchronize(backend); + if (ok) { + out.resize(in.size()); + ggml_backend_tensor_get(t, out.data(), 0, out.size() * sizeof(float)); + } + ggml_backend_buffer_free(buf); + ggml_free(ctx); + return ok; +} + +} // namespace + +int main() { + if (ggml_backend_cuda_get_device_count() < 1) { + std::printf("test_cpy_strided_dst: no GPU device, skipped\n"); + return 77; + } + ggml_backend_t cpu = ggml_backend_cpu_init(); + ggml_backend_t gpu = ggml_backend_cuda_init(0); + if (!cpu || !gpu) { + std::fprintf(stderr, "test_cpy_strided_dst: backend init failed (cpu=%p gpu=%p)\n", + (void *) cpu, (void *) gpu); + return 1; + } + int failures = 0; + const float sentinel = std::numeric_limits::quiet_NaN(); + const int chunk = 64; + // n = query rows per tile (32 = the scorer's kTile, 1 = a single query + // token), H = heads. S covers remainders 0, 1, 2 and chunk-1. + const int Ss[] = {2 * chunk, 2 * chunk + 1, 2 * chunk + 2, 3 * chunk - 1, 1, chunk + 1}; + const int ns[] = {32, 7, 1}; + const int H = 8; + for (const int S : Ss) { + for (const int n : ns) { + std::vector> parts; + std::vector expected((size_t) S * n * H, sentinel); + for (int b = 0; b < S; b += chunk) { + const int nk = std::min(chunk, S - b); + std::vector p((size_t) nk * n * H); + for (auto & v : p) v = lcg_uniform(); + for (int h = 0; h < H; ++h) { + for (int q = 0; q < n; ++q) { + for (int k = 0; k < nk; ++k) { + expected[((size_t) h * n + q) * S + b + k] = p[((size_t) h * n + q) * nk + k]; + } + } + } + parts.push_back(std::move(p)); + } + std::vector got_gpu, got_cpu; + if (!run_chunked(gpu, S, n, H, chunk, parts, sentinel, got_gpu) || + !run_chunked(cpu, S, n, H, chunk, parts, sentinel, got_cpu)) { + std::fprintf(stderr, "FAIL chunked S=%d n=%d: compute failed\n", S, n); + ++failures; + continue; + } + size_t nonfinite = 0; + for (const float v : got_gpu) nonfinite += std::isfinite(v) ? 0 : 1; + const size_t mm_ref = bit_mismatches(got_gpu, expected); + const size_t mm_cpu = bit_mismatches(got_gpu, got_cpu); + const bool pass = mm_ref == 0 && mm_cpu == 0 && nonfinite == 0; + std::printf("%s chunked S=%d (S%%chunk=%d) n=%d H=%d: mismatches ref=%zu cpu=%zu nonfinite=%zu\n", + pass ? "ok " : "FAIL", S, S % chunk, n, H, mm_ref, mm_cpu, nonfinite); + failures += pass ? 0 : 1; + } + } + const int shapes[][2] = {{256, 65}, {33, 1000}, {1, 64}, {64, 1}}; + for (const auto & s : shapes) { + const int rows = s[0], cols = s[1]; + std::vector in((size_t) rows * cols); + for (auto & v : in) v = lcg_uniform(); + std::vector expected((size_t) rows * cols); + for (int c = 0; c < cols; ++c) { + for (int r = 0; r < rows; ++r) expected[(size_t) r * cols + c] = in[(size_t) c * rows + r]; + } + std::vector got; + const bool ok = run_transpose(gpu, rows, cols, in, got); + const size_t mm = ok ? bit_mismatches(got, expected) : expected.size(); + const bool pass = ok && mm == 0; + std::printf("%s transpose %dx%d: mismatches=%zu\n", pass ? "ok " : "FAIL", rows, cols, mm); + failures += pass ? 0 : 1; + } + ggml_backend_free(gpu); + ggml_backend_free(cpu); + if (failures) { + std::fprintf(stderr, "test_cpy_strided_dst: %d failure(s)\n", failures); + return 1; + } + std::printf("test_cpy_strided_dst: all passed\n"); + return 0; +} From 2fa2c7e58fa870ab90044069ec15e3cf2e00266f Mon Sep 17 00:00:00 2001 From: Graffioh Date: Tue, 13 Oct 2026 08:06:48 +0000 Subject: [PATCH 2/5] feat(pflash): Qwen3.5-0.8B drafter scorer with strict selection, chat views, request probe and resumable scoring sessions PFlash compresses a long prompt by scoring it with a small drafter and serving the target only the spans that matter. This commit holds the scorer and everything the served view is built from. Drafter and selection: - The drafter is Qwen3.5-0.8B: blocks 0..14 run as a prefix and block 15's NoPE Q/K attention mass from the query window scores every key. Optional PFLASH_SCORING_HEAD_GGUF and PFLASH_SEGMENT_PROBE_GGUF companions are validated against an explicit contract and fail closed. The Qwen3-0.6B scorer is removed; src/pflash/ owns the pipeline and the IPC gains compress2/compress3. - Strict selection (PFLASH_SELECT_MODE, PFLASH_SELECT_SCORE) keeps the structurally required spans and stops at the token budget. The scorer query comes from the latest user turn or the pflash_query field; pflash_required pins client spans; PFLASH_TRACE_PATH writes a record per compression. - The causal mask is built in the graph, head+tail query windows (PFLASH_SELECT_QUERY_HEAD) and line-aligned splits are added, and a fold selection (PFLASH_SELECT_STRATEGY=fold) stays an off-by-default experiment with pflash_fold_replay. Query: - A request probe (PFLASH_REQUEST_PROBE_GGUF, PFLASH_SELECT_QUERY_PROBE=1) picks the scorer query from the human user messages, also on the chat parser. An optional stage 2 look-back (PFLASH_REQUEST_STAGE2_GGUF) and relative query candidates with a verbatim pin (PFLASH_QUERY_CANDIDATES, PFLASH_QUERY_PIN) refine it. - An identifier query stays as an evaluation reference arm, the server validates the selection config at startup, and PFLASH_VIEW_TRACE_PATH records the served view with its provenance. Parity and capture scripts for the probes are included. Chat views and admission: - Per conversation the server keeps the prompt it served and, on a follow-up, appends a small new turn verbatim, recalls dropped passages the new turn needs and rebuilds a drifted view (PFLASH_CHAT_VIEW, PFLASH_CHAT_RECALL, PFLASH_CHAT_COMPRESS_NEW_TOKENS, PFLASH_CHAT_REBUILD_*). - Cache-aware admission (PFLASH_ADMISSION=cache) compresses only when the target would otherwise prefill a threshold's worth of uncached droppable tokens, and PFLASH_ARRIVE=1 compresses just a large uncached chunk. Scoring sessions: - The drafter keeps per-conversation scoring sessions (PFLASH_DRAFTER_SESSIONS), so a shared prefix is scored once. Resume is exact by construction (the fresh call's ubatch and key-chunk partition, also for the query step), sessions are copied to host RAM across a drafter release (plain host memory by default, PFLASH_SESSION_OFFLOAD_PINNED=1 for pinned) and their capacity is clamped to the engine and drafter context. - pflash_file_identity reads st_mtime on Windows, where MSVC's struct stat has no st_mtim (the identity is only compared on the machine that wrote it). Unit tests cover selection, fold, request probe, stage 2, query candidates, identifier query, scoring resume, session offload, the causal mask and the drafter IPC; test_server_unit covers the server side. Co-Authored-By: Claude Sonnet 5.5 Co-Authored-By: Claude Opus 5.5 --- optimizations/kvflash/DESIGN.md | 6 +- optimizations/kvflash/README.md | 6 +- optimizations/pflash/README.md | 32 +- optimizations/pflash/pflash/dflash_client.py | 2 +- optimizations/pflash/tests/bench_niah_cpp.py | 8 +- optimizations/pflash/tests/niah_gen.py | 2 +- server/CMakeLists.txt | 60 +- server/README.md | 149 +- server/docs/DS4.md | 2 +- server/docs/ENVIRONMENT.md | 2 - server/docs/SPEC_PREFILL.md | 13 +- server/docs/laguna_integration_plan.md | 4 +- .../scripts/gen_request_probe_v2_fixtures.py | 65 + server/scripts/laguna_pflash_niah.py | 8 +- .../scripts/pflash_request_probe_capture.py | 255 ++ .../pflash_request_probe_capture_validate.py | 87 + server/scripts/pflash_request_probe_gguf.py | 87 + server/scripts/phase_split_dual_gpu.py | 6 +- server/scripts/quality_ab_simple.py | 2 +- server/scripts/quality_humaneval_plus.py | 2 +- server/src/common/kvflash_pager.h | 10 +- server/src/common/kvflash_scorer.h | 4 +- server/src/common/model_backend.h | 46 + server/src/common/pflash_drafter_ipc.cpp | 228 +- server/src/common/pflash_drafter_ipc.h | 38 +- .../src/common/pflash_drafter_ipc_daemon.cpp | 34 +- server/src/common/pflash_types.h | 58 + server/src/common/score_range.h | 31 - server/src/deepseek4/deepseek4_backend.cpp | 27 +- server/src/deepseek4/deepseek4_backend.h | 2 +- server/src/flashprefill.h | 2 +- server/src/flashprefill_q8.cpp | 11 +- server/src/gemma4/gemma4_backend.cpp | 12 +- server/src/gemma4/gemma4_backend.h | 4 +- .../src/gemma4/gemma4_layer_split_adapter.cpp | 4 +- .../src/gemma4/gemma4_layer_split_adapter.h | 2 +- server/src/laguna/laguna_backend.cpp | 13 +- server/src/laguna/laguna_backend.h | 4 +- .../src/laguna/laguna_layer_split_adapter.cpp | 4 +- .../src/laguna/laguna_layer_split_adapter.h | 2 +- server/src/{qwen3 => pflash}/anchor_params.h | 0 server/src/{qwen3 => pflash}/anchor_scan.cpp | 4 +- server/src/{qwen3 => pflash}/anchor_scan.h | 4 +- .../kvflash_drafter_scorer.cpp} | 39 +- .../kvflash_drafter_scorer.h} | 14 +- server/src/pflash/pflash_compress.cpp | 757 ++++ server/src/pflash/pflash_compress.h | 204 ++ server/src/pflash/pflash_drafter.cpp | 268 ++ server/src/pflash/pflash_drafter.h | 121 + server/src/pflash/pflash_fold.cpp | 1022 ++++++ server/src/pflash/pflash_fold.h | 175 + server/src/pflash/pflash_ident_query.cpp | 272 ++ server/src/pflash/pflash_ident_query.h | 81 + server/src/pflash/pflash_query_candidates.cpp | 656 ++++ server/src/pflash/pflash_query_candidates.h | 274 ++ server/src/pflash/pflash_request_probe.cpp | 665 ++++ server/src/pflash/pflash_request_probe.h | 445 +++ server/src/pflash/pflash_request_stage2.cpp | 727 ++++ server/src/pflash/pflash_request_stage2.h | 230 ++ server/src/pflash/pflash_scoring_resume.cpp | 74 + server/src/pflash/pflash_scoring_resume.h | 74 + server/src/pflash/pflash_selection.cpp | 819 +++++ server/src/pflash/pflash_selection.h | 283 ++ server/src/pflash/pflash_session_offload.cpp | 318 ++ server/src/pflash/pflash_session_offload.h | 131 + server/src/pflash/qwen35_drafter.cpp | 3256 +++++++++++++++++ server/src/pflash/qwen35_drafter.h | 256 ++ server/src/pflash/qwen35_loader.cpp | 764 ++++ server/src/qwen3/qwen3_backend.cpp | 29 +- server/src/qwen3/qwen3_backend.h | 12 +- server/src/qwen3/qwen3_buffer_plan.h | 29 - server/src/qwen3/qwen3_drafter.cpp | 887 ----- server/src/qwen3/qwen3_drafter.h | 87 - server/src/qwen3/qwen3_drafter_model.h | 129 - server/src/qwen3/qwen3_graph.cpp | 906 ----- server/src/qwen3/qwen3_loader.cpp | 33 +- server/src/qwen3/qwen3_model.h | 74 + server/src/qwen35/gguf_target_loader.cpp | 4 +- server/src/qwen35/qwen35_backend.cpp | 51 +- server/src/qwen35/qwen35_backend.h | 2 +- .../src/qwen35/qwen35_layer_split_adapter.cpp | 33 +- .../src/qwen35/qwen35_layer_split_adapter.h | 2 +- server/src/server/adaptive_keep_ratio.h | 17 +- server/src/server/http_server.cpp | 2895 ++++++++++++++- server/src/server/http_server.h | 436 ++- server/src/server/server_main.cpp | 23 +- server/src/server/sse_emitter.cpp | 4 +- server/src/server/sse_emitter.h | 3 + server/src/server/tokenizer.cpp | 7 + server/src/server/tokenizer.h | 4 + server/test/bench/bench_laguna_pflash.cpp | 10 +- server/test/pflash_fold_replay.cpp | 316 ++ .../test/python/test_full_compress_cache.py | 4 +- server/test/smoke/pflash_daemon.cpp | 125 +- server/test/smoke/smoke_qwen3_forward.cpp | 16 +- server/test/unit/agent_client_requests.inc | 6 + server/test/unit/request_probe_v2_parity.inc | 83 + server/test/unit/test_anchor_params.cpp | 2 +- server/test/unit/test_anchor_transitive.cpp | 30 +- server/test/unit/test_bandit_integration.cpp | 11 + server/test/unit/test_dflash.cpp | 34 +- .../test_drafter_early_exit_score_range.cpp | 90 - server/test/unit/test_kvflash.cpp | 16 +- server/test/unit/test_pflash_drafter_ipc.cpp | 187 + server/test/unit/test_pflash_fold.cpp | 655 ++++ server/test/unit/test_pflash_ident_query.cpp | 101 + .../unit/test_pflash_query_candidates.cpp | 521 +++ .../test/unit/test_pflash_request_probe.cpp | 448 +++ .../test/unit/test_pflash_request_stage2.cpp | 1114 ++++++ .../test/unit/test_pflash_scoring_resume.cpp | 151 + server/test/unit/test_pflash_selection.cpp | 953 +++++ .../test/unit/test_pflash_session_offload.cpp | 409 +++ server/test/unit/test_qwen35_causal_mask.cpp | 101 + server/test/unit/test_qwen3_buffer_plan.cpp | 71 - server/test/unit/test_server_unit.cpp | 2287 +++++++++++- 115 files changed, 23993 insertions(+), 2682 deletions(-) create mode 100644 server/scripts/gen_request_probe_v2_fixtures.py create mode 100644 server/scripts/pflash_request_probe_capture.py create mode 100644 server/scripts/pflash_request_probe_capture_validate.py create mode 100644 server/scripts/pflash_request_probe_gguf.py create mode 100644 server/src/common/pflash_types.h delete mode 100644 server/src/common/score_range.h rename server/src/{qwen3 => pflash}/anchor_params.h (100%) rename server/src/{qwen3 => pflash}/anchor_scan.cpp (99%) rename server/src/{qwen3 => pflash}/anchor_scan.h (96%) rename server/src/{qwen3/qwen3_kvflash_scorer.cpp => pflash/kvflash_drafter_scorer.cpp} (88%) rename server/src/{qwen3/qwen3_kvflash_scorer.h => pflash/kvflash_drafter_scorer.h} (82%) create mode 100644 server/src/pflash/pflash_compress.cpp create mode 100644 server/src/pflash/pflash_compress.h create mode 100644 server/src/pflash/pflash_drafter.cpp create mode 100644 server/src/pflash/pflash_drafter.h create mode 100644 server/src/pflash/pflash_fold.cpp create mode 100644 server/src/pflash/pflash_fold.h create mode 100644 server/src/pflash/pflash_ident_query.cpp create mode 100644 server/src/pflash/pflash_ident_query.h create mode 100644 server/src/pflash/pflash_query_candidates.cpp create mode 100644 server/src/pflash/pflash_query_candidates.h create mode 100644 server/src/pflash/pflash_request_probe.cpp create mode 100644 server/src/pflash/pflash_request_probe.h create mode 100644 server/src/pflash/pflash_request_stage2.cpp create mode 100644 server/src/pflash/pflash_request_stage2.h create mode 100644 server/src/pflash/pflash_scoring_resume.cpp create mode 100644 server/src/pflash/pflash_scoring_resume.h create mode 100644 server/src/pflash/pflash_selection.cpp create mode 100644 server/src/pflash/pflash_selection.h create mode 100644 server/src/pflash/pflash_session_offload.cpp create mode 100644 server/src/pflash/pflash_session_offload.h create mode 100644 server/src/pflash/qwen35_drafter.cpp create mode 100644 server/src/pflash/qwen35_drafter.h create mode 100644 server/src/pflash/qwen35_loader.cpp delete mode 100644 server/src/qwen3/qwen3_buffer_plan.h delete mode 100644 server/src/qwen3/qwen3_drafter.cpp delete mode 100644 server/src/qwen3/qwen3_drafter.h delete mode 100644 server/src/qwen3/qwen3_drafter_model.h delete mode 100644 server/src/qwen3/qwen3_graph.cpp create mode 100644 server/src/qwen3/qwen3_model.h create mode 100644 server/test/pflash_fold_replay.cpp create mode 100644 server/test/unit/agent_client_requests.inc create mode 100644 server/test/unit/request_probe_v2_parity.inc delete mode 100644 server/test/unit/test_drafter_early_exit_score_range.cpp create mode 100644 server/test/unit/test_pflash_drafter_ipc.cpp create mode 100644 server/test/unit/test_pflash_fold.cpp create mode 100644 server/test/unit/test_pflash_ident_query.cpp create mode 100644 server/test/unit/test_pflash_query_candidates.cpp create mode 100644 server/test/unit/test_pflash_request_probe.cpp create mode 100644 server/test/unit/test_pflash_request_stage2.cpp create mode 100644 server/test/unit/test_pflash_scoring_resume.cpp create mode 100644 server/test/unit/test_pflash_selection.cpp create mode 100644 server/test/unit/test_pflash_session_offload.cpp create mode 100644 server/test/unit/test_qwen35_causal_mask.cpp delete mode 100644 server/test/unit/test_qwen3_buffer_plan.cpp diff --git a/optimizations/kvflash/DESIGN.md b/optimizations/kvflash/DESIGN.md index 965dde709..856c53a75 100644 --- a/optimizations/kvflash/DESIGN.md +++ b/optimizations/kvflash/DESIGN.md @@ -106,7 +106,7 @@ FA span traffic is bandwidth-realistic: ## Full LSA loop (drafter as Memory Indexer) — measured Test run F implements the paper's complete inference paradigm with the -pflash drafter (Qwen3-0.6B, `/opt/lucebox/models/drafter/`) standing in +pflash drafter (Qwen3.5-0.8B, `/opt/lucebox/models/drafter/`) standing in for the trained indexer: prompt (2048) larger than the pool (1024) so prefill itself evicts, then every τ=64 decoded tokens the drafter rescores the full sequence (tail attention = indexer query, chunk means @@ -176,7 +176,7 @@ The pool is wired into the qwen35 backend behind `--kvflash ` pool (live LRU eviction mid-request). Coherent story end to end, 36.9 tok/s, clean finish. Second request (per-request pager reset) ok. 2. WITH pflash: `--kvflash 2048 --prefill-compression always - --prefill-threshold 256 --prefill-drafter `. Compression + --prefill-threshold 256 --prefill-drafter `. Compression 1468 -> 60 tokens, then `[kvflash] drafter scorer attached (tau=64)` automatically; 400 coherent tokens answering from the compressed context. Same binary, zero pflash-specific configuration on the pool. @@ -246,7 +246,7 @@ and masks through it. What differs per arch: 3.09, identical text). Policy: drafter-scored residency is the default on all four archs. The -server probes for the Qwen3-0.6B next to the model (or --prefill-drafter) +server probes for the Qwen3.5-0.8B next to the model (or --prefill-drafter) and lazy-loads it at the first reselect; `--kvflash-policy lru` opts out. qwen35/qwen35moe feed the drafter target ids directly; laguna/gemma4 use KvFlashCrossTokScorer (detokenize -> re-tokenize -> score -> map back by diff --git a/optimizations/kvflash/README.md b/optimizations/kvflash/README.md index ed3d6cb75..9a86018ee 100644 --- a/optimizations/kvflash/README.md +++ b/optimizations/kvflash/README.md @@ -41,7 +41,7 @@ does not fit at all.) # recommended: drafter-scored residency, pool auto-sized from VRAM. # pass --prefill-drafter so the drafter is guaranteed (no silent LRU fallback). luce_server model.gguf --max-ctx 32768 --kvflash auto \ - --prefill-drafter /opt/lucebox/models/drafter/Qwen3-0.6B-BF16.gguf + --prefill-drafter /opt/lucebox/models/drafter/Qwen3.5-0.8B-BF16.gguf # drop the path to auto-probe (model dir, drafter/, draft/, /opt/lucebox/models/drafter/); # falls back to LRU if none is found, so check the banner reads policy=drafter @@ -52,7 +52,7 @@ luce_server model.gguf --max-ctx 32768 --kvflash 8192 --kvflash-policy lru ``` Drafter-scored residency is the DEFAULT policy on every model family: -the server probes for `Qwen3-0.6B-BF16.gguf` next to the model (same +the server probes for `Qwen3.5-0.8B-BF16.gguf` next to the model (same dir, `drafter/`, `draft/`, then `/opt/lucebox/models/drafter/`) and lazy-loads it on the first reselect; `--prefill-drafter` overrides the location, prefill compression can stay off either way. Qwen-family @@ -147,7 +147,7 @@ Env: `LUCE_KVFLASH_POLICY=qk`. Bench: `test_kvflash --qkbench`. - `server/src/common/kvflash_qk.h` — target-QK scorer: pure scoring math (unit-tested in `server/test/unit/test_kvflash_qk.cpp`), seal-time key pooling, `KvFlashTargetQkScorer` -- `server/src/qwen3/qwen3_kvflash_scorer.{h,cpp}` — pflash-drafter scorer +- `server/src/pflash/kvflash_drafter_scorer.{h,cpp}` — pflash-drafter scorer (tail attention; bisects on allocation pressure) - `server/src/qwen35/*` — cache `ctx_alloc`, masked pooled decode, slot-mapped spec verify, daemon flags diff --git a/optimizations/pflash/README.md b/optimizations/pflash/README.md index b53638c96..060f7da66 100644 --- a/optimizations/pflash/README.md +++ b/optimizations/pflash/README.md @@ -39,7 +39,7 @@ Long-context prefill is O(S²): vanilla llama.cpp on a single RTX 3090 takes **~ **What was missing:** no implementation that sits in front of a quantized GGUF target on a 24 GB card without dragging Python+Triton into the runtime path. PFlash is that: - C++/CUDA daemon-resident drafter + scoring + target generation, all in one process, one ggml allocator. -- Custom Qwen3-0.6B BF16 forward (`qwen3_0p6b_loader.cpp` + `qwen3_0p6b_graph.cpp`) — no libllama. +- Custom Qwen3.5-0.8B BF16 forward (`src/pflash/qwen35_loader.cpp` + the qwen35 target graph) — no libllama. - 4 CUDA kernels for the FlashPrefill `mean_K → score → select → sparse_fwd` algorithm (`flashprefill_kernels.cu`). - BSA ([mit-han-lab/Block-Sparse-Attention](https://github.com/mit-han-lab/Block-Sparse-Attention), FA-2 derived, sm_80+) for the long-context drafter forward, wired without `libtorch` via 3 ATen/c10 header stubs (`server/deps/bsa_stubs/`). - 128K → 2.6K span selection at `keep_ratio=0.05`, NIAH retrieved at every measured context, decode ~74 tok/s downstream. @@ -72,13 +72,13 @@ cmake --build server/build --target test_dflash test_flashprefill_kernels -j # 2. fetch weights (target + spec-decode draft + drafter scorer) uv run hf download unsloth/Qwen3.6-27B-GGUF Qwen3.6-27B-Q4_K_M.gguf --local-dir server/models/ -uv run hf download Qwen/Qwen3-0.6B model.safetensors tokenizer.json --local-dir server/models/drafter/ +uv run hf download Qwen/Qwen3.5-0.8B model.safetensors tokenizer.json --local-dir server/models/drafter/ uv run hf download z-lab/Qwen3.6-27B-DFlash model.safetensors --local-dir server/models/draft/ -# 2b. convert the drafter (Qwen3-0.6B HF) to a BF16 GGUF for the C++ scorer. +# 2b. convert the drafter (Qwen3.5-0.8B HF) to a BF16 GGUF for the C++ scorer. # The submodule already vendors llama.cpp at deps/llama.cpp. uv run python server/deps/llama.cpp/convert_hf_to_gguf.py server/models/drafter \ - --outtype bf16 --outfile server/models/Qwen3-0.6B-BF16.gguf + --outtype bf16 --outfile server/models/Qwen3.5-0.8B-BF16.gguf # 3. generate NIAH cases + run head-to-head bench against the C++ daemon uv run --directory pflash python tests/niah_gen.py --n 1 --ctx 131072 --out /tmp/niah_128k.jsonl @@ -86,7 +86,7 @@ uv run --directory pflash python tests/bench_niah_cpp.py \ --bin ../server/build/test_dflash \ --target ../server/models/Qwen3.6-27B-Q4_K_M.gguf \ --draft-spec ../server/models/draft/model.safetensors \ - --drafter-gguf ../server/models/Qwen3-0.6B-BF16.gguf \ + --drafter-gguf ../server/models/Qwen3.5-0.8B-BF16.gguf \ --cases /tmp/niah_128k.jsonl --keep-ratio 0.05 --n-gen 256 ``` @@ -99,8 +99,8 @@ For an OpenAI-compatible server with transparent compression on long prompts, ru | `--prefill-compression` | `off` / `auto` / `always` | `off` | When to run pflash. `auto` compresses when total prompt ≥ threshold; `always` compresses every request. | | `--prefill-threshold` | int (tokens) | `32000` | Token threshold for `auto` mode. | | `--prefill-keep-ratio` | float `(0, 1]` | `0.05` | Fraction of source tokens to keep after compression. `0.02` for 128K, `0.10` for 32K. | -| `--prefill-drafter` | path to `.gguf` | required when not `off` | Drafter weights (Qwen3-0.6B BF16 GGUF). | -| `--prefill-drafter-tokenizer` | HF repo id | `Qwen/Qwen3-0.6B` | HF tokenizer for the drafter vocab. | +| `--prefill-drafter` | path to `.gguf` | required when not `off` | Drafter weights (Qwen3.5-0.8B BF16 GGUF). | +| `--prefill-drafter-tokenizer` | HF repo id | `Qwen/Qwen3.5-0.8B` | HF tokenizer for the drafter vocab. | When `--prefill-compression != off`, the server auto-sets `LUCE_LM_HEAD_FIX=0` and `LUCE_FA_WINDOW=0` (matching the bench harness — needed so the post-compress draft graph fits on a 24 GB card without OOM). @@ -111,7 +111,7 @@ When `--prefill-compression != off`, the server auto-sets `LUCE_LM_HEAD_FIX=0` a --prefill-compression auto \ --prefill-threshold 4096 \ --prefill-keep-ratio 0.02 \ - --prefill-drafter server/models/Qwen3-0.6B-BF16.gguf + --prefill-drafter server/models/Qwen3.5-0.8B-BF16.gguf ``` Below the threshold the server runs the standard target generate (no compression). Above it, the server transparently runs `compress` on the daemon, swaps the prompt for the compressed text, and continues the normal `/v1/chat/completions` flow. Tool-calling requests (`req.tools` non-empty) skip compression so JSON tool definitions stay intact. @@ -155,7 +155,7 @@ prompt (≤ 128K tokens) ▼ ┌──────────────────────────────────────────────┐ │ drafter (in-process) │ -│ custom Qwen3-0.6B BF16 forward in ggml │ +│ custom Qwen3.5-0.8B BF16 forward in ggml │ │ FlashPrefill block-sparse via BSA (≥ 32K) │ │ tail-attention scoring → score [S] │ │ chunk(128) + alpha-threshold → top blocks │ @@ -177,7 +177,7 @@ prompt (≤ 128K tokens) └──────────────────────────────────────────────┘ ``` -**Drafter forward.** Custom Qwen3-0.6B graph (`qwen3_0p6b_graph.cpp`) per-layer A/FP/B blocks: dense attention up to ~32K source, FlashPrefill sparse attention at and above. The 4 FP kernels live in `flashprefill_kernels.cu`; BSA dispatch is in `bsa_launcher.cu` + `bsa_fwd_inst.cu`. +**Drafter forward.** The Qwen3.5-0.8B drafter (`src/pflash/qwen35_drafter.cpp` on the qwen35 target graph) runs the model's first fifteen blocks and scores the context with block 15's NoPE Q/K attention-mass head; `PFLASH_QWEN35_LEGACY_SCORER=1` selects the all-layer running-max scorer instead. **Scoring + selection.** Tail attention `Q[-N:] @ K^T / sqrt(d)` per layer/head, max over (L, H), mean over the tail window. Block-level threshold by `alpha * mean(scores)` selects which K-blocks each Q-block attends to. Configurable via `LUCE_FP_ALPHA`. @@ -205,14 +205,14 @@ What we built: - C++/CUDA port of the FlashPrefill algorithm: 4 kernels (`mean_K / score / select / sparse_fwd`), no Triton dependency. - BSA ([mit-han-lab/Block-Sparse-Attention](https://github.com/mit-han-lab/Block-Sparse-Attention)) wired without `libtorch` via 3 ATen/c10 header stubs (`server/deps/bsa_stubs/`). -- Custom Qwen3-0.6B BF16 forward so the drafter runs through the same ggml allocator as the 27B target. +- Custom Qwen3.5-0.8B BF16 forward so the drafter runs through the same ggml allocator as the 27B target. - Daemon stdin protocol (`compress` / `generate` / `park` / `unpark` / `free drafter`) so target + drafter coexist on a 24 GB card. - NIAH harness against `llama-bench` for end-to-end validation. ## Scope and limits - **Single 24 GB GPU** target (RTX 3090 reference). On 32+ GB cards, drafter + target can coexist and the park/unpark dance disappears. -- **Qwen3.6-27B Q4_K_M target + Qwen3-0.6B drafter** is the validated pair. Other targets/drafters need keep_ratio + alpha re-calibration. +- **Qwen3.6-27B Q4_K_M target + Qwen3.5-0.8B drafter** is the validated pair. Other targets/drafters need keep_ratio + alpha re-calibration. - **NIAH single-needle** is the only retrieval task validated end-to-end. Multi-doc QA, long-form code retrieval, etc. still TBD. - **sm_80+** required for BSA (RTX 3090 sm_86 is the reference). On sm_75 (Turing) the build auto-disables BSA and falls back to the WMMA path; expect a slower drafter forward at long ctx. @@ -242,16 +242,16 @@ These are operator-side flags on the launcher; they do not change PFlash semantics. A short prompt lane should keep the original defaults. -### Drafter selection: BF16 Qwen3-0.6B for compress +### Drafter selection: BF16 Qwen3.5-0.8B for compress PFlash compress benefits from a small, fast drafter. The validated -choice is **Qwen3-0.6B** in **BF16 safetensors** with ~5 attention +choice is **Qwen3.5-0.8B** in **BF16 safetensors** with ~5 attention layers. The DFlash drafter for the same target works correctly during decode-after-unpark but is heavier than ideal for compress. Practical guidance: -- Use Qwen3-0.6B BF16 for `compress` (PFlash side). +- Use Qwen3.5-0.8B BF16 for `compress` (PFlash side). - Reuse the larger DFlash drafter for `decode` after unpark (DFlash side). @@ -262,7 +262,7 @@ simultaneously on a 24 GB GPU. Reproducible comparison vs Ollama native `/api/chat` on the same 64K unique-prompt summary task, RTX 6000 Ada sm_89, -Qwen3.6-27B-Q4_K_M, FA_WINDOW=0. Drafter setup: Qwen3-0.6B BF16 +Qwen3.6-27B-Q4_K_M, FA_WINDOW=0. Drafter setup: Qwen3.5-0.8B BF16 GGUF for the PFlash compress path (see "Drafter selection" above); the larger DFlash drafter on the luce daemon side ran as FP16 safetensors during decode-after-unpark on this run. Feel free to diff --git a/optimizations/pflash/pflash/dflash_client.py b/optimizations/pflash/pflash/dflash_client.py index 89c214892..942c6bba4 100644 --- a/optimizations/pflash/pflash/dflash_client.py +++ b/optimizations/pflash/pflash/dflash_client.py @@ -230,7 +230,7 @@ def park_target(self): self._send("park target\n") def unpark_target(self): self._send("unpark target\n") def compress(self, prompt_ids: list[int], keep_ratio: float, drafter_gguf: str, - drafter_arch: str = "qwen3-0.6b") -> list[int]: + drafter_arch: str = "qwen35-0.8b") -> list[int]: """C++ drafter score+compress via daemon. Returns compressed token ids. Daemon command: compress diff --git a/optimizations/pflash/tests/bench_niah_cpp.py b/optimizations/pflash/tests/bench_niah_cpp.py index e289d2579..8f4dbb85a 100644 --- a/optimizations/pflash/tests/bench_niah_cpp.py +++ b/optimizations/pflash/tests/bench_niah_cpp.py @@ -27,12 +27,12 @@ def main(): ap.add_argument("--target", default="/opt/lucebox/models/Qwen3.6-27B-Q4_K_M.gguf") ap.add_argument("--draft-spec", default="/home/lucebox/lucebox-hub/server/models/draft/model.safetensors", help="draft model used for spec decoding (NOT drafter scorer)") - ap.add_argument("--drafter-gguf", default="/home/lucebox/lucebox-hub/server/models/Qwen3-0.6B-BF16.gguf", - help="C++ drafter scorer GGUF (Qwen3-0.6B BF16)") - ap.add_argument("--drafter-arch", default="qwen3-0.6b", choices=["qwen3-0.6b", "qwen35-0.8b"], + ap.add_argument("--drafter-gguf", default="/home/lucebox/lucebox-hub/server/models/Qwen3.5-0.8B-BF16.gguf", + help="C++ drafter scorer GGUF (Qwen3.5-0.8B BF16)") + ap.add_argument("--drafter-arch", default="qwen35-0.8b", choices=["qwen35-0.8b"], help="C++ drafter architecture selector") ap.add_argument("--target-tokenizer", default="Qwen/Qwen3.6-27B") - ap.add_argument("--drafter-tokenizer", default="Qwen/Qwen3-0.6B") + ap.add_argument("--drafter-tokenizer", default="Qwen/Qwen3.5-0.8B") ap.add_argument("--max-ctx", type=int, default=16384, help="daemon KV cache max ctx; sized for compressed prompt+gen, NOT source") ap.add_argument("--keep-ratio", type=float, default=0.020) diff --git a/optimizations/pflash/tests/niah_gen.py b/optimizations/pflash/tests/niah_gen.py index 39db4f1cb..c307e5588 100644 --- a/optimizations/pflash/tests/niah_gen.py +++ b/optimizations/pflash/tests/niah_gen.py @@ -110,7 +110,7 @@ def main(): # Default matches bench_niah_cpp.py's --drafter-tokenizer, since that is # the tokenizer the downstream NIAH bench uses to size case["prompt"] # for the drafter forward. Override for any other harness. - ap.add_argument("--tokenizer", default="Qwen/Qwen3-0.6B") + ap.add_argument("--tokenizer", default="Qwen/Qwen3.5-0.8B") args = ap.parse_args() tok = AutoTokenizer.from_pretrained(args.tokenizer) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index f323d3fba..81606ae44 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -197,9 +197,9 @@ endif() # ─── ggml (vendored from llama.cpp) ────────────────────────────────── # -# We use only ggml from the vendored llama.cpp snapshot. The drafter is -# loaded via our own custom Qwen3-0.6B forward -# (src/qwen3/qwen3_loader.cpp + src/qwen3/qwen3_graph.cpp) +# We use only ggml from the vendored llama.cpp snapshot. The PFlash drafter +# is loaded via our own custom Qwen3.5-0.8B forward +# (src/pflash/qwen35_loader.cpp + src/qwen35/qwen35_target_graph.cpp) # rather than libllama, so libllama is not built. # # No BLAS, no Metal, no Vulkan, no examples/tests/tools. @@ -570,6 +570,7 @@ set(LUCE_SRC_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/src/qwen4exp ${CMAKE_CURRENT_SOURCE_DIR}/src/laguna ${CMAKE_CURRENT_SOURCE_DIR}/src/qwen3 + ${CMAKE_CURRENT_SOURCE_DIR}/src/pflash ${CMAKE_CURRENT_SOURCE_DIR}/src/gemma4 ${CMAKE_CURRENT_SOURCE_DIR}/src/deepseek4 ${CMAKE_CURRENT_SOURCE_DIR}/src/server @@ -591,11 +592,21 @@ add_library(luce_common STATIC src/draft/draft_gguf_loader.cpp src/draft/draft_safetensors_loader.cpp src/draft/draft_graph.cpp - src/qwen3/anchor_scan.cpp - src/qwen3/qwen3_drafter.cpp - src/qwen3/qwen3_kvflash_scorer.cpp + src/pflash/anchor_scan.cpp + src/pflash/pflash_selection.cpp + src/pflash/pflash_fold.cpp + src/pflash/pflash_request_probe.cpp + src/pflash/pflash_request_stage2.cpp + src/pflash/pflash_query_candidates.cpp + src/pflash/pflash_scoring_resume.cpp + src/pflash/pflash_session_offload.cpp + src/pflash/pflash_ident_query.cpp + src/pflash/pflash_drafter.cpp + src/pflash/pflash_compress.cpp + src/pflash/qwen35_drafter.cpp + src/pflash/qwen35_loader.cpp + src/pflash/kvflash_drafter_scorer.cpp src/qwen3/qwen3_loader.cpp - src/qwen3/qwen3_graph.cpp src/qwen3/qwen3_backend.cpp src/qwen3/qwen3_daemon.cpp src/gemma4/gemma4_loader.cpp @@ -827,7 +838,7 @@ endif() # - CUDA sm_60–sm_69 (Pascal): scalar F16, no tensor cores — flashprefill_scalar.cu # - HIP Phase 1 (default): ggml q8 fallback, no custom kernels. # - HIP Phase 2 (LUCE_HIP_SM80_EQUIV=ON): rocWMMA-native kernels. -# The dispatch in qwen3_graph.cpp checks buffer type at runtime: +# The dispatch in flashprefill.h checks buffer type at runtime: # BF16 buffers → bf16 WMMA kernel; F16 buffers → f16 WMMA kernel; else → ggml FA. if(LUCE_GPU_BACKEND STREQUAL "hip") # rms_norm_hip.cu is needed by the HIP chunk-B graph path regardless of SM80_EQUIV. @@ -2276,6 +2287,25 @@ if(LUCE_TESTS) endif() endif() + # PFlash fold replay: the runtime's fold selection over recorded traces + # (CPU only; the drafter GGUF supplies the vocabulary). + add_executable(pflash_fold_replay test/pflash_fold_replay.cpp) + target_include_directories(pflash_fold_replay PRIVATE ${LUCE_SRC_INCLUDE_DIRS}) + if(LUCE_GPU_BACKEND STREQUAL "hip") + target_compile_definitions(pflash_fold_replay PRIVATE LUCE_BACKEND_HIP=1 GGML_USE_HIP) + else() + target_compile_definitions(pflash_fold_replay PRIVATE + LUCE_BACKEND_CUDA=1 + LUCE_CUDA_MIN_SM=${_luce_cuda_min_sm}) + endif() + target_link_libraries(pflash_fold_replay PRIVATE luce_common ggml ${LUCE_GGML_BACKEND_TARGET}) + if(LUCE_GPU_BACKEND STREQUAL "cuda") + find_package(CUDAToolkit REQUIRED) + target_link_libraries(pflash_fold_replay PRIVATE CUDA::cudart) + else() + target_link_libraries(pflash_fold_replay PRIVATE hip::host) + endif() + # ─── Unit tests (no GPU, no model files) ──────────────────────────── # Shared image input: request transport and JPEG/PNG decoding. add_executable(test_image_input test/unit/test_image_input.cpp src/server/image_input.cpp) @@ -2445,8 +2475,16 @@ if(LUCE_TESTS) test/unit/test_chain_rollback_policy.cpp test/unit/test_ddtree_tau.cpp test/unit/test_anchor_transitive.cpp - test/unit/test_drafter_early_exit_score_range.cpp - test/unit/test_qwen3_buffer_plan.cpp + test/unit/test_pflash_drafter_ipc.cpp + test/unit/test_pflash_selection.cpp + test/unit/test_pflash_fold.cpp + test/unit/test_pflash_request_probe.cpp + test/unit/test_pflash_request_stage2.cpp + test/unit/test_pflash_query_candidates.cpp + test/unit/test_pflash_scoring_resume.cpp + test/unit/test_pflash_session_offload.cpp + test/unit/test_pflash_ident_query.cpp + test/unit/test_qwen35_causal_mask.cpp test/unit/test_model_test_paths.cpp test/unit/test_gguf_mmap.cpp test/unit/test_kv_quant.cpp @@ -2471,7 +2509,7 @@ if(LUCE_TESTS) src/server/scheduler.cpp src/server/model_card.cpp src/server/prompt_normalize.cpp - src/qwen3/anchor_scan.cpp) + src/pflash/anchor_scan.cpp) target_include_directories(test_server_unit PRIVATE ${LUCE_SRC_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/src diff --git a/server/README.md b/server/README.md index f264586ab..05524f402 100644 --- a/server/README.md +++ b/server/README.md @@ -194,7 +194,7 @@ server is byte-identical to local-inference mode. ```bash ./build/luce_server models/Qwen3.6-27B-Q4_K_M.gguf \ --prefill-compression auto --prefill-threshold 10000 \ - --prefill-drafter models/Qwen3-0.6B-BF16.gguf \ + --prefill-drafter models/Qwen3.5-0.8B-BF16.gguf \ --prefill-curve 10000:0.5 40000:0.2 100000:0.1 \ --prefill-upstream-base http://127.0.0.1:8099 \ --prefill-upstream-model my-upstream-model \ @@ -391,11 +391,148 @@ the whole request's device footprint. `/status/json` reports | `--prefill-threshold ` | `32000` | Token threshold used by auto mode. | | `--prefill-keep-ratio ` | `0.05` | Fraction of source tokens kept. | | `--prefill-curve T:R [T:R ...]` | none | Piecewise keep-ratio curve; overrides the flat ratio. | -| `--prefill-drafter ` | none | PFlash drafter GGUF. | +| `--prefill-drafter ` | none | PFlash drafter GGUF (Qwen3.5-0.8B). | | `--prefill-skip-park` | off | Keep target and decode draft resident while PFlash runs. | | `--prefill-upstream-base ` | none | Enable compression-proxy mode. | | `--prefill-upstream-key ` | none | Bearer token for the upstream. | | `--prefill-upstream-model ` | none | Model name forwarded upstream. | +| `PFLASH_SELECT_MODE=top_k` + `PFLASH_SELECT_TOPK ` | budget-only fill | Rank rule: keep the K highest-scoring optional segments in score order instead of filling the keep ratio, with the keep-ratio budget still a hard ceiling (min(K segments, the budget)). Use it where the evidence is compact and sits in the first few ranks -- needle retrieval, passage QA, code -- so a small K reaches it for a fraction of the budget's tokens. Do not use it where the answer needs a whole document identified, since the evidence there spans many segments and K cuts it off. | +| `PFLASH_SELECT_QUERY_HEAD=1` (with `PFLASH_SELECT_QUERY_PARSER=latest_user`) | off | The question may come before pasted material. The user's leading prose block (up to the first paste-like line: a code fence, heading, separator, record/file header, tag, JSON or table data, indented or code-shaped line) is scored as an extra query window and kept verbatim; the trailing prose block after the last paste becomes the query window instead of the fixed tail. Caps: `PFLASH_SELECT_QUERY_HEAD_TOKENS` / `PFLASH_SELECT_QUERY_TAIL_TOKENS` (256); a longer block contributes its first and last cap/2 tokens. Blocks under 16 tokens are ignored; a message with no paste structure keeps today's window, except that a short opening paragraph ending in `?` before a long plain text counts as the head. | +| `PFLASH_SELECT_STRATEGY=fold` (with `PFLASH_SELECT_MODE=budget_only`) | `segments` | Fold selection: each of the top 20 segments (`PFLASH_SELECT_FOLD_K`) is completed to its smallest enclosing block -- a function or class in code (Python indentation or brace-matched C/C++/JS/TS/Go/Rust bodies), a numbered record such as `Document N:`, a markdown section, else a blank-line paragraph (`PFLASH_SELECT_FOLD_PARAGRAPHS=0` turns that last fallback off). A block over 1500 tokens (`PFLASH_SELECT_FOLD_CAP`) keeps its first 150 (`PFLASH_SELECT_FOLD_HEAD`) plus the segment. Code-like prompts (most non-blank lines inside functions; JSON and markdown are not) rank segments by mass (density x length), others by density. The keep-ratio budget stays the ceiling. Needs the head scorer; the drafter loads its vocabulary to read the prompt's structure. | + +With a Qwen3.5-0.8B drafter and strict budget selection +(`PFLASH_SELECT_MODE=budget_only`, `PFLASH_SELECT_CHUNK_SIZE`, +`PFLASH_SELECT_QUERY_TOKENS`), the drafter runs only its first fifteen +blocks and scores the context with block 15's NoPE Q/K projections as an +attention-mass scorer. Its 262K native context covers very long inputs. `PFLASH_SCORING_HEAD_GGUF` accepts a trained block-15 head +(schema `qwen3_5_0_8b_nope_qk_mass_v1`); `PFLASH_QWEN35_LEGACY_SCORER=1` +restores the previous all-layer running-max scorer. The Qwen3.5 attention +runs dense (`ggml_flash_attn_ext`); the block-sparse FlashPrefill kernels +still dispatch head dimension 128 only. + +The scorer query of a chat is the prompt's last token: the end of the +generation prompt, where the model starts answering, having read the whole +request. Nothing is parsed out of the user's text, so the question can sit +anywhere in the message -- before a pasted document, in the middle of it, or +among the user's own sentences -- and the user's own words score far above +the material they paste. The tail (`PFLASH_SELECT_QUERY_TOKENS`) of the +latest user turn scores alongside it as a second query window at the same +weight: the last token reads the whole request, the user's own tokens match +literal strings -- an identifier, a described function -- that it does not +carry. Strict selection keeps the generation prompt and +the latest user turn's role header, and it runs on every turn of a +multi-turn chat. In an agent loop the assistant and tool turns after the +user's turn are scored like the rest of the conversation. A prompt without +chat markers scores the tail (`PFLASH_SELECT_QUERY_TOKENS`, default 8) of +its content. + +The keep ratio applies to the droppable tokens only: what strict selection +keeps anyway (system and developer messages, tool definitions, the query and +its turn's envelope, the generation prompt) is added on top, so a long +system prompt no longer exhausts the budget. Auto mode compares +`--prefill-threshold` with the droppable tokens too. PFlash never compresses +the system prompt: one that alone would not fit the context fails the +request. Developer messages and tool definitions that would not fit lose +their pin and are scored like any other context. + +Multi-turn chats keep a view: the prompt served for a turn is remembered, +and when the next request's prompt continues it (same tokens up to the old +generation prompt), PFlash serves that view plus the new turns instead of a +fresh compression, so the target restores its prefix-cache snapshot of the +view (taken at the start of its generation prompt) and prefills only what is +new. Segments the fresh selection keeps for the new question that the view +lacks are recalled as excerpts at the start of the new user turn. When the +view grows past twice the fresh prompt, or past the context, the fresh prompt +starts a new view. `PFLASH_CHAT_VIEW=0` serves the fresh compression every +turn. + +What a turn adds is appended verbatim while it is small, the way full +prefill appends a follow-up; from `PFLASH_CHAT_COMPRESS_NEW_TOKENS` (default +16384) tokens of new material (a pasted document, a large tool output) only +what the fresh selection keeps of it is appended, and the view before it +stays cached. `PFLASH_CHAT_RECALL=0` turns recall off: a small follow-up is +then served without running the drafter at all. Recall takes what a fresh +selection for the new question keeps that the view lacks: a question on the +view's topic misses little and is served by appending it, and a question +that needs more than a third of a fresh selection -- the conversation moved +to other material -- starts a new view from that selection instead. Kept pieces that were not adjacent in the prompt are joined by a +paragraph break when neither side has one (`PFLASH_SELECT_PARAGRAPH_JOIN=0` +turns it off; the breaks do not count against the token ceiling). +`PFLASH_VIEW_TRACE_PATH` +appends each compressed request's served prompt as JSONL, for evidence +checks in evaluations. + +Every turn of a multi-turn chat keeps its role header, and user turns (the +latest included) and assistant answers up to `PFLASH_CHAT_SKELETON_TOKENS` +(default 256 drafter tokens; 0 keeps headers only) stay whole: the +conversation's skeleton, as opposed to the material it quotes. Like +instructions, the skeleton is scored as context when it alone would not fit. +The last `PFLASH_CHAT_HISTORY_QUERIES` (default 3) earlier user turns score +the context alongside the current query, each through the last token of the +header of the reply that followed it (that turn's own prompt end), their +masses mixed in at weights 1/2, 1/4, 1/8, so what the conversation keeps +coming back to stays selected. + +The drafter keeps a scoring session per conversation +(`PFLASH_DRAFTER_SESSIONS`, default 2, least recently used evicted; 0 scores +every prompt from scratch): the cache of blocks 0-14, the block-15 keys and +the probe logits of the prompt it last scored, with the recurrent state +checkpointed at the last 8192-token grid point at or before 64 tokens from +its end. A prompt that shares that prefix runs only its new tokens through +the drafter, from the checkpoint, and the new query scores against every +stored key. Every call keeps a fresh call's partition (ubatches and key +chunks counted from token 0), so a resumed selection is the one a fresh pass +would make; the query step reads its rows from this call or from an exact F32 +store, and a request that cannot resume exactly is scored from token 0 with +the reason logged (`[qwen35-scorer] redo from token 0`). +With the default `--draft-residency auto` the drafter is released after each +compression; the sessions a call used are copied to host RAM first (plain +pageable memory; `PFLASH_SESSION_OFFLOAD_PINNED=1` uses the device's pinned +host memory instead, which on an R9700 raised peak VRAM by up to 1 GiB near +the 225,280-token cap, kept about 0.5 GiB of VRAM in use between requests and +slowed target prefill), only the positions a resume reads, and when the drafter loads again +only the session the new prompt can resume from comes back to the GPU. A +stored session is restored only under the drafter, head, probe and session +settings it was stored with; anything else drops it, and the call scores +from token 0. +`--draft-residency persistent` keeps the sessions on the GPU and copies +nothing. +A new session is sized with headroom (S + S/2 + 4096 tokens for an S-token +prompt) so later turns append without reallocating, but never past the +engine's `--max-ctx` or the drafter's trained context, and never below the +prompt itself. +A request's `pflash_query` string replaces the derived query and keeps its +whole span; it is meant for benchmarks. `PFLASH_SELECT_QUERY_PARSER=latest_user` +selects the benchmark parser, which finds the latest user message through +sentinel renders. + +`PFLASH_SEGMENT_PROBE_GGUF` loads a segment probe (schema +`qwen3_5_0_8b_segment_probe_v1`): a 264K-parameter network on the same block-14 +tap that scores every token for "a new unit of text starts here". With it +loaded, the context is cut at every boundary above the probe's threshold +(the query start and instruction-span edges are always cut; minimum and +maximum segment lengths come from the GGUF metadata) and the strict selector +ranks the resulting whole functions, classes, files or paragraphs by mass +density, skipping segments that do not fit the remaining budget, so a kept +piece is never a definition cut in half. It falls back to fixed chunks when +the probe finds fewer than four boundaries in a context. +`PFLASH_SELECT_SEGMENTS=auto|fixed|probe` and +`PFLASH_SELECT_SCORE=auto|sum|density` override the defaults (auto = +probe segments and density when a probe is loaded, fixed chunks and mass sum +otherwise); the compression trace records `segmentation`, `candidate_score` +and the segment spans. + +The per-session adaptive keep ratio applies to this path unchanged: a request +carrying a `session_id` retains the session's ratio, the strict selector fills +its token budget from it, and the ratio is updated from the smoothed DFlash +acceptance rate after every turn where speculative decoding ran (below 75% +acceptance retain more, above 85% retain less, 0.5-1 point per turn, bounded +to 2.5-20%; `server/src/server/adaptive_keep_ratio.h`). A new session starts +from the configured ratio for its prompt length (`--prefill-keep-ratio` or +`--prefill-curve`), so the controller adapts around the real-use budget +instead of a fixed 10%. Acceptance is a proxy for compression quality: it +does not detect a dropped answer document directly, so the ratio curve and +the retention benchmarks remain the quality reference. ### Reasoning and MoE controls @@ -540,9 +677,9 @@ drives both arches end-to-end. The only thing the user changes is the model path ```bash cmake --build build --target test_dflash test_laguna_daemon pflash_daemon -j -# 19 GB Q4_K_M target + 1.2 GB Qwen3-0.6B BF16 drafter + tokenizers +# 19 GB Q4_K_M target + ~1.6 GB Qwen3.5-0.8B BF16 drafter + tokenizers hf download Lucebox/Laguna-XS.2-GGUF laguna-xs2-Q4_K_M.gguf --local-dir models/ -hf download unsloth/Qwen3-0.6B-GGUF Qwen3-0.6B-BF16.gguf --local-dir models/ +hf download unsloth/Qwen3.5-0.8B-GGUF Qwen3.5-0.8B-BF16.gguf --local-dir models/ hf download poolside/Laguna-XS.2 --local-dir models/Laguna-XS-2 \ --include 'tokenizer*' '*.json' @@ -564,9 +701,9 @@ LUCE_KV_TYPE=q4_0 ./build/bench_laguna_ttft models/laguna-xs2-Q4_K_M.gguf '4096, # standalone test_laguna_daemon binary so it can run without luce_server. python3 scripts/laguna_pflash_niah.py \ --target models/laguna-xs2-Q4_K_M.gguf \ - --drafter models/Qwen3-0.6B-BF16.gguf \ + --drafter models/Qwen3.5-0.8B-BF16.gguf \ --laguna-tok models/Laguna-XS-2 \ - --drafter-tok Qwen/Qwen3-0.6B \ + --drafter-tok Qwen/Qwen3.5-0.8B \ --pflash-bin ./build/pflash_daemon \ --laguna-bin ./build/test_laguna_daemon \ --ctx 131072 --depth 0.5 --keep 0.10 --target-kv q4_0 diff --git a/server/docs/DS4.md b/server/docs/DS4.md index 29b9a7729..46cde365a 100644 --- a/server/docs/DS4.md +++ b/server/docs/DS4.md @@ -1005,7 +1005,7 @@ park a target while it owns live sequence state. ./server/build-hip/luce_server /path/to/deepseek4-target.gguf \ --target-device hip:0 \ --prefill-compression auto \ - --prefill-drafter /path/to/Qwen3-0.6B-BF16.gguf \ + --prefill-drafter /path/to/Qwen3.5-0.8B-BF16.gguf \ --prefill-skip-park ``` diff --git a/server/docs/ENVIRONMENT.md b/server/docs/ENVIRONMENT.md index 4ea943c66..e92cf0556 100644 --- a/server/docs/ENVIRONMENT.md +++ b/server/docs/ENVIRONMENT.md @@ -435,7 +435,5 @@ consolidation of this list into CLI flags is tracked as follow-up work. - `LUCE_PC_DEEP_FIRST_MIN` - prefix_cache.cpp - `LUCE_QK_FUSE_LAYERS` - laguna_target_graph.cpp - `LUCE_QK_FUSE_MODE` - laguna_target_graph.cpp -- `PFLASH_DRAFTER_EARLY_EXIT_N` - qwen3_graph.cpp -- `PFLASH_DRAFTER_SCORE_LAYERS` - qwen3_graph.cpp - `PFLASH_FREEZE_HOT_WINDOW` - http_server.cpp - `TMPDIR` - backend_ipc.cpp, moe_expert_compute_ipc.cpp diff --git a/server/docs/SPEC_PREFILL.md b/server/docs/SPEC_PREFILL.md index 171541778..a4cc3a38f 100644 --- a/server/docs/SPEC_PREFILL.md +++ b/server/docs/SPEC_PREFILL.md @@ -60,7 +60,7 @@ PFlash phase or DFlash draft-process boundary. See ## Performance NIAH single-needle end-to-end on RTX 3090 (Qwen3.6-27B Q4_K_M target, -Qwen3-0.6B drafter, in-process daemon, `LUCE_FP_USE_BSA=1`, +Qwen3.5-0.8B drafter, in-process daemon, `LUCE_FP_USE_BSA=1`, `LUCE_FP_ALPHA=0.85`, `keep_ratio=0.05`): | Source S | dflash TTFT | llama.cpp baseline | Speedup | NIAH | @@ -81,10 +81,15 @@ src/ flashprefill_select.cpp Host fallback for block_select (rarely used) bsa_launcher.cu BSA launcher: blockmask conversion + Flash_fwd_params bsa_fwd_inst.cu Single-TU instantiation of BSA's hdim128 kernel - qwen3/ Qwen3-0.6B drafter model code + pflash/ PFlash drafter (Qwen3.5-0.8B scorer) code + qwen35_loader.cpp GGUF → Qwen3.5-0.8B weights + scoring head + probe + qwen35_drafter.{h,cpp} block-15 head scorer + legacy running-max scorer + pflash_drafter.{h,cpp} drafter_score_and_compress() entry point + pflash_compress.{h,cpp} scores → strict selection → compressed ids + pflash_selection.{h,cpp} strict budget selection + segment probing + qwen3/ Qwen3-0.6B standalone inference (not the drafter) qwen3_loader.cpp GGUF → Qwen3-0.6B BF16 weight tensors - qwen3_graph.cpp Custom Qwen3-0.6B forward (per-layer A/FP/B graphs) - qwen3_drafter.{h,cpp} drafter_score_and_compress() entry point + qwen3_backend.{h,cpp} step forward + ModelBackend qwen35/ Qwen3.5/3.6 target + DFlash draft model code qwen35_target_graph.cpp Qwen3.5/3.6 target graph (ggml) gguf_target_loader.cpp Qwen3.5 target GGUF loader diff --git a/server/docs/laguna_integration_plan.md b/server/docs/laguna_integration_plan.md index 6883d10ff..d1ed4295c 100644 --- a/server/docs/laguna_integration_plan.md +++ b/server/docs/laguna_integration_plan.md @@ -4,7 +4,7 @@ Status: scaffolding. PR #115 in lucebox-hub bumps llama.cpp submodule to `luce-d ## Context -- `pflash_daemon` (test/smoke/pflash_daemon.cpp): drafter-only stdin compressor, loads Qwen3-0.6B via luce's own loader, emits compressed token IDs in DRAFTER vocab. Already model-agnostic on the target side. **No change needed.** +- `pflash_daemon` (test/smoke/pflash_daemon.cpp): drafter-only stdin compressor, loads Qwen3.5-0.8B via luce's own loader, emits compressed token IDs in DRAFTER vocab. Already model-agnostic on the target side. **No change needed.** - `test_dflash` (test/unit/test_dflash.cpp 190 KB): main target runner. Hand-rolled CUDA forward graph for qwen35 hybrid. Loads via `load_target_gguf` which hardcodes `arch == "qwen35"`. **Hard-blocked on Laguna.** - `qwen35_target_graph.cpp` (60 KB): hand-rolled CUDA forward, builds full-attn + delta-net + FFN. Uses `flash_prefill_forward_bf16` for sparse prefill. - `flashprefill.{h,cpp}` + `flashprefill_kernels.cu`: model-agnostic block-sparse FA. Takes Q/K/V tensors, returns O. Already works for any GQA arch with head_dim 128. **Reusable as-is.** @@ -57,7 +57,7 @@ No libllama dependency in dflash runtime. Keep ggml-only stack. (libllama+LAGUNA - Detect arch from loaded weights - For Laguna arch, use `LagunaTargetCache` + `build_laguna_graph` instead of qwen35 equivalents - Adjust per-layer-head-count in attention buffer sizing - - PFlash drafter call unchanged (drafter is Qwen3-0.6B regardless of target) + - PFlash drafter call unchanged (drafter is Qwen3.5-0.8B regardless of target) - Cross-tokenizer mapping (Qwen3 IDs → Laguna IDs): byte-level round-trip via existing optimizations/pflash/ Python module OR port to C++ helper ## Phasing diff --git a/server/scripts/gen_request_probe_v2_fixtures.py b/server/scripts/gen_request_probe_v2_fixtures.py new file mode 100644 index 000000000..d115aa825 --- /dev/null +++ b/server/scripts/gen_request_probe_v2_fixtures.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Generate test/unit/request_probe_v2_parity.inc: replay-parity fixtures for the V2-A3 candidate rule against the +trainer's offline mirror rule.probe_query_v2 (luce_box lucebox_training/pflash, CANDIDATE_RULE +cand_v3_latest_human_prev8_plus_all_nonsystem_turns, V2-A5). Deterministic (seed 0). + python3 gen_request_probe_v2_fixtures.py TRAINER_SRC_DIR OUT.inc""" +import random +import sys + +import numpy as np + +sys.path.insert(0, sys.argv[1]) +from pflash_scorer.request_probe import contract # noqa: E402 +from pflash_scorer.request_probe.rule import probe_query_v2 # noqa: E402 + +assert contract.CANDIDATE_RULE == "cand_v3_latest_human_prev8_plus_all_nonsystem_turns" +rng = random.Random(0) +cases = [] +for case in range(80): + roles = ["system"] if rng.random() < 0.7 else [] + n_msgs = rng.randint(1, 9) + for _ in range(n_msgs): + roles.append(rng.choice(["user", "user", "assistant", "tool", "tool"])) + big = case % 8 == 0 + spans, pos = [], 0 + for r in roles: + L = rng.randint(0, 900 if big else 60) + spans.append((pos, pos + L)) + pos += L + rng.randint(1, 4) + n = pos + 3 + hits = [] + lg = np.full(n, -1.0, np.float32) + for b, e in spans: + k = b + while k < e: + if rng.random() < (0.35 if big else 0.15): + L = rng.randint(1, 12 if not big else 400) + z = min(e, k + L) + hits.append((k, z)) + lg[k:z] = 1.0 + k = z + rng.randint(1, 5) + else: + k += rng.randint(1, 8) + q, _runs, info = probe_query_v2(lg, 0.0, list(zip(roles, spans))) + qs = [] + for t in q: + if qs and qs[-1][1] == t: + qs[-1][1] = t + 1 + else: + qs.append([t, t + 1]) + cases.append((roles, spans, n, hits, qs, info["candidates"])) + +with open(sys.argv[2], "w") as f: + import hashlib + import os + src = os.path.join(sys.argv[1], "pflash_scorer", "request_probe") + shas = {n: hashlib.sha256(open(os.path.join(src, n), "rb").read()).hexdigest()[:16] for n in ("rule.py", "contract.py")} + f.write("// Generated by server/scripts/gen_request_probe_v2_fixtures.py from the trainer's rule.probe_query_v2\n") + f.write("// trainer sources: rule.py %s, contract.py %s\n" % (shas["rule.py"], shas["contract.py"])) + f.write("// (luce_box v2 branch, CANDIDATE_RULE cand_v3_latest_human_prev8_plus_all_nonsystem_turns). Do not edit.\n") + for i, (roles, spans, n, hits, qs, cands) in enumerate(cases): + f.write("{%d, {%s}, {%s}, {%s}, {%s}, {%s}},\n" % ( + n, ", ".join('"%s"' % r for r in roles), ", ".join("{%d, %d}" % s for s in spans), + ", ".join("{%d, %d}" % h for h in hits), ", ".join("{%d, %d}" % tuple(x) for x in qs), + ", ".join(str(c) for c in cands))) +print(sys.argv[2], len(cases), "cases") diff --git a/server/scripts/laguna_pflash_niah.py b/server/scripts/laguna_pflash_niah.py index c936ab930..a075e735d 100644 --- a/server/scripts/laguna_pflash_niah.py +++ b/server/scripts/laguna_pflash_niah.py @@ -23,9 +23,9 @@ Usage: python3 laguna_pflash_niah.py \\ --target /path/to/laguna-xs2-Q4_K_M.gguf \\ - --drafter /path/to/Qwen3-0.6B-BF16.gguf \\ + --drafter /path/to/Qwen3.5-0.8B-BF16.gguf \\ --laguna-tok /path/to/Laguna-XS.2 \\ - --drafter-tok /path/to/Qwen3-0.6B \\ + --drafter-tok /path/to/Qwen3.5-0.8B \\ --pflash-bin /path/to/pflash_daemon \\ --laguna-bin /path/to/test_laguna_daemon \\ --ctx 16384 --depth 0.5 --keep 0.10 @@ -308,9 +308,9 @@ def close(self): def main(): ap = argparse.ArgumentParser() ap.add_argument("--target", required=True, type=Path, help="Laguna GGUF") - ap.add_argument("--drafter", required=True, type=Path, help="Qwen3-0.6B drafter GGUF") + ap.add_argument("--drafter", required=True, type=Path, help="Qwen3.5-0.8B drafter GGUF") ap.add_argument("--laguna-tok", required=True, type=Path, help="Laguna HF dir with tokenizer.json") - ap.add_argument("--drafter-tok", required=True, type=Path, help="Qwen3 HF dir with tokenizer.json") + ap.add_argument("--drafter-tok", required=True, type=Path, help="Qwen3.5 HF dir with tokenizer.json") ap.add_argument("--pflash-bin", required=True, type=Path) ap.add_argument("--laguna-bin", required=True, type=Path) ap.add_argument("--ctx", type=int, default=16384) diff --git a/server/scripts/pflash_request_probe_capture.py b/server/scripts/pflash_request_probe_capture.py new file mode 100644 index 000000000..683b5da63 --- /dev/null +++ b/server/scripts/pflash_request_probe_capture.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +"""Request-probe FEATURE CAPTURE driver (SPEC A6): block-14 rows from the pinned runtime. + +Runs the drafter-only pflash_daemon in the SERVED config (Q4_0 KV default, segment probe + u314 head, strict +selection) with the request probe on, one conversation per FRESH session (PFLASH_DRAFTER_SESSIONS=0) over its full +causal prefix, and keeps the block-14 rows of user-role and tool tokens in F32 (exactly the rows the served trunk +reads, rather than bf16) plus the runtime's own request logits (f32, the A6 probe). The runtime writes each call's rows with PFLASH_TRACE_REQUEST_FEATURES +(record "RPF2": i32 resume, n_new, hidden, f32 rows, f32 logits). + +Input JSONL, one conversation per line (the A6 kit format works as is): + {"conversation_id" | "id": str, "ids": [drafter token ids, the full rendered conversation], + "messages": [{"index": int, "role": "user"|"tool"|"assistant"|"system"|..., "span": [b, e)}, ...]} + role "user" = human (the request probe's role rule); role "tool" = tool results / observations. + A message counts as labelled when it has "request_spans" or "labelled": true, else every user/tool message does. + +Output (OUT dir, possibly on another host via --remote host): shard-NNNNN.rows.f32 (float32 [rows, hidden]), +shard-NNNNN.logits.f32 ([rows]), shard-NNNNN.index.jsonl (per conversation: id, n_tokens, row_offset, rows, +messages [{index, role, span, row_offset}], tokens = merged captured token spans in row order), part-*.json summary. +Rows of a conversation are its user/tool tokens in ascending token order. + + capture.py --daemon BIN --drafter GGUF --scoring-head GGUF --segment-probe GGUF --request-probe GGUF + --input convs.jsonl --out DIR [--part i/n] [--remote HOST] [--shard-mb 1024] +""" +import argparse, hashlib, json, os, shutil, struct, subprocess, sys, tempfile, time +from pathlib import Path +import numpy as np + +# Runtime artifacts, set from the command line in main(). +DAEMON = DRAFTER = HEAD = SEGPROBE = REQPROBE = None +CAPTURE_ROLES = ("user", "tool") + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for b in iter(lambda: f.read(1 << 22), b""): + h.update(b) + return h.hexdigest() + + +def merged(spans): + out = [] + for b, e in sorted(spans): + if out and b <= out[-1][1]: + out[-1][1] = max(out[-1][1], e) + elif e > b: + out.append([b, e]) + return out + + +class Daemon: + def __init__(self, workdir, env_extra): + self.dump = Path(workdir) / "features.rpf" + env = {k: v for k, v in os.environ.items() if not k.startswith(("PFLASH_", "LUCE_KV"))} + env.update({ + "PFLASH_SCORING_HEAD_GGUF": str(HEAD), "PFLASH_SEGMENT_PROBE_GGUF": str(SEGPROBE), + "PFLASH_SELECT_MODE": "budget_only", "PFLASH_SELECT_CHUNK_SIZE": "256", "PFLASH_SELECT_SCORE": "density", + "PFLASH_SELECT_SEGMENTS": "probe", "PFLASH_SELECT_QUERY_PARSER": "latest_user", + "PFLASH_SELECT_QUERY_TOKENS": "128", "PFLASH_SELECT_PARAGRAPH_JOIN": "0", + "PFLASH_SELECT_QUERY_PROBE": "1", "PFLASH_REQUEST_PROBE_GGUF": str(REQPROBE), + "PFLASH_DRAFTER_SESSIONS": "0", "PFLASH_DAEMON_KEEP_X1000": "1000", + "PFLASH_TRACE_REQUEST_FEATURES": str(self.dump), + }) + env.update(env_extra) + self.env = env + rfd, wfd = os.pipe() + self.log = open(Path(workdir) / "daemon.log", "a") + self.proc = subprocess.Popen([str(DAEMON), str(DRAFTER), f"--stream-fd={wfd}"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=self.log, env=env, + pass_fds=(wfd,), text=True) + os.close(wfd) + self.stream = os.fdopen(rfd, "rb") + if "ready" not in self._line(["ready"]): + raise RuntimeError("pflash_daemon did not start (see daemon.log)") + + def _line(self, words): + while True: + line = self.proc.stdout.readline() + if not line: + return "" + if line.startswith("[pflash-daemon]") and any(w in line for w in words): + return line.strip() + + def capture(self, ids, human, path): + """Rows [S, hidden] f32 and request logits [S] of the whole prompt, from a fresh session.""" + self.dump.unlink(missing_ok=True) + with open(path, "wb") as f: + f.write(struct.pack("