diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index bb995815f..0e605b3f2 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -747,6 +747,7 @@ add_library(luce_common STATIC src/server/json_schema_grammar.cpp src/server/tool_hint.cpp src/server/tool_memory.cpp + src/server/pflash_unified.cpp src/server/response_error.cpp src/server/sse_emitter.cpp src/server/prefix_cache.cpp @@ -2482,6 +2483,7 @@ if(LUCE_TESTS) test/pflash/test_pflash_drafter_ipc.cpp test/pflash/test_pflash_selection.cpp test/pflash/test_pflash_fold.cpp + test/pflash/test_pflash_unified.cpp test/pflash/test_pflash_request_probe.cpp test/pflash/test_pflash_request_stage2.cpp test/pflash/test_pflash_query_candidates.cpp diff --git a/server/README.md b/server/README.md index 539a06fdc..b4a2a38d1 100644 --- a/server/README.md +++ b/server/README.md @@ -390,7 +390,8 @@ the whole request's device footprint. `/status/json` reports With a trained scoring head (`PFLASH_SCORING_HEAD_GGUF`), every PFlash setting left unset takes the measured release profile: threshold 16384, the curve `16384:0.315 32768:0.270 65536:0.270 120000:0.225`, budget selection over probe -segments and the request-probe query. Without a head the built-in defaults +segments, the request-probe query and the unified view for chats and agent +sessions (`PFLASH_UNIFIED=1`, below). Without a head the built-in defaults below apply. The [PFlash configuration reference](docs/PFLASH_CONFIGURATION.md) lists both, the validated overrides and the active experiments. @@ -507,6 +508,51 @@ drafter selection (up to 48 s more per follow-up in a 10-turn coding session on an R9700; the session took 281 s against 258 s with full prefill). Chats without tools are not affected. +`PFLASH_UNIFIED=1` (on in the release profile, which also gives it the +strict-selection environment and `PFLASH_ADMISSION=cache` it needs; +`PFLASH_UNIFIED=0` turns it off) serves chat and agent sessions with +one mechanism, the unified view. Only the newest turn is ever paged: the user +messages since the last reply, or the tool results of the last tool call. A +message there with at least `--prefill-threshold` droppable tokens that can be +cut into lines becomes a region, and only what the drafter selection keeps of +it is served, in whole lines, with a note saying the content was shortened. +Tool outputs are paged for the tools in `PFLASH_REGION_TOOLS` (default `*`, +every tool; a comma-separated list names tools; empty pages none). Everything +before the newest turn is the view already served and stays cached: the +served prompt only grows by appending, and is never compressed again. A new +user question recalls, as excerpts at the start of its message, the lines of +earlier regions it needs that the view does not hold. Recall is gated: the +drafter scores the context for the question, and when the units the view does +not hold carry less than `PFLASH_RECALL_GATE` of that score mass (default +0.075, frozen from a development replay as the largest gate whose answers stay +within 3 of ungated recall; `off` recalls by the regions' caps alone) nothing +is appended; otherwise the best unheld units are recalled until what stays +unheld is below the gate. `PFLASH_RECALL=0` turns recall off. The request +probe's pins (the question it finds and its recency pin) stay whole, as in the +chat view. When the drafter pass fails, the newest content is served verbatim +and the view continues; `PFLASH_FAIL_CLOSED=1` refuses the request instead. A +re-read of a paged tool output is served verbatim. A view that would no longer fit the +context starts over from the raw prompt. + +Requests the unified view does not take are handed back to the chat view +described above and served exactly as without the flag: a request without a +messages array, a remote drafter, a turn structure or generation prompt it +cannot read, and a conversation it would start (no stored view continues it) +whose uncached history before the newest turn, with newest content that +cannot be cut into lines, reaches `--prefill-threshold` (reason `uncached`, +or `reset_uncached` when a stored view could not continue), and a selection +that gives no per-candidate scores (`unscored_selection`: the legacy or split +scorer, `PFLASH_QWEN35_LEGACY_SCORER=1`, or the fold strategy). A conversation +handed back, or served raw after a PFlash failure, loses its unified views, so +no later turn continues one. Such a conversation stays with the chat view while its history is +uncached. The view trace records `pflash.unified = {decision: handback, +reason}` for these requests. Without a strict-selection environment the +unified view serves nothing (startup says so) and every request takes the +chat view. +`PFLASH_AGENT_PASTE`, `PFLASH_ARRIVE`, `PFLASH_CHAT_RECALL` and the other +chat-view knobs apply only to handed-back requests. `PFLASH_RECALL_SOURCE` +was removed. + Every turn of a multi-turn chat keeps its role header, and user turns (the latest included) and assistant answers up to `PFLASH_CHAT_SKELETON_TOKENS` (default 256 drafter tokens; 0 keeps headers only) stay whole: the diff --git a/server/docs/PFLASH_CONFIGURATION.md b/server/docs/PFLASH_CONFIGURATION.md index 2b8b941cc..d1af9fcdc 100644 --- a/server/docs/PFLASH_CONFIGURATION.md +++ b/server/docs/PFLASH_CONFIGURATION.md @@ -25,6 +25,9 @@ allowed below. Valid integer overrides retain their existing upper clamps. | `--prefill-keep-ratio` | `0.05` | curve | Flat ratio; `--prefill-curve tokens:ratio ...` overrides it. The release curve is `16384:0.315 32768:0.270 65536:0.270 120000:0.225`; either flag replaces it | | `--prefill-skip-park` | off | | Keep the target resident while the drafter scores; see the release profile | | `PFLASH_ADMISSION` | `length` | `cache` | `length` or `cache`; cache admission accounts for restorable target KV | +| `PFLASH_UNIFIED` | `0` | `1` | One view and recall mechanism for chats and agent sessions: only the newest turn's long uncached content is paged, in whole lines. Needs a strict-selection environment; the chat-view settings below apply only to the requests it hands back. See the [server README](../README.md#pflash) | +| `PFLASH_REGION_TOOLS` | `*` | | Under the unified view: tools whose outputs may be paged; a comma-separated list names tools, empty pages none | +| `PFLASH_RECALL`, `PFLASH_RECALL_GATE` | `1`, `0.075` | | Under the unified view: recall for a new question, and the share of score mass outside the view below which nothing is recalled (`off` recalls up to the regions' caps) | | `PFLASH_AGENT_PASTE` | `0` | | With cache admission in auto mode, `1` permits a large newest user paste in a tool session; earlier history stays raw | | `PFLASH_CHAT_VIEW`, `PFLASH_CHAT_RECALL` | `1`, `1` | | Continue the served view; recall omitted material for a new question | | `PFLASH_CHAT_COMPRESS_NEW_TOKENS` | `16384` | | Integer >= 1; smaller additions stay verbatim; capped at 2^30 | diff --git a/server/docs/PREFIX_CACHE.md b/server/docs/PREFIX_CACHE.md index 198f98c8d..371edbc4d 100644 --- a/server/docs/PREFIX_CACHE.md +++ b/server/docs/PREFIX_CACHE.md @@ -150,9 +150,28 @@ When a session's state is written, its shallower files on disk are deleted, except a head at or below `--kv-cache-cold-max` tokens or, without one, the shallowest (a longer head). -Compressed prompts (PFlash, FlowKV) are not raw history and never get a disk -key; a request whose disk scope is not `full` gives its snapshots no key -either. +Compressed prompts (PFlash, FlowKV) are not raw history and get no disk key, +except a prompt served from a PFlash unified view (`PFLASH_UNIFIED=1`): its +key is the tokens it served, and the view itself is kept on disk too, so the +conversation's next request serves the same tokens again and restores them. A +request whose disk scope is not `full` gives its snapshots no key either. + +**Shortened conversations.** With the disk cache on, the unified view keeps its +views (what each shortened conversation served, and which lines it left out) in +`/.pflash-views`: a view is written when it leaves the RAM view +store and at shutdown, and read when a request finds no view in RAM, so neither +a restart nor an eviction turns a shortened conversation cold. One file per +view, a one-line header indexed at startup; at most 256 views, the least +recently written dropped first, and the cache's TTL applies, counted from a +view's last write and checked at startup and at every lookup (a view in use +returns to RAM and is written again when it leaves). Views written for another +model, template or context (the cache's identity salt) are never read. A view +file that does not load, or whose offsets and per-line and per-unit data do not +agree, is deleted and the request proceeds without it. A conversation served +raw after a PFlash failure, or handed to the chat view, loses its views in RAM +and on disk, so no later turn continues them. +The drafter's own scoring state stays in RAM only: after a restart, the first +question that needs recall scores the earlier messages again. **How it is written.** Eviction hands the snapshot's buffer to the disk writer without a copy (Qwen keeps snapshots in host memory; other backends copy to @@ -217,11 +236,17 @@ them go through the same asynchronous writer. the concurrent `qwen4exp` engine keeps its checkpoints outside the backend's snapshot slots: both run without a disk cache (logged at startup). -**Follow-up: cache-aware admission.** Compression admission that counts the -cached prefix as warm (on the PFlash branches, not on `main`) consults RAM -only. `DiskPrefixCache::longest_prefix(prompt)` returns the deepest restorable -key length and KV position without reading a file, so admission can take the -max of the RAM and disk prefixes. +**Cache-aware admission.** PFlash's admission counts the deeper of the RAM +prefix snapshot and the deepest disk entry for the prompt +(`DiskPrefixCache::longest_prefix`, an index lookup that reads no file; ties +go to RAM), in the chat view (`PFLASH_ADMISSION=cache`) and in the unified view +(`PFLASH_UNIFIED=1`). A conversation restored after a restart, or written out +on eviction, is then cached history: the unified view serves it raw, from disk, +and pages only new long content, instead of handing a cold history to the chat +view. The disk counts only with `--kv-cache-dir` set and the request's disk +scope `full`. With the disk cache on, the chat view's admission record carries +`cached_source` (`ram`, `disk` or `none`) and the unified view's stats a +`cached` object with the tokens and source. ## Snapshot Memory Management diff --git a/server/src/common/model_backend.h b/server/src/common/model_backend.h index f64d91313..e19bd49ae 100644 --- a/server/src/common/model_backend.h +++ b/server/src/common/model_backend.h @@ -516,6 +516,8 @@ struct ModelBackend { // Strict selection with the head: every candidate's attention lift // (mass per token relative to uniform); empty when unknown. std::vector> candidate_lifts; + // The request probe's own pins in that selection (pflash_last_probe_pins). + std::vector probe_pins; int scorer_resume = -1; int scorer_new_tokens = -1; double scorer_forward_s = 0.0; diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index ad8776df7..db0eec86c 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -5740,6 +5740,7 @@ std::vector DeepSeek4Backend::compress_batch( for (const auto & candidate : pflash_last_candidate_lifts()) { result.candidate_lifts.push_back({candidate.span, candidate.lift}); } + result.probe_pins = pflash_last_probe_pins(); } } diff --git a/server/src/pflash/pflash_compress.cpp b/server/src/pflash/pflash_compress.cpp index 97385000d..20b08a71b 100644 --- a/server/src/pflash/pflash_compress.cpp +++ b/server/src/pflash/pflash_compress.cpp @@ -469,6 +469,7 @@ namespace { thread_local std::vector g_last_kept_spans; thread_local PFlashScoringStats g_last_scoring_stats; thread_local std::vector g_last_candidate_lifts; +thread_local std::vector g_last_probe_pins; thread_local PFlashStage2Outcome g_last_stage2_outcome; thread_local PFlashRelevanceSpread g_last_relevance_spread; } // namespace @@ -520,6 +521,14 @@ const std::vector & pflash_last_candidate_lifts() { return g_last_candidate_lifts; } +const std::vector & pflash_last_probe_pins() { + return g_last_probe_pins; +} + +void pflash_set_probe_pins(std::vector pins) { + g_last_probe_pins = std::move(pins); +} + const PFlashScoringStats & pflash_last_scoring_stats() { return g_last_scoring_stats; } @@ -536,6 +545,7 @@ void pflash_clear_kept_spans() { g_last_kept_spans.clear(); g_last_scoring_stats = {}; g_last_candidate_lifts.clear(); + g_last_probe_pins.clear(); g_last_stage2_outcome = {}; g_last_relevance_spread = {}; } diff --git a/server/src/pflash/pflash_compress.h b/server/src/pflash/pflash_compress.h index 322934ea3..26c1211c0 100644 --- a/server/src/pflash/pflash_compress.h +++ b/server/src/pflash/pflash_compress.h @@ -188,6 +188,13 @@ struct PFlashCandidateLift { }; const std::vector & pflash_last_candidate_lifts(); +// The pins the request probe added to the last strict selection on this +// thread: the query tokens it picked and its recency pin (not the caller's +// pins, nor PFLASH_QUERY_PIN's top units). A caller that selects by the +// candidate lifts itself keeps them. Cleared with the kept spans. +const std::vector & pflash_last_probe_pins(); +void pflash_set_probe_pins(std::vector pins); + // Relevance spread of one selection (PFLASH_FALLBACK_SPREAD): with // p_i = s_i / sum(s) over every candidate's score (the trace's // ``chunk_scores``), H = -sum_{p_i > 0} p_i ln p_i and k = the selected diff --git a/server/src/pflash/qwen35_drafter.cpp b/server/src/pflash/qwen35_drafter.cpp index 5f663ac89..29cc60e31 100644 --- a/server/src/pflash/qwen35_drafter.cpp +++ b/server/src/pflash/qwen35_drafter.cpp @@ -3024,6 +3024,11 @@ static std::vector strict_score_impl( request_trace.mandatory_tokens = charged(pins, segments); request_trace.mandatory_tokens_base = request_trace.mandatory_tokens; } + if (request_set) { + auto probe_pins = luce::pflash::pflash_request_token_spans(request_trace.set.tokens); + if (request_trace.recency_pin.begin >= 0) probe_pins.push_back(request_trace.recency_pin); + pflash_set_probe_pins(std::move(probe_pins)); + } std::string policy_note; if (request_trace.policy_evaluated) { char buf[256]; diff --git a/server/src/qwen35/qwen35_backend.cpp b/server/src/qwen35/qwen35_backend.cpp index c434662bb..d303cb14f 100644 --- a/server/src/qwen35/qwen35_backend.cpp +++ b/server/src/qwen35/qwen35_backend.cpp @@ -1509,6 +1509,7 @@ std::vector Qwen35Backend::compress_batch( for (const auto & candidate : pflash_last_candidate_lifts()) { result.candidate_lifts.push_back({candidate.span, candidate.lift}); } + result.probe_pins = pflash_last_probe_pins(); } if (result.ok) { std::fprintf(stderr, "[compress] %zu -> %zu tokens\n", diff --git a/server/src/qwen35/qwen35_layer_split_adapter.cpp b/server/src/qwen35/qwen35_layer_split_adapter.cpp index 5aa224d2b..e709d3b67 100644 --- a/server/src/qwen35/qwen35_layer_split_adapter.cpp +++ b/server/src/qwen35/qwen35_layer_split_adapter.cpp @@ -1416,6 +1416,7 @@ Qwen35LayerSplitAdapter::compress(const ModelBackend::CompressRequest & req) { for (const auto & candidate : pflash_last_candidate_lifts()) { result.candidate_lifts.push_back({candidate.span, candidate.lift}); } + result.probe_pins = pflash_last_probe_pins(); } if (result.ok) { std::fprintf(stderr, "[target-split][compress] %zu -> %zu tokens\n", diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index a51003145..c181b2fd2 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -1236,6 +1236,14 @@ int pflash_raw_base_target( return target_text == drafter_text ? cut : -1; } +PflashRestorablePrefix pflash_restorable_prefix( + int ram_tokens, bool disk_enabled, int disk_tokens) noexcept { + const int ram = (std::max)(0, ram_tokens); + const int disk = disk_enabled ? (std::max)(0, disk_tokens) : 0; + if (disk > ram) return {disk, "disk"}; + return {ram, ram > 0 ? "ram" : "none"}; +} + int pflash_span_tokens_within( const std::vector & spans, int begin, int end) noexcept { int tokens = 0; @@ -1317,6 +1325,13 @@ bool pflash_fail_closed_from_env() noexcept { return value != nullptr && *value && std::strcmp(value, "0") != 0; } +bool pflash_selection_gives_lifts(const luce::pflash::PFlashSelectionConfig & selection) noexcept { + const char * legacy = std::getenv("PFLASH_QWEN35_LEGACY_SCORER"); + return selection.scorer == luce::pflash::PFlashScorer::Head && + selection.strategy == luce::pflash::PFlashSelectStrategy::Segments && + !(legacy && std::strcmp(legacy, "1") == 0); +} + std::string pflash_failure_reason(const std::string & error, const std::string & detail) { const auto has = [](const std::string & text, const char * needle) { @@ -2706,6 +2721,121 @@ HttpServer::HttpServer(luce::engine::LuceEngine & engine, restore_replay_blobs(); status_html_path_ = resolve_status_html(); + // PFLASH_UNIFIED=1: read once here; a bad tool list or an unsupported variant fails startup. + // Defaults under the flag: every tool's output may be paged (PFLASH_REGION_TOOLS unset = "*"; set it empty for + // none) and recall is gated at theta 0.075 (PFLASH_RECALL_GATE; "off" = ungated recall). + if (const char * unified = std::getenv("PFLASH_UNIFIED"); unified && std::strcmp(unified, "1") == 0) { + const auto fail = [] (const std::string & message) { + std::fprintf(stderr, "[pflash-unified] ERROR: %s\n", message.c_str()); + std::fflush(stderr); + throw std::runtime_error(message); + }; + pflash_unified_ = true; + std::string tools_error; + const char * tools = std::getenv("PFLASH_REGION_TOOLS"); + if (!pflash_unified::parse_tool_list(tools ? tools : "*", pflash_region_tools_, tools_error)) { + fail(tools_error); + } + for (const char * arm : {"PFLASH_UNIFIED_INPLACE", "PFLASH_UNIFIED_DEFRAG"}) { + if (const char * value = std::getenv(arm); value && *value) { + fail(std::string(arm) + " is not supported"); + } + } + if (!luce::pflash::has_pflash_selection_environment()) { + std::fprintf(stderr, "[pflash-unified] no strict-selection environment: every request is served by the " + "chat view, as without PFLASH_UNIFIED\n"); + } else { + luce::pflash::PFlashSelectionConfig selection; + std::string selection_error; + if (luce::pflash::resolve_pflash_selection(1 << 16, 32, selection, selection_error) && + !http_detail::pflash_selection_gives_lifts(selection)) { + std::fprintf(stderr, "[pflash-unified] scorer %s, strategy %s give no per-candidate scores: every " + "request is served by the chat view, as without PFLASH_UNIFIED\n", + luce::pflash::pflash_scorer_name(selection.scorer), + luce::pflash::pflash_select_strategy_name(selection.strategy)); + } + } + const char * recall = std::getenv("PFLASH_RECALL"); + pflash_unified_recall_ = !(recall && std::strcmp(recall, "0") == 0); + // The default theta was frozen from the dev replay (largest theta whose served gold stays within 3 of + // ungated recall); see the recall-gate notes in server/README.md. + pflash_recall_gate_ = 0.075; + bool gate_given = false; + if (const char * gate = std::getenv("PFLASH_RECALL_GATE"); gate && *gate) { + if (std::strcmp(gate, "off") == 0) { + pflash_recall_gate_ = -1.0; + } else { + gate_given = true; + char * end = nullptr; + const double theta = std::strtod(gate, &end); + if (end == gate || *end != '\0' || !std::isfinite(theta) || theta <= 0.0 || theta >= 1.0) { + fail(std::string("PFLASH_RECALL_GATE must be a number in (0, 1) or 'off', got '") + gate + "'"); + } + pflash_recall_gate_ = theta; + } + } + if (const char * source = std::getenv("PFLASH_RECALL_SOURCE"); source && *source) { + fail("PFLASH_RECALL_SOURCE was removed: recall takes the unheld units, gated by PFLASH_RECALL_GATE"); + } + if (const char * trace = std::getenv("PFLASH_RECALL_TRACE_UNITS"); trace && *trace) { + if (std::strcmp(trace, "1") != 0 && std::strcmp(trace, "0") != 0) { + fail(std::string("PFLASH_RECALL_TRACE_UNITS must be 0 or 1, got '") + trace + "'"); + } + pflash_recall_trace_units_ = std::strcmp(trace, "1") == 0; + } + if (gate_given && !pflash_unified_recall_) { + fail("PFLASH_RECALL_GATE needs recall on (PFLASH_RECALL=0 is set)"); + } + if (!pflash_unified_recall_) pflash_recall_gate_ = -1.0; + // Knobs of the legacy chat view and tool-session rules: they apply only to the requests the unified view + // hands back to that path (apply_unified_view), named once. + for (const char * legacy : {"PFLASH_AGENT_PASTE", "PFLASH_ARRIVE", "PFLASH_CHAT_RECALL", + "PFLASH_CHAT_COMPRESS_NEW_TOKENS", "PFLASH_CHAT_REBUILD_MIN_TOKENS", + "PFLASH_CHAT_REBUILD_RATIO", "PFLASH_SELECT_PARAGRAPH_JOIN"}) { + if (std::getenv(legacy)) { + std::fprintf(stderr, "[pflash-unified] %s applies only to requests handed back to the chat view\n", + legacy); + } + } + const auto env = [] (const char * name) { + const char * value = std::getenv(name); + return std::string(value ? value : ""); + }; + pflash_unified_config_ = "unified-v1|tools=" + pflash_region_tools_.identity + + "|recall=" + (pflash_unified_recall_ ? "1" : "0") + + (pflash_recall_gate_ > 0.0 ? "|recall_gate=" + std::to_string(pflash_recall_gate_) : std::string()) + + "|threshold=" + std::to_string(config_.pflash_threshold) + + "|drafter=" + config_.pflash_drafter_path + + "|head=" + env("PFLASH_SCORING_HEAD_GGUF") + "|probe=" + env("PFLASH_SEGMENT_PROBE_GGUF") + + "|request_probe=" + env("PFLASH_REQUEST_PROBE_GGUF") + + "|curve="; + for (int tokens : {16384, 32768, 65536, 131072}) { + pflash_unified_config_ += std::to_string(pflash_keep_ratio(config_, tokens)) + ","; + } + char gate_text[32] = "off"; + if (pflash_recall_gate_ > 0.0) std::snprintf(gate_text, sizeof gate_text, "%g", pflash_recall_gate_); + std::fprintf(stderr, "[pflash-unified] on: region tools=%s recall=%d recall_gate=%s\n", + pflash_region_tools_.identity.empty() ? "(none)" : pflash_region_tools_.identity.c_str(), + (int) pflash_unified_recall_, gate_text); + // With the disk session cache on, the views of shortened conversations are kept on disk too: written when + // they leave the RAM store and at shutdown, read on a RAM miss, so neither a restart nor an eviction turns a + // shortened conversation cold. Their identity is the disk cache's (model, template, context); the directory + // starts with a dot, so the KV cache's scan skips it. + if (!disk_cache_.disabled()) { + std::string identity; + for (uint8_t b : compute_disk_cache_salt(config_)) { + char hex_byte[3]; + std::snprintf(hex_byte, sizeof hex_byte, "%02x", b); + identity += hex_byte; + } + pflash_unified_disk_views_.init(config_.disk_cache_dir + "/.pflash-views", identity, + disk_cache_ttl_seconds(config_.disk_cache_ttl_hours), 256); + // A new conversation must not take the lineage of one on disk (forget_lineage would drop that one). + pflash_unified_lineage_ = pflash_unified_disk_views_.max_lineage(); + } + std::fflush(stderr); + } + // PPP env overrides (operator-facing; no CLI flags required). auto env_truthy = [](const char * v) -> bool { if (!v || !*v) return false; @@ -3186,6 +3316,9 @@ void HttpServer::handle_client(SocketHandle fd) { if (auto keep = pflash_keep_history_json(); !keep.is_null()) { body["pflash"]["keep_history"] = std::move(keep); } + if (auto unified = pflash_unified_json(); !unified.is_null()) { + body["pflash"]["unified"] = std::move(unified); + } } body.update(model_routing_status()); send_response(fd, 200, "application/json", body.dump() + "\n"); @@ -3208,6 +3341,9 @@ void HttpServer::handle_client(SocketHandle fd) { if (auto keep = pflash_keep_history_json(); !keep.is_null()) { body["pflash"]["keep_history"] = std::move(keep); } + if (auto unified = pflash_unified_json(); !unified.is_null()) { + body["pflash"]["unified"] = std::move(unified); + } } send_response(fd, 200, "application/json", body.dump() + "\n"); socket_close(fd); @@ -4893,6 +5029,8 @@ std::string HttpServer::apply_pflash_compression( // message's content span (the query head's source). PFlashTokenSpan human_query_content{-1, -1}; http_detail::PflashChatTurnSpan chat_turn; + // PFLASH_UNIFIED=1: the normalised messages the prompt was rendered from (unused otherwise). + std::vector unified_messages; if (!experiment.configured || tail_parser) { ChatMarkers chat_markers; if (resolve_chat_markers(tokenizer_, chat_markers)) { @@ -4911,6 +5049,7 @@ std::string HttpServer::apply_pflash_compression( if (messages.empty()) { return "PFlash strict selection normalized messages are empty"; } + if (pflash_unified_) unified_messages = messages; int last_user_index = -1; for (int index = (int) messages.size() - 1; index >= 0; --index) { @@ -5681,6 +5820,41 @@ std::string HttpServer::apply_pflash_compression( std::fflush(stderr); } + if (pflash_unified_) { + // The unified view takes over after the shared query and pins. + ModelBackend::CompressRequest unified_request; + auto pins = required_instruction_spans; + pins.insert(pins.end(), system_spans.begin(), system_spans.end()); + pins.insert(pins.end(), instruction_role_spans.begin(), instruction_role_spans.end()); + unified_request.required_instruction_spans = + http_detail::canonicalize_pflash_token_spans(std::move(pins)); + unified_request.query_suffix_candidates = query_suffix_candidates; + unified_request.history_query_spans = history_query_spans; + unified_request.turn_query_span = turn_query_span; + unified_request.extra_query_spans = extra_query_spans; + unified_request.request_messages = request_probe_messages; + if (query_window.valid()) { + unified_request.score_query_end = query_window.end; + unified_request.score_query_tokens = query_window.tokens; + } + unified_request.input_ids = drafter_ids; + std::string not_ready; + if (!messages_input) not_ready = "not_messages"; + else if (!experiment.selection_active) not_ready = "no_strict_selection"; + else if (!query_window.valid()) not_ready = "no_query"; + else if (!chat_turn.valid()) not_ready = "turns_unparsed"; + else if (config_.pflash_remote_drafter) not_ready = "remote_drafter"; + else if (!http_detail::pflash_selection_gives_lifts(experiment)) not_ready = "unscored_selection"; + else if (unified_messages.empty()) not_ready = "messages_unreadable"; + std::string handback; + auto unified_error = apply_unified_view(req, prepared, chat_turn, unified_messages, + std::move(unified_request), experiment.chunk_size, not_ready, + handback); + if (handback.empty()) return unified_error; + // Handed back: the chat view below serves this request as it would without PFLASH_UNIFIED. + prepared.unified_handback = handback; + } + // Strict selection spends the keep ratio on the droppable tokens only: // what it keeps anyway (instructions, tools, the query and its turn's // envelope, the generation prompt) is already cheap -- a stable system @@ -5943,8 +6117,9 @@ std::string HttpServer::apply_pflash_compression( existing.view_gen_begin, input_tokens, true, 0}; } } else { - const int cached_target = - restorable_prefix_tokens(req.prompt_tokens); + const auto restorable = restorable_prefix_tokens( + req.prompt_tokens, req.disk_cache_policy); + const int cached_target = restorable.tokens; const int cached_drafter = prompt_tokens > 0 ? (std::min)(input_tokens, (int) std::floor( (double) input_tokens * (double) cached_target / @@ -6018,12 +6193,17 @@ std::string HttpServer::apply_pflash_compression( if (!keep_history_stats.is_null()) { admission_stats["keep_history"] = keep_history_stats; } + // Disk cache off: no source field, the record is unchanged. + const bool disk_cache_on = !disk_cache_.disabled(); + if (disk_cache_on) admission_stats["cached_source"] = restorable.source; std::fprintf(stderr, "[pflash-admission] %s cached=%d cached_droppable~%d " - "uncached_droppable~%d droppable~%d threshold=%d\n", + "uncached_droppable~%d droppable~%d threshold=%d%s%s\n", http_detail::pflash_admission_name(admission), cached_target, cached_droppable, uncached_droppable, - droppable_target, config_.pflash_threshold); + droppable_target, config_.pflash_threshold, + disk_cache_on ? " source=" : "", + disk_cache_on ? restorable.source : ""); if (admission != http_detail::PflashAdmission::Compress) { // Arrival on a raw conversation the cache holds: compress // the large uncached chunk only; otherwise it stays raw. @@ -6541,7 +6721,23 @@ std::string HttpServer::apply_pflash_compression( return {}; } -int HttpServer::restorable_prefix_tokens( +http_detail::PflashRestorablePrefix HttpServer::restorable_prefix_tokens( + const std::vector & prompt, + const DiskPrefixCachePolicy & disk_policy) { + const int ram = ram_restorable_prefix_tokens(prompt); + // The generation path looks the whole prompt up on disk only in Full + // scope; its probe is index-only, so is this one. + const bool disk_enabled = !disk_cache_.disabled() && + disk_policy.mode == DiskPrefixCacheMode::Full; + int disk = 0; + if (disk_enabled) { + const auto probe = disk_cache_.longest_prefix(prompt); + disk = (std::min)(probe.len, probe.cur_pos); + } + return http_detail::pflash_restorable_prefix(ram, disk_enabled, disk); +} + +int HttpServer::ram_restorable_prefix_tokens( const std::vector & prompt) { if (prefix_cache_.disabled()) return 0; int limit = (int) prompt.size(); @@ -6571,6 +6767,18 @@ json HttpServer::pflash_failure_counters_json() const { }; } +json HttpServer::pflash_unified_json() const { + if (!pflash_unified_) return nullptr; + std::lock_guard lk(pflash_failure_mu_); + return {{"region_tools", pflash_region_tools_.identity}, + {"recall", pflash_unified_recall_}, + {"recall_gate", pflash_recall_gate_ > 0.0 ? json(pflash_recall_gate_) : json(nullptr)}, + {"views", pflash_unified_views_.size()}, + {"view_bytes", pflash_unified_views_.bytes()}, + {"disk_views", pflash_unified_disk_views_.size()}, + {"counts", pflash_unified_counts_}}; +} + json HttpServer::pflash_keep_history_json() const { if (config_.pflash_mode != ServerConfig::PflashMode::AUTO || !http_detail::pflash_cache_admission_enabled()) { @@ -6586,7 +6794,7 @@ void HttpServer::trace_pflash_served( const char * path = std::getenv("PFLASH_VIEW_TRACE_PATH"); if (!path || !*path) return; json record = { - {"schema_version", 2}, + {"schema_version", pflash_unified_ ? 3 : 2}, {"kind", prepared.compressed ? "compressed" : "raw"}, {"request_id", req.response_id}, {"prompt_tokens", req.prompt_tokens.size()}, @@ -6594,6 +6802,10 @@ void HttpServer::trace_pflash_served( {"pflash", prepared.pflash_stats}, {"served_text", tokenizer_.decode(prepared.tokens)}, }; + if (!prepared.unified_handback.empty()) { + if (!record["pflash"].is_object()) record["pflash"] = json::object(); + record["pflash"]["unified"] = {{"decision", "handback"}, {"reason", prepared.unified_handback}}; + } // Failure policy D5: a raw record served because compression failed says so. if (prepared.pflash_stats.is_object() && prepared.pflash_stats.contains("pflash_fallback")) { record["pflash_fallback"] = prepared.pflash_stats["pflash_fallback"]; @@ -7046,6 +7258,1067 @@ bool HttpServer::serve_pflash_region_view( return true; } +size_t HttpServer::forget_unified_views(const std::vector & raw_tokens) { + if (!pflash_unified_) return 0; + auto lineages = pflash_unified_views_.lineages_continued_by(raw_tokens); + const auto on_disk = pflash_unified_disk_views_.lineages_continued_by(raw_tokens); + lineages.insert(lineages.end(), on_disk.begin(), on_disk.end()); + std::sort(lineages.begin(), lineages.end()); + lineages.erase(std::unique(lineages.begin(), lineages.end()), lineages.end()); + for (const uint64_t lineage : lineages) { + pflash_unified_views_.forget_lineage(lineage); + pflash_unified_disk_views_.forget_lineage(lineage); + } + return lineages.size(); +} + +std::string HttpServer::apply_unified_view( + const ParsedRequest & req, PreparedPrompt & prepared, + const http_detail::PflashChatTurnSpan & turn, + const std::vector & messages, + ModelBackend::CompressRequest request, int chunk_size, + const std::string & not_ready, std::string & handback) { + namespace pu = pflash_unified; + const std::vector & raw = req.prompt_tokens; + const int prompt_tokens = (int) raw.size(); + json stats = {{"version", 1}}; + const auto count = [this] (const std::string & key) { + std::lock_guard lk(pflash_failure_mu_); + ++pflash_unified_counts_[key]; + }; + // Full prefill of the raw prompt: nothing is paged and no view continues. + const auto serve_raw = [&] (const std::string & reason) { + count("raw." + reason); + stats["decision"] = "raw"; + stats["reason"] = reason; + prepared.pflash_stats = {{"unified", stats}}; + std::fprintf(stderr, "[pflash-unified] raw prompt (%d tokens): %s\n", prompt_tokens, reason.c_str()); + std::fflush(stderr); + return std::string(); + }; + // The chat view serves what the unified view does not take; nothing here changed yet. Once it accepts the + // request, no later turn continues a view of this conversation (prepare_prompt). + const auto hand_back = [&] (const std::string & reason) { + count("handback." + reason); + handback = reason; + std::fprintf(stderr, "[pflash-unified] handed back to the chat view (%d tokens): %s\n", prompt_tokens, + reason.c_str()); + std::fflush(stderr); + return std::string(); + }; + if (!not_ready.empty()) return hand_back(not_ready); + + const std::vector & drafter_ids = request.input_ids; + const int input = (int) drafter_ids.size(); + if (turn.generation_begin <= 0 || turn.generation_begin >= input) return hand_back("no_generation_prompt"); + // The generation prompt, in target tokens: the raw prompt and every served prompt end with it. + const auto generation = tokenizer_.encode(drafter_tokenizer_->decode( + std::vector(drafter_ids.begin() + turn.generation_begin, drafter_ids.end()))); + if (generation.empty() || generation.size() >= raw.size() || + !std::equal(generation.begin(), generation.end(), raw.end() - (long) generation.size())) { + return hand_back("generation_mismatch"); + } + const int raw_gen_begin = prompt_tokens - (int) generation.size(); + + // Both tokenisations decode to one text; units are cut where both start a token. + const auto td = http_detail::decode_prompt_with_offsets(tokenizer_, raw); + const auto dd = http_detail::decode_prompt_with_offsets(*drafter_tokenizer_, drafter_ids); + if (td.text != dd.text) return hand_back("text_mismatch"); + const std::string & text = td.text; + // The first token starting exactly at ``at`` (the text's end maps to the token count), else -1. + const auto token_at = [] (const http_detail::DecodedPrompt & decoded, size_t at) { + const auto it = std::lower_bound(decoded.token_begin.begin(), decoded.token_begin.end(), at); + if (it == decoded.token_begin.end()) { + return at == decoded.text.size() ? (int) decoded.token_begin.size() : -1; + } + return *it == at ? (int) (it - decoded.token_begin.begin()) : -1; + }; + // The first token starting at or after ``at``. + const auto token_from = [] (const http_detail::DecodedPrompt & decoded, size_t at) { + return (int) (std::lower_bound(decoded.token_begin.begin(), decoded.token_begin.end(), at) - + decoded.token_begin.begin()); + }; + const auto target_estimate = [&] (int drafter_tokens) { + return (int) std::ceil((double) prompt_tokens * (double) drafter_tokens / (double) input); + }; + const auto encode_len = [&] (const std::string & piece) { return (int) tokenizer_.encode(piece).size(); }; + const auto & pins = request.required_instruction_spans; + const int query_end = request.score_query_end; + const int query_begin = request.score_query_end - request.score_query_tokens; + // The selector's own rule for a structurally required candidate (pflash_chunk_is_structurally_required). + const auto pinned = [&] (int begin, int end) { + if (begin < query_end && end > query_begin) return true; + if (!request.query_suffix_candidates && end > query_end) return true; + for (const auto & span : pins) { + if (begin < span.end && end > span.begin) return true; + } + return false; + }; + + // The newest turn -- every message since the last reply: the run of user messages (a document sent on + // its own, then the question), or the group of tool messages -- holds the candidates. + struct Candidate { + size_t message = 0; + pu::RegionKind kind = pu::RegionKind::UserContent; + std::string brief; + std::string call_id; + std::string skip; // why it cannot be a region; "" = it may be + bool located = false; + size_t cb = 0, ce = 0; + int db = 0, de = 0, tb = 0, te = 0; + pu::Lines lines; + bool lines_ready = false; + }; + std::vector newest; + const size_t last = messages.size() - 1; + if (messages[last].role == "user") { + size_t first = last; + while (first > 0 && messages[first - 1].role == "user") --first; + for (size_t index = first; index <= last; ++index) { + Candidate c; + c.message = index; + newest.push_back(std::move(c)); + } + } else if (messages[last].role == "tool") { + size_t first = last; + while (first > 0 && messages[first - 1].role == "tool") --first; + const ChatMessage * caller = + first > 0 && messages[first - 1].role == "assistant" ? &messages[first - 1] : nullptr; + for (size_t index = first; index <= last; ++index) { + Candidate c; + c.message = index; + c.kind = pu::RegionKind::ToolOutput; + const std::string & id = messages[index].tool_call_id; + int calls = 0; + int results = 0; + const ChatToolCall * call = nullptr; + if (caller && !id.empty()) { + for (const auto & entry : caller->tool_calls) { + if (entry.id == id) { + ++calls; + call = &entry; + } + } + for (size_t other = first; other <= last; ++other) { + if (messages[other].tool_call_id == id) ++results; + } + } + if (calls != 1 || results != 1) { + c.skip = "unpaired"; + } else { + c.call_id = id; + c.brief = pu::call_brief(call->name, call->arguments); + if (!pflash_region_tools_.listed(call->name)) c.skip = "unlisted"; + } + newest.push_back(std::move(c)); + } + } + // Their exact content in the prompt text, from one render with a placeholder per newest message. + if (!newest.empty()) { + std::vector placeholder_messages = messages; + std::vector placeholders(messages.size()); + std::vector contents(messages.size()); + for (const auto & c : newest) { + placeholders[c.message] = "\x1e" "PFLASH-UNIFIED-" + std::to_string(c.message) + "\x1f"; + contents[c.message] = messages[c.message].content; + placeholder_messages[c.message].content = placeholders[c.message]; + } + std::string rendered; + std::string error; + std::vector> chars; + if (render_messages_to_text(placeholder_messages, req, /*add_generation_prompt=*/true, rendered, error) && + http_detail::pflash_content_char_spans(rendered, placeholders, contents, text, chars, error)) { + for (auto & c : newest) { + c.located = true; + c.cb = chars[c.message].first; + c.ce = chars[c.message].second; + c.db = http_detail::token_at_offset(dd, c.cb); + c.de = token_from(dd, c.ce); + c.tb = http_detail::token_at_offset(td, c.cb); + c.te = token_from(td, c.ce); + } + } else { + count("unlocated"); + std::fprintf(stderr, "[pflash-unified] newest turn not located (%s): nothing is paged\n", + error.c_str()); + for (auto & c : newest) c.skip = "unlocated"; + } + } + const auto candidate_lines = [&] (Candidate & c) -> const pu::Lines & { + if (!c.lines_ready) { + c.lines = pu::detect_lines(text, c.cb, c.ce); + c.lines_ready = true; + } + return c.lines; + }; + + pu::View view; + bool has_view = pflash_unified_views_.find(raw, drafter_ids, pflash_unified_config_, view); + // A view the RAM store no longer holds (a restart, or its eviction) may be on disk: a longer one wins. + if (pflash_unified_disk_views_.enabled()) { + const size_t ram_len = has_view ? (size_t) (std::max)(0, view.raw_gen_begin) : 0; + pu::View stored; + bool loaded = pflash_unified_disk_views_.find(raw, drafter_ids, pflash_unified_config_, ram_len, stored); + // A view read from disk is consistent in itself (view_from_json); its regions must also be this request's + // bytes, and its unit cuts the token starts they were taken at, in both tokenisations. + const auto matches_text = [&] (const pu::View & v) { + for (const auto & region : v.regions) { + if (region.ce > text.size() || + pu::sha1_hex(text.substr(region.cb, region.ce - region.cb)) != region.content_sha) { + return false; + } + const auto & cuts = region.layout.cuts; + for (size_t u = 0; u + 1 < cuts.size(); ++u) { + if (token_at(td, cuts[u]) != region.unit_tb[u] || token_at(td, cuts[u + 1]) != region.unit_te[u] || + token_at(dd, cuts[u]) != region.unit_db[u] || token_at(dd, cuts[u + 1]) != region.unit_de[u]) { + return false; + } + } + } + return true; + }; + if (loaded && !matches_text(stored)) { + std::fprintf(stderr, "[pflash-unified] disk view of lineage %llu does not match the prompt: dropped\n", + (unsigned long long) stored.lineage); + std::fflush(stderr); + pflash_unified_disk_views_.forget_lineage(stored.lineage); + count("disk_view_rejected"); + loaded = false; + } + if (loaded) { + count("disk_view"); + stats["disk_view"] = true; + view = std::move(stored); + has_view = true; + } + } + // A repeat serves what was served only when the view holds every span this request pins (a pflash_required + // text added on a retry may name lines the view left out); otherwise the request starts a new view. + if (has_view && view.raw_tokens == raw && view.drafter_ids == drafter_ids) { + bool served = true; + for (const auto & span : pins) { + for (const auto & region : view.regions) { + for (size_t u = 0; u < region.unit_db.size(); ++u) { + served = served && !(span.begin < region.unit_de[u] && span.end > region.unit_db[u] && + !region.held[u] && !region.mandatory[u]); + } + } + } + if (!served) { + count("repeat.pins_unserved"); + has_view = false; + } + } + const bool found_view = has_view; + if (has_view && view.raw_tokens == raw && view.drafter_ids == drafter_ids) { + // The same request again: what was served. + count("repeat"); + stats["decision"] = "repeat"; + stats["turn"] = view.turns; + stats["served_tokens"] = view.view_tokens.size(); + prepared.tokens = view.view_tokens; + prepared.compressed = view.view_tokens != raw; + prepared.unified_view = true; + prepared.snapshot_cut = view.view_gen_begin; + prepared.pflash_stats = {{"unified", stats}, {"compress_ms", 0.0}, + {"drafter_input_tokens", input}}; + if (prepared.compressed) trace_pflash_served(req, prepared); + std::fprintf(stderr, "[pflash-unified] repeat turn=%d served=%zu\n", view.turns, view.view_tokens.size()); + std::fflush(stderr); + return {}; + } + if (has_view && !(view.drafter_gen_begin < turn.generation_begin && view.raw_gen_begin <= raw_gen_begin && + view.view_gen_begin > 0 && view.view_gen_begin <= (int) view.view_tokens.size())) { + count("reset.unusable"); + stats["reset"] = "unusable"; + has_view = false; + } + // A conversation the disk session cache holds (a restarted server, or a session written out on eviction) counts + // as cached, as in the chat view. Disk cache off: no cached record, the stats are unchanged. + const auto restorable_prefix = restorable_prefix_tokens(raw, req.disk_cache_policy); + const int restorable = restorable_prefix.tokens; + if (!disk_cache_.disabled()) stats["cached"] = {{"tokens", restorable}, {"source", restorable_prefix.source}}; + + for (int pass = 0; pass < 2; ++pass) { + const bool continues = has_view && pass == 0; + const int new_from = continues ? view.drafter_gen_begin : 0; + std::vector held_regions = continues ? view.regions : std::vector{}; + + // Admission: eligible, new, uncached, big, not a re-read. + json admission = json::array(); + std::vector region_candidates; + std::vector verbatim; + for (size_t k = 0; k < newest.size(); ++k) { + Candidate & c = newest[k]; + std::string why = c.skip; + int droppable = 0; + if (why.empty() && c.db < new_from) why = "not_new"; + if (why.empty() && c.tb < restorable) why = "cached"; + if (why.empty()) { + droppable = target_estimate((std::max)(0, (c.de - c.db) - + http_detail::pflash_span_tokens_within(pins, c.db, c.de))); + if (droppable < config_.pflash_threshold) why = "small"; + } + if (why.empty() && candidate_lines(c).count() < 5) why = "no_lines"; + if (why.empty() && c.kind == pu::RegionKind::ToolOutput) { + const auto hashes = pu::line_hashes(text, candidate_lines(c)); + for (const auto & region : held_regions) { + if (pu::match_reread(hashes, region.hashes).is_reread()) { + why = "reread"; + break; + } + } + } + admission.push_back({{"message", c.message}, + {"kind", c.kind == pu::RegionKind::ToolOutput ? "tool_output" : "user_content"}, + {"decision", why.empty() ? "region" : why}, + {"droppable", droppable}}); + if (why.empty()) { + region_candidates.push_back(k); + } else { + verbatim.push_back(k); + } + } + + // A conversation the view would start: when the part it can never page -- the uncached, unpinned history + // before the newest turn, and newest content it cannot cut into lines -- is a threshold's worth, the chat + // view takes the request (its cache admission compresses such a prompt) instead of a raw prefill of it. + // Newest content the view serves verbatim by design (unlisted tool outputs, small messages) does not count. + if (!continues) { + const int uncached_from = (std::min)(turn.generation_begin, (int) std::floor( + (double) restorable * (double) input / (double) (std::max)(1, prompt_tokens))); + const auto unpinned = [&] (int begin, int end) { + begin = (std::max)(begin, uncached_from); + return end > begin ? (end - begin) - http_detail::pflash_span_tokens_within(pins, begin, end) : 0; + }; + int newest_begin = turn.generation_begin; + for (const auto & c : newest) { + if (c.located) newest_begin = (std::min)(newest_begin, c.db); + } + int cannot_page = unpinned(0, newest_begin); + for (size_t k = 0; k < newest.size(); ++k) { + if (admission[k].value("decision", "") == "no_lines") cannot_page += unpinned(newest[k].db, newest[k].de); + } + if (target_estimate(cannot_page) >= config_.pflash_threshold) { + // Once the chat view accepts it, the view this turn would have continued goes, with its lineage. + return hand_back(found_view ? "reset_uncached" : "uncached"); + } + } + + // Recall trigger: a new user question on a view with regions. + bool recall = false; + int recall_at = -1; + if (continues && pflash_unified_recall_ && !held_regions.empty() && !newest.empty() && + newest.back().kind == pu::RegionKind::UserContent && newest.back().located && + newest.back().db >= view.drafter_gen_begin) { + recall_at = token_at(td, newest.back().cb); + // Gate arms: no drafter pass for a question when every unit of every region is already held. + bool unheld = !(pflash_recall_gate_ > 0.0); + for (const auto & region : held_regions) { + unheld = unheld || std::find(region.held.begin(), region.held.end(), 0) != region.held.end(); + } + if (recall_at < view.raw_gen_begin) { + count("recall.unplaceable"); + } else if (!unheld) { + count("recall.complete"); + stats["recall_gate"] = {{"decision", "complete"}}; + } else { + recall = true; + } + } + + // One drafter pass scores the regions to page and, at a question, the earlier regions. + std::vector> lifts; + ModelBackend::CompressResult result; + bool scored = false; + double compress_ms = 0.0; + if (!region_candidates.empty() || recall) { + std::vector scored_spans; + for (size_t k : region_candidates) scored_spans.push_back({newest[k].db, newest[k].de}); + if (recall) { + for (const auto & region : held_regions) scored_spans.push_back({region.db, region.de}); + } + scored_spans = http_detail::canonicalize_pflash_token_spans(std::move(scored_spans)); + // Everything outside the scored regions is kept: the selector spends its budget inside them. + auto mandatory = pins; + const auto outside = http_detail::pflash_complement_token_spans(scored_spans, input); + mandatory.insert(mandatory.end(), outside.begin(), outside.end()); + mandatory = http_detail::canonicalize_pflash_token_spans(std::move(mandatory)); + const int kept = http_detail::pflash_kept_tokens(input, chunk_size, query_begin, query_end, mandatory, + !request.query_suffix_candidates); + const float ratio = http_detail::resolve_pflash_keep_ratio( + pflash_keep_ratio(config_, (std::max)(1, target_estimate((std::max)(0, input - kept)))), + req.session_id, sessions_); + ModelBackend::CompressRequest call = request; + call.required_instruction_spans = std::move(mandatory); + call.keep_ratio = (float) http_detail::pflash_effective_keep_ratio(input, kept, ratio); + call.drafter_path = config_.pflash_drafter_path; + call.drafter_gpu = config_.pflash_drafter_gpu; + call.skip_park = config_.pflash_skip_park; + call.residency_action = resolve_draft_residency_action( + config_.draft_residency, + DraftResidencyContext{DraftResidencyUse::PFlashCompress, config_.lazy_draft, + !config_.draft_path.empty()}); + const auto started = std::chrono::steady_clock::now(); + result = backend_.compress(call); + compress_ms = std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + if (result.ok && !result.candidate_lifts.empty()) { + lifts = result.candidate_lifts; + scored = true; + } else { + const char * detail = result.ok ? "no candidate scores" : luce_last_error(); + count(result.ok ? "drafter.no_lifts" : "drafter.failed"); + stats["drafter_error"] = detail ? detail : ""; + // PFLASH_FAIL_CLOSED=1 refuses the request, as a failed compression on the chat view; otherwise the + // newest content is served verbatim and the view continues. + if (http_detail::pflash_fail_closed_from_env()) { + prepared.pflash_failure_detail = detail ? detail : ""; + return "PFlash unified view: the drafter pass gave no scores"; + } + std::fprintf(stderr, "[pflash-unified] drafter pass gave no scores (%s): nothing paged or recalled\n", + detail ? detail : ""); + std::fflush(stderr); + } + } + // The request probe's own pins (the question it found, its recency pin) stay whole, as in the chat view's + // selection. An earlier region kept its question's units when it was paged. + const auto probe_pinned = [&] (int begin, int end) { + for (const auto & span : result.probe_pins) { + if (begin < span.end && end > span.begin) return true; + } + return false; + }; + // A unit's score: the token-weighted mean lift of the candidates over it. + const auto unit_score = [&] (int begin, int end) { + double sum = 0.0; + double weight = 0.0; + for (const auto & lift : lifts) { + const int lo = (std::max)(begin, lift.first.begin); + const int hi = (std::min)(end, lift.first.end); + if (hi > lo) { + sum += lift.second * (double) (hi - lo); + weight += (double) (hi - lo); + } + } + return weight > 0.0 ? sum / weight : std::nan(""); + }; + // A region's cap: the release curve on its droppable tokens (kept in the region); a pass whose stage 2 + // widened the budget widens it once, paging and recall alike. + const auto base_cap = [&] (int droppable) { + const float ratio = http_detail::resolve_pflash_keep_ratio( + pflash_keep_ratio(config_, (std::max)(1, droppable)), req.session_id, sessions_); + return (int) std::floor((double) ratio * (double) droppable); + }; + const auto widened_cap = [&] (int cap, int droppable) { + return result.budget_widened ? (std::min)(droppable, cap + cap / 2) : cap; + }; + + // Paging, per region: units between clean line cuts, admitted by score under the cap. + struct Splice { + int begin = 0; // raw target tokens [begin, end) replaced by ``tokens`` + int end = 0; + std::vector tokens; + std::vector pieces; // relative to ``tokens`` + }; + std::vector splices; + std::vector new_regions; + const int region_base = (int) held_regions.size(); + for (size_t slot = 0; slot < region_candidates.size(); ++slot) { + const size_t k = region_candidates[slot]; + Candidate & c = newest[k]; + const auto verbatim_because = [&] (const std::string & why) { + admission[k]["decision"] = why; + count("verbatim." + why); + verbatim.push_back(k); + }; + if (!scored) { + verbatim_because("unscored"); + continue; + } + const pu::Lines & lines = candidate_lines(c); + std::vector common; + { + const auto t0 = std::lower_bound(td.token_begin.begin(), td.token_begin.end(), lines.begin); + const auto t1 = std::upper_bound(td.token_begin.begin(), td.token_begin.end(), lines.end); + const auto d0 = std::lower_bound(dd.token_begin.begin(), dd.token_begin.end(), lines.begin); + const auto d1 = std::upper_bound(dd.token_begin.begin(), dd.token_begin.end(), lines.end); + std::set_intersection(t0, t1, d0, d1, std::back_inserter(common)); + common.erase(std::unique(common.begin(), common.end()), common.end()); + } + std::vector proposed; + for (const auto & lift : lifts) { + if (lift.first.begin > c.db && lift.first.begin < c.de) { + proposed.push_back(dd.token_begin[(size_t) lift.first.begin]); + } + } + pu::Region region; + std::string why; + if (!pu::layout_region(lines, common, proposed, region.layout, why)) { + verbatim_because("layout"); + continue; + } + const int units = region.layout.units(); + std::vector cut_t; + std::vector cut_d; + for (size_t cut : region.layout.cuts) { + cut_t.push_back(token_at(td, cut)); + cut_d.push_back(token_at(dd, cut)); + } + if (std::count(cut_t.begin(), cut_t.end(), -1) || std::count(cut_d.begin(), cut_d.end(), -1)) { + verbatim_because("unmapped_cut"); + continue; + } + std::vector cost((size_t) units); + std::vector score((size_t) units); + region.mandatory.assign((size_t) units, 0); + int droppable = 0; + for (int u = 0; u < units; ++u) { + region.unit_tb.push_back(cut_t[(size_t) u]); + region.unit_te.push_back(cut_t[(size_t) u + 1]); + region.unit_db.push_back(cut_d[(size_t) u]); + region.unit_de.push_back(cut_d[(size_t) u + 1]); + cost[(size_t) u] = cut_t[(size_t) u + 1] - cut_t[(size_t) u]; + score[(size_t) u] = unit_score(cut_d[(size_t) u], cut_d[(size_t) u + 1]); + region.mandatory[(size_t) u] = + pinned(cut_d[(size_t) u], cut_d[(size_t) u + 1]) || + probe_pinned(cut_d[(size_t) u], cut_d[(size_t) u + 1]) ? 1 : 0; + if (!region.mandatory[(size_t) u]) droppable += cost[(size_t) u]; + } + if (droppable <= 0) { + verbatim_because("all_pinned"); + continue; + } + const int unwidened = base_cap(droppable); + const int cap = widened_cap(unwidened, droppable); + std::map, int> marker_costs; + const auto marker_line = [&] (int first, int last_unit) { + return pu::inserted_line(text, lines, region.layout.cuts[(size_t) first], + region.layout.cuts[(size_t) last_unit + 1], + pu::marker_text(region.layout.line_count(first, last_unit))); + }; + const auto marker_cost = [&] (int first, int last_unit) { + const auto key = std::make_pair(first, last_unit); + const auto it = marker_costs.find(key); + if (it != marker_costs.end()) return it->second; + const int tokens = encode_len(marker_line(first, last_unit)); + marker_costs.emplace(key, tokens); + return tokens; + }; + const int total_lines = (int) lines.count(); + const auto preamble_line = [&] (int shown) { + return pu::inserted_line(text, lines, region.layout.head_end, region.layout.head_end, + pu::preamble_text(c.kind, shown, total_lines)); + }; + int fixed = encode_len(preamble_line(total_lines)); + const auto order = pu::rank_units(score, region.mandatory); + pu::Admission admitted; + std::string preamble; + for (int attempt = 0; attempt < 3; ++attempt) { + admitted = pu::admit_paging(cost, region.mandatory, order, fixed, cap, marker_cost); + if (!admitted.ok) break; + int dropped_lines = 0; + for (int u = 0; u < units; ++u) { + if (!admitted.admitted[(size_t) u]) dropped_lines += region.layout.line_count(u, u); + } + preamble = preamble_line(total_lines - dropped_lines); + const int actual = encode_len(preamble); + if (admitted.cost - fixed + actual <= cap) { + admitted.cost += actual - fixed; + break; + } + fixed = actual; + admitted.ok = false; + } + if (!admitted.ok) { + verbatim_because("cap_below_overhead"); + continue; + } + if (std::find(admitted.admitted.begin(), admitted.admitted.end(), 0) == admitted.admitted.end()) { + verbatim_because("nothing_dropped"); + continue; + } + const int region_index = region_base + (int) new_regions.size(); + Splice splice; + splice.begin = cut_t.front(); + splice.end = cut_t.back(); + const auto add = [&] (std::vector tokens, const char * kind) { + const int at = (int) splice.tokens.size(); + splice.tokens.insert(splice.tokens.end(), tokens.begin(), tokens.end()); + splice.pieces.push_back({at, (int) splice.tokens.size(), kind, region_index, -1}); + }; + add(tokenizer_.encode(preamble), "preamble"); + int kept_tokens = 0; + for (int u = 0; u < units;) { + if (admitted.admitted[(size_t) u]) { + int b = u; + while (b + 1 < units && admitted.admitted[(size_t) b + 1]) ++b; + add(std::vector(raw.begin() + cut_t[(size_t) u], raw.begin() + cut_t[(size_t) b + 1]), + "paged_text"); + kept_tokens += cut_t[(size_t) b + 1] - cut_t[(size_t) u]; + u = b + 1; + continue; + } + int b = u; + while (b + 1 < units && !admitted.admitted[(size_t) b + 1]) ++b; + add(tokenizer_.encode(marker_line(u, b)), "marker"); + u = b + 1; + } + region.kind = c.kind; + region.brief = c.brief; + region.tool_call_id = c.call_id; + region.content_sha = pu::sha1_hex(text.substr(c.cb, c.ce - c.cb)); + region.cb = c.cb; + region.ce = c.ce; + region.tb = c.tb; + region.te = c.te; + region.db = c.db; + region.de = c.de; + region.lines = lines; + region.hashes = pu::line_hashes(text, lines); + region.kept = admitted.admitted; + region.held = admitted.admitted; + region.reread.assign(lines.count(), 0); + region.cap = unwidened; + region.droppable = droppable; + if (pflash_recall_trace_units_) { + std::vector cuts; + for (size_t cut : region.layout.cuts) cuts.push_back((long) cut - (long) region.cb); + admission[k]["trace_units"] = {{"cb", region.cb}, {"content_bytes", region.ce - region.cb}, + {"cuts", cuts}, {"kept", region.kept}}; + } + admission[k]["units"] = units; + admission[k]["kept_units"] = (int) std::count(admitted.admitted.begin(), admitted.admitted.end(), 1); + admission[k]["mandatory_units"] = + (int) std::count(region.mandatory.begin(), region.mandatory.end(), 1); + admission[k]["cap"] = cap; + admission[k]["unwidened_cap"] = unwidened; + admission[k]["optional_served"] = admitted.cost; + admission[k]["region_tokens"] = splice.end - splice.begin; + admission[k]["served_tokens"] = splice.tokens.size(); + admission[k]["kept_tokens"] = kept_tokens; + admission[k]["lines"] = total_lines; + admission[k]["escaped_lines"] = lines.escaped; + admission[k]["region"] = region_index; + count(c.kind == pu::RegionKind::ToolOutput ? "region.tool_output" : "region.user_content"); + new_regions.push_back(std::move(region)); + splices.push_back(std::move(splice)); + } + + // Re-reads: a newest message served verbatim (rejected above, or fallen back while paging) that repeats lines of + // an earlier region covers them; a unit whose informative lines are all covered is held (never recalled). + int covered_units = 0; + for (size_t k : verbatim) { + Candidate & c = newest[k]; + if (!c.located || c.ce <= c.cb || held_regions.empty()) continue; + const auto hashes = pu::line_hashes(text, candidate_lines(c)); + for (auto & region : held_regions) { + const auto match = pu::match_reread(hashes, region.hashes); + if (match.matched_candidate == 0) continue; + for (size_t line = 0; line < region.reread.size(); ++line) { + if (match.region_covered[line]) region.reread[line] = 1; + } + for (int u = 0; u < region.layout.units(); ++u) { + if (region.held[(size_t) u]) continue; + if (pu::lines_covered(region.reread, match.region_informative, region.layout.first_line(u), + region.layout.cut_line[(size_t) u + 1])) { + region.held[(size_t) u] = 1; + ++covered_units; + } + } + } + } + + // Recall: each earlier region's unheld units, best first, under the region's cap, appended as + // labelled excerpts at the start of the question; nothing served before is touched. + json recall_stats = nullptr; + json gate_stats = nullptr; + bool recall_fired = false; + if (recall && scored) { + Splice block; + block.begin = block.end = recall_at; + const std::string open = pu::recall_open(); + const std::string close = pu::recall_close(); + const int wrapper = encode_len(open) + encode_len(close); + size_t largest = 0; + for (size_t ri = 1; ri < held_regions.size(); ++ri) { + if (held_regions[ri].cap > held_regions[largest].cap) largest = ri; + } + // Each region's candidates: unheld units, their scores, costs and the region's cap. + const size_t region_count = held_regions.size(); + std::vector candidates(region_count); + std::vector> orders(region_count); + std::vector region_caps(region_count); + for (size_t ri = 0; ri < region_count; ++ri) { + pu::Region & region = held_regions[ri]; + pu::RecallRegion & rc = candidates[ri]; + const int units = region.layout.units(); + rc.score.assign((size_t) units, 0.0); + rc.unit_cost.assign((size_t) units, 0); + rc.held = region.held; + for (int u = 0; u < units; ++u) { + rc.score[(size_t) u] = unit_score(region.unit_db[(size_t) u], region.unit_de[(size_t) u]); + if (region.held[(size_t) u]) continue; + if (region.lines.escaped) { + const size_t from = region.layout.cuts[(size_t) u]; + const size_t to = region.layout.cuts[(size_t) u + 1]; + rc.unit_cost[(size_t) u] = encode_len(pu::json_unescape(text, from, to)); + } else { + rc.unit_cost[(size_t) u] = region.unit_te[(size_t) u] - region.unit_tb[(size_t) u]; + } + } + rc.label_cost = encode_len(pu::recall_label_text(region.kind, region.brief) + "\n") + encode_len("\n"); + region_caps[ri] = widened_cap(region.cap, region.droppable); + rc.cap = region_caps[ri] - (ri == largest ? wrapper : 0); + orders[ri] = pu::rank_units(rc.score, rc.held); + } + std::vector planned(region_count); + const auto nothing = [&] { + for (size_t ri = 0; ri < region_count; ++ri) { + planned[ri].ok = true; + planned[ri].admitted.assign((size_t) held_regions[ri].layout.units(), 0); + } + }; + // The head's mass on a span: its candidates' lift x overlap / input (the lifts are mass per token + // relative to uniform, pflash_compress.cpp), pinned candidates excluded so that D <= S. S: every + // candidate that is not pinned, less what this request's new regions dropped (that content is + // neither present nor recallable here, and counting it would dilute the deficit of the older + // regions). A non-finite lift anywhere leaves S non-finite: unscorable, never a silent zero. + double context = 0.0; + double new_dropped = 0.0; + if (pflash_recall_gate_ > 0.0 || pflash_recall_trace_units_) { + const auto mass_of = [&] (int begin, int end) { + double mass = 0.0; + for (const auto & lift : lifts) { + if (pinned(lift.first.begin, lift.first.end)) continue; + const int lo = (std::max)(begin, lift.first.begin); + const int hi = (std::min)(end, lift.first.end); + if (hi > lo) mass += lift.second * (double) (hi - lo); + } + return mass / (double) (std::max)(1, input); + }; + for (const auto & lift : lifts) { + if (pinned(lift.first.begin, lift.first.end)) continue; + context += lift.second * (double) (lift.first.end - lift.first.begin) / (double) (std::max)(1, input); + } + for (const auto & region : new_regions) { + for (int u = 0; u < region.layout.units(); ++u) { + if (!region.held[(size_t) u]) { + new_dropped += mass_of(region.unit_db[(size_t) u], region.unit_de[(size_t) u]); + } + } + } + for (size_t ri = 0; ri < region_count; ++ri) { + const pu::Region & region = held_regions[ri]; + auto & mass = candidates[ri].mass; + mass.assign((size_t) region.layout.units(), 0.0); + for (int u = 0; u < region.layout.units(); ++u) { + if (!region.held[(size_t) u] || pflash_recall_trace_units_) { + mass[(size_t) u] = mass_of(region.unit_db[(size_t) u], region.unit_de[(size_t) u]); + } + } + } + } + if (pflash_recall_trace_units_) { + json regions_trace = json::array(); + for (size_t ri = 0; ri < region_count; ++ri) { + const pu::Region & region = held_regions[ri]; + std::vector cost((size_t) region.layout.units()); + for (int u = 0; u < region.layout.units(); ++u) { + cost[(size_t) u] = region.lines.escaped + ? encode_len(pu::json_unescape(text, region.layout.cuts[(size_t) u], + region.layout.cuts[(size_t) u + 1])) + : region.unit_te[(size_t) u] - region.unit_tb[(size_t) u]; + } + regions_trace.push_back({{"region", ri}, {"cb", region.cb}, {"held", region.held}, + {"mass", candidates[ri].mass}, {"cost", cost}, + {"score", candidates[ri].score}, {"cap", candidates[ri].cap}, + {"label_cost", candidates[ri].label_cost}}); + } + stats["recall_units"] = {{"context_mass", context}, {"new_dropped_mass", new_dropped}, + {"input", input}, {"regions", regions_trace}}; + } + const double gate_context = context - new_dropped; + if (pflash_recall_gate_ > 0.0) { + if (!(gate_context > 0.0) || !std::isfinite(gate_context)) { + count("recall.unscorable"); + gate_stats = {{"decision", "unscorable"}, {"theta", pflash_recall_gate_}}; + nothing(); + } else { + auto gated = pu::admit_recall_gated(candidates, gate_context, pflash_recall_gate_); + planned = std::move(gated.regions); + count(gated.gated ? "recall.gate_present" : "recall.gate_admitted"); + gate_stats = {{"decision", gated.gated ? "present" : "admitted"}, + {"theta", pflash_recall_gate_}, {"context_mass", gate_context}, + {"new_dropped_mass", new_dropped}, {"deficit_before", gated.deficit_before}, + {"deficit_planned", gated.deficit_after}}; + } + } else { + for (size_t ri = 0; ri < region_count; ++ri) { + const int label_cost = candidates[ri].label_cost; + planned[ri] = pu::admit_recall(candidates[ri].unit_cost, candidates[ri].held, orders[ri], + candidates[ri].cap, [&] (int, int) { return label_cost; }); + } + } + double final_deficit = 0.0; + int trimmed_units = 0; + std::vector excerpts; + std::vector excerpt_pieces; + bool excerpts_trail = true; // the last excerpt so far ends with a newline + recall_stats = json::array(); + for (size_t ri = 0; ri < region_count; ++ri) { + pu::Region & region = held_regions[ri]; + const int units = region.layout.units(); + const auto run_text = [&] (int first, int last_unit) { + const size_t from = region.layout.cuts[(size_t) first]; + const size_t to = region.layout.cuts[(size_t) last_unit + 1]; + return region.lines.escaped ? pu::json_unescape(text, from, to) : text.substr(from, to - from); + }; + const std::string label = pu::recall_label_text(region.kind, region.brief); + const int region_cap = region_caps[ri]; + const int cap = candidates[ri].cap; + const auto & order = orders[ri]; + auto admitted = std::move(planned[ri]); + const int planned_taken = admitted.taken; + // The excerpts as served; over the cap (a tokenisation join), the lowest-ranked unit goes. + std::vector tokens; + std::vector pieces; + bool trail = excerpts_trail; + const int region_index = (int) ri; + // Excerpts are separated by one blank line: label line, the run's lines, blank line, next label. + const auto build = [&] { + tokens.clear(); + pieces.clear(); + trail = excerpts_trail; + bool first = excerpts.empty(); + const auto add = [&] (std::vector part, const char * kind) { + const int at = (int) tokens.size(); + tokens.insert(tokens.end(), part.begin(), part.end()); + pieces.push_back({at, (int) tokens.size(), kind, region_index, -1}); + }; + for (int u = 0; u < units;) { + if (!admitted.admitted[(size_t) u]) { + ++u; + continue; + } + int b = u; + while (b + 1 < units && admitted.admitted[(size_t) b + 1]) ++b; + const std::string body = run_text(u, b); + const bool lead = !body.empty() && body.front() == '\n'; + if (!first) add(tokenizer_.encode(trail ? "\n" : "\n\n"), "recall_wrapper"); + first = false; + add(tokenizer_.encode(label + (lead ? "" : "\n")), "recall_label"); + if (region.lines.escaped) { + add(tokenizer_.encode(body), "recall_text"); + } else { + add(std::vector(raw.begin() + region.unit_tb[(size_t) u], + raw.begin() + region.unit_te[(size_t) b]), + "recall_text"); + } + trail = !body.empty() && body.back() == '\n'; + u = b + 1; + } + }; + build(); + for (auto it = admitted.taken > 0 ? order.rbegin() : order.rend(); + (int) tokens.size() > cap && it != order.rend(); ++it) { + if (!admitted.admitted[(size_t) *it]) continue; + admitted.admitted[(size_t) *it] = 0; + --admitted.taken; + build(); + } + int appended_units = 0; + for (int u = 0; u < units; ++u) { + if (!admitted.admitted[(size_t) u]) { + // The gate's result as served: what the cap trim removed is still missing. + if (!candidates[ri].held[(size_t) u] && (size_t) u < candidates[ri].mass.size()) { + final_deficit += candidates[ri].mass[(size_t) u]; + } + continue; + } + region.held[(size_t) u] = 1; + ++appended_units; + } + trimmed_units += planned_taken - admitted.taken; + region.recalled_tokens += (int) tokens.size(); + if (!tokens.empty()) excerpts_trail = trail; + for (auto piece : pieces) { + piece.begin += (int) excerpts.size(); + piece.end += (int) excerpts.size(); + excerpt_pieces.push_back(piece); + } + excerpts.insert(excerpts.end(), tokens.begin(), tokens.end()); + recall_stats.push_back({{"region", region_index}, {"units", appended_units}, + {"tokens", tokens.size()}, {"cap", region_cap}, + {"wrapper", ri == largest ? wrapper : 0}, + {"cumulative_tokens", region.recalled_tokens}}); + } + if (gate_stats.is_object() && gate_stats.contains("deficit_planned")) { + gate_stats["deficit_after"] = final_deficit / gate_context; + gate_stats["trimmed_units"] = trimmed_units; + } + if (excerpts.empty()) { + count("recall.empty"); + } else { + recall_fired = true; + count("recall.fired"); + const auto open_tokens = tokenizer_.encode(open); + block.tokens = open_tokens; + block.pieces.push_back({0, (int) open_tokens.size(), "recall_wrapper", -1, -1}); + for (auto piece : excerpt_pieces) { + piece.begin += (int) open_tokens.size(); + piece.end += (int) open_tokens.size(); + block.pieces.push_back(piece); + } + block.tokens.insert(block.tokens.end(), excerpts.begin(), excerpts.end()); + const int close_at = (int) block.tokens.size(); + const auto close_tokens = tokenizer_.encode(close); + block.tokens.insert(block.tokens.end(), close_tokens.begin(), close_tokens.end()); + block.pieces.push_back({close_at, (int) block.tokens.size(), "recall_wrapper", -1, -1}); + splices.insert(splices.begin(), std::move(block)); + } + } + + // The served prompt = the view's served prefix, then the raw prompt from where the view ended with + // every splice applied (raw target slices between them). + std::stable_sort(splices.begin(), splices.end(), + [] (const Splice & a, const Splice & b) { return a.begin < b.begin; }); + std::vector served; + std::vector pieces; + const int turn_index = (continues ? view.turns : 0) + 1; + int position = 0; + if (continues) { + served.assign(view.view_tokens.begin(), view.view_tokens.begin() + view.view_gen_begin); + for (auto piece : view.pieces) { + if (piece.begin >= view.view_gen_begin) continue; + piece.end = (std::min)(piece.end, view.view_gen_begin); + pieces.push_back(piece); + } + position = view.raw_gen_begin; + } + const auto add_raw = [&] (int from, int to, const char * kind) { + if (to <= from) return; + const int at = (int) served.size(); + served.insert(served.end(), raw.begin() + from, raw.begin() + to); + pieces.push_back({at, (int) served.size(), kind, -1, turn_index}); + }; + bool ordered = true; + for (auto & splice : splices) { + if (splice.begin < position || splice.end > raw_gen_begin) { + ordered = false; + break; + } + add_raw(position, splice.begin, "raw"); + const int at = (int) served.size(); + served.insert(served.end(), splice.tokens.begin(), splice.tokens.end()); + for (auto piece : splice.pieces) { + piece.begin += at; + piece.end += at; + piece.turn = turn_index; + pieces.push_back(piece); + } + position = splice.end; + } + if (!ordered) return serve_raw("splice_order"); + add_raw(position, raw_gen_begin, "raw"); + add_raw(raw_gen_begin, prompt_tokens, "generation"); + + if (continues && config_.max_ctx > 0 && (int) served.size() + req.max_output > config_.max_ctx) { + // A view that cannot continue is reset to admission on the raw prompt. + count("reset.context"); + stats["reset"] = "context"; + stats.erase("recall_gate"); + stats.erase("recall_units"); + continue; + } + // A turn served raw instead of continuing a stored view leaves a raw view (no regions), so its + // descendants continue what it served rather than the view it replaced. Not when the raw prompt does + // not fit either: a retry with a smaller output allowance may still continue that view. + const bool raw_fits = config_.max_ctx <= 0 || prompt_tokens + req.max_output <= config_.max_ctx; + if (!continues && new_regions.empty() && !(found_view && raw_fits)) { + stats["admission"] = admission; + return serve_raw(region_candidates.empty() ? "no_region" : "no_region_paged"); + } + + pu::View next; + next.lineage = continues ? view.lineage : ++pflash_unified_lineage_; + next.version = continues ? view.version + 1 : 1; + next.config = pflash_unified_config_; + next.raw_tokens = raw; + next.raw_gen_begin = raw_gen_begin; + next.drafter_ids = drafter_ids; + next.drafter_gen_begin = turn.generation_begin; + next.view_tokens = served; + next.view_gen_begin = (int) (served.size() - generation.size()); + next.regions = std::move(held_regions); + for (auto & region : new_regions) next.regions.push_back(std::move(region)); + next.pieces = pieces; + next.turns = turn_index; + + const char * decision = !new_regions.empty() ? (recall_fired ? "region_recall" : "region") + : recall_fired ? "recall" : continues ? "continue" : "raw"; + if (std::strcmp(decision, "raw") == 0) { + count("raw.reset"); + stats["reason"] = "reset"; + } else { + count(std::string("serve.") + decision); + } + stats["decision"] = decision; + stats["turn"] = turn_index; + stats["lineage"] = next.lineage; + stats["continued"] = continues; + stats["reused_tokens"] = continues ? view.view_gen_begin : 0; + stats["served_tokens"] = served.size(); + stats["over_raw"] = (int) served.size() - prompt_tokens; + stats["admission"] = admission; + stats["regions"] = next.regions.size(); + stats["reread_held_units"] = covered_units; + if (!recall_stats.is_null()) stats["recall"] = recall_stats; + if (!gate_stats.is_null()) stats["recall_gate"] = gate_stats; + if (recall && !scored) stats["recall"] = "unscored"; + json piece_list = json::array(); + for (const auto & piece : pieces) { + piece_list.push_back({piece.begin, piece.end, piece.kind, piece.region, piece.turn}); + } + stats["pieces"] = std::move(piece_list); + + prepared.tokens = served; + prepared.compressed = served != raw; + prepared.unified_view = true; + prepared.snapshot_cut = next.view_gen_begin; + prepared.pflash_stats = { + {"unified", stats}, + {"compress_ms", std::round(compress_ms * 10.0) / 10.0}, + {"drafter_input_tokens", input}, + {"compressed_tokens", served.size()}, + {"drafter_fingerprint_fnv1a64", http_detail::pflash_token_fingerprint(drafter_ids)}, + }; + if (scored || !result.candidate_lifts.empty() || result.ok) { + prepared.pflash_stats["scorer_resume"] = result.scorer_resume; + prepared.pflash_stats["scorer_new_tokens"] = result.scorer_new_tokens; + prepared.pflash_stats["scorer_forward_ms"] = std::round(result.scorer_forward_s * 10000.0) / 10.0; + if (result.spread_valid) prepared.pflash_stats["spread"] = {{"spread", result.spread}, + {"spread_k", result.spread_k}}; + if (result.budget_widened) prepared.pflash_stats["stage2"] = {{"widened", true}}; + } + std::fprintf(stderr, + "[pflash-unified] %s turn=%d served=%zu raw=%d reused=%d regions=+%zu/%zu recall=%s%s%s\n", + decision, turn_index, served.size(), prompt_tokens, continues ? view.view_gen_begin : 0, + new_regions.size(), next.regions.size(), + recall_fired ? "fired" : (recall_stats.is_array() ? "empty" : (recall ? "unscored" : "no")), + disk_cache_.disabled() ? "" : " cached_source=", + disk_cache_.disabled() ? "" : restorable_prefix.source); + std::fflush(stderr); + // A view the RAM store evicts goes to disk (with the disk session cache on), as an evicted KV snapshot does. + for (const auto & evicted : pflash_unified_views_.remember(std::move(next))) { + if (pflash_unified_disk_views_.enabled()) pflash_unified_disk_views_.save(evicted); + } + if (prepared.compressed) trace_pflash_served(req, prepared); + return {}; + } + return serve_raw("reset_failed"); +} + HttpServer::PreparedPrompt HttpServer::prepare_prompt( const ParsedRequest & req) { PreparedPrompt prepared; @@ -7178,7 +8451,11 @@ HttpServer::PreparedPrompt HttpServer::prepare_prompt( // admission and revive dropped history on the next turn. const size_t forgotten = pflash_views_.forget(req.prompt_tokens); if (forgotten > 0) record["views_forgotten"] = forgotten; + const size_t unified_forgotten = forget_unified_views(req.prompt_tokens); + if (unified_forgotten > 0) record["unified_lineages_forgotten"] = unified_forgotten; + const std::string unified_handback = prepared.unified_handback; prepared = std::move(uncompressed); + prepared.unified_handback = unified_handback; prepared.pflash_stats = {{"pflash_fallback", std::move(record)}}; } } @@ -7199,6 +8476,9 @@ HttpServer::PreparedPrompt HttpServer::prepare_prompt( prepared.error = context_overflow_message( config_.max_ctx, prompt_length, req.max_output); } + // A request the chat view serves after a hand-back ends the conversation's unified views; one refused keeps + // them, so a retry can still continue its view. + if (prepared.error.empty() && !prepared.unified_handback.empty()) forget_unified_views(req.prompt_tokens); return prepared; } @@ -7486,7 +8766,8 @@ HttpServer::GenerationCacheState HttpServer::prepare_generation_cache( bool disk_lookup = !cache.using_restore; const int ram_prefix_len = cache.using_restore ? cache.prefix_len : 0; if (cache.using_restore && !disk_cache_.disabled() && - !prepared.compressed && disk_probe.len > 0 && disk_probe.cur_pos > cache.prefix_len && + (!prepared.compressed || prepared.unified_view) && disk_probe.len > 0 && + disk_probe.cur_pos > cache.prefix_len && disk_probe.len < (int) effective_prompt.size()) { lookup_lengths.assign(1, disk_probe.len); disk_lookup = true; @@ -7842,10 +9123,12 @@ void HttpServer::finalize_generation_cache( backend_.snapshot_bytes_estimate(saved_position)); trim_snapshots_after_commit(cache.snap_slot); // The disk key is exact: it must not claim rows the snapshot - // lacks. A compressed view is never written to disk, so it - // gets no key. The write happens only if this snapshot is - // evicted from RAM (evict_inline_slot) or at shutdown. - if (!prepared.compressed && + // lacks. A compressed prompt gets no key, except one served + // from a unified view: that view is kept on disk too, so the + // conversation's next request serves the same tokens again. + // The write happens only if this snapshot is evicted from RAM + // (evict_inline_slot) or at shutdown. + if ((!prepared.compressed || prepared.unified_view) && cache.disk_policy.mode == DiskPrefixCacheMode::Full) { slot_tokens_[cache.snap_slot] = std::vector( effective_prompt.begin(), @@ -8030,6 +9313,13 @@ void HttpServer::flush_sessions_to_disk(const char * reason) { return std::chrono::duration( std::chrono::steady_clock::now() - t0).count(); }; + // The views of shortened conversations first: they are small, and a shortened conversation's KV state is only + // reused through its view. + if (pflash_unified_disk_views_.enabled()) { + size_t views = 0; + for (const auto & view : pflash_unified_views_.all()) views += pflash_unified_disk_views_.save(view) ? 1 : 0; + std::fprintf(stderr, "[pflash-unified] %s flush: %zu views written\n", reason, views); + } size_t queued = 0, backups = 0, skipped = 0; const auto before = disk_cache_.stats(); // Graceful shutdown evicts everything: each conversation's newest diff --git a/server/src/server/http_server.h b/server/src/server/http_server.h index 974f815e0..71c695663 100644 --- a/server/src/server/http_server.h +++ b/server/src/server/http_server.h @@ -32,6 +32,8 @@ #include "adaptive_keep_ratio.h" #include "server_status.h" #include "sse_emitter.h" +#include "pflash_unified.h" +#include "pflash/pflash_selection.h" #include #include @@ -664,6 +666,9 @@ struct PflashFallbackDecision { std::string refused; }; bool pflash_fail_closed_from_env() noexcept; +// Whether a strict selection scores every candidate (the block-15 head over segments), which the unified view pages +// by; the legacy and split scorers (or PFLASH_QWEN35_LEGACY_SCORER=1) and the fold strategy do not. +bool pflash_selection_gives_lifts(const luce::pflash::PFlashSelectionConfig & selection) noexcept; // Short machine-readable reason for a PFlash error string (and the backend's // last error, when the drafter reported one): "nonfinite_scores", // "compress_failed", "ceiling_exceeded", "mapping_failed", "remote_drafter", @@ -752,6 +757,17 @@ int pflash_raw_base_target( const std::vector & prompt, const Tokenizer & drafter, const std::vector & drafter_ids, const std::vector & turns, int turn); +// The restorable prefix PFlash admission counts: the RAM prefix snapshot, or +// the deepest entry of the disk session cache when that cache is on and holds +// more (a restarted server, or a session written out on eviction). Ties go to +// RAM. With the disk cache off it is the RAM prefix, and ``source`` is "ram" +// (or "none" when nothing is cached). +struct PflashRestorablePrefix { + int tokens = 0; + const char * source = "none"; +}; +PflashRestorablePrefix pflash_restorable_prefix( + int ram_tokens, bool disk_enabled, int disk_tokens) noexcept; // Tokens of the canonical ``spans`` inside [begin, end). int pflash_span_tokens_within( const std::vector & spans, int begin, int end) noexcept; @@ -976,6 +992,11 @@ class HttpServer { int snapshot_cut = -1; // PFlash details for usage.timings.pflash (see build_timings_json). nlohmann::json pflash_stats; + // PFLASH_UNIFIED=1: why the unified view handed this request to the chat view ("" = it did not). + std::string unified_handback; + // PFLASH_UNIFIED=1: served from a unified view. A compressed one may still get a disk key (its served tokens): + // the view is kept on disk too, so a resumed conversation serves the same tokens again. + bool unified_view = false; int error_status = 0; std::string error; // The drafter's own error when compression failed in the backend @@ -993,6 +1014,22 @@ class HttpServer { PreparedPrompt & prepared); std::string apply_pflash_compression(const ParsedRequest & req, PreparedPrompt & prepared); + // PFLASH_UNIFIED=1, after the shared query and pins: pages the newest turn's large uncached content + // (a user message, or listed tool outputs), continues the conversation's view, and recalls unheld units of + // earlier regions at a new user question. ``request`` carries the drafter ids, the pinned spans and the + // learned query; ``messages`` the normalised messages the prompt was rendered from. ``handback`` set (the + // reason): the view did not take the request -- ``not_ready`` non-empty, a prompt it cannot cut, or a conversation it would + // start while a threshold's worth of uncached, unpinned prompt lies outside its regions -- and the chat view + // (the release path) serves it; nothing of the unified view's state changed. + std::string apply_unified_view(const ParsedRequest & req, PreparedPrompt & prepared, + const http_detail::PflashChatTurnSpan & turn, + const std::vector & messages, + ModelBackend::CompressRequest request, int chunk_size, + const std::string & not_ready, std::string & handback); + // A conversation that left the unified view (a raw fallback, or a hand-back to the chat view) is not continued + // later from one of its views: drops the lineages these raw tokens would continue, in RAM and on disk. Returns + // how many. + size_t forget_unified_views(const std::vector & raw_tokens); // Multi-turn: serve the conversation's previous view plus this turn's // new material -- verbatim below PFLASH_CHAT_COMPRESS_NEW_TOKENS, else // the parts the fresh selection keeps -- with the segments the fresh @@ -1019,10 +1056,15 @@ class HttpServer { bool keep_history = false); // PFLASH_VIEW_TRACE_PATH: one JSONL record per compressed request with // the served prompt's text, for evidence checks in evaluations. - // Target tokens of ``prompt`` the reader can restore from its in-memory - // prefix cache right now (a live backend snapshot, its authoritative - // position); no cache state changes. - int restorable_prefix_tokens(const std::vector & prompt); + // Target tokens of ``prompt`` the reader can restore right now: its + // in-memory prefix cache (a live backend snapshot, its authoritative + // position) or, when the disk session cache is on and the request's disk + // policy is Full (the scope the generation path looks up), the deepest + // disk entry -- an index lookup, no file read. No cache state changes. + http_detail::PflashRestorablePrefix restorable_prefix_tokens( + const std::vector & prompt, + const DiskPrefixCachePolicy & disk_policy); + int ram_restorable_prefix_tokens(const std::vector & prompt); // Serve a PflashRegionPlan turn from the selection's kept spans; false // when it cannot be served (the caller serves the raw prompt). bool serve_pflash_region_view( @@ -1244,6 +1286,23 @@ class HttpServer { // Multi-turn PFlash views, matched by raw prompt prefix. http_detail::PflashChatViewStore pflash_views_; + // PFLASH_UNIFIED=1, read once at construction: one view and recall mechanism for chat + // and agent sessions. Unset, none of it runs and nothing below is read. + bool pflash_unified_ = false; + bool pflash_unified_recall_ = true; + // PFLASH_RECALL_GATE= (default 0.075 under PFLASH_UNIFIED=1): recall only + // while the head mass on unheld units is at least theta of the context mass; also skips the drafter pass when + // every unit of every region is held. < 0 = "off" (fill each region's cap). + double pflash_recall_gate_ = -1.0; + // PFLASH_RECALL_TRACE_UNITS=1: the view record carries each region's units (byte cuts, kept at paging; at a + // question: held, head mass, cost, score) for an offline replay of recall policies. Stats only. + bool pflash_recall_trace_units_ = false; + pflash_unified::ToolList pflash_region_tools_; + std::string pflash_unified_config_; // config identity a stored view must match + pflash_unified::ViewStore pflash_unified_views_; + pflash_unified::ViewDiskStore pflash_unified_disk_views_; // with the disk session cache on + std::atomic pflash_unified_lineage_{0}; + // Live status tracker (read by /status/json, written by worker thread). ServerStatus status_; @@ -1329,8 +1388,11 @@ class HttpServer { std::map pflash_refused_reasons_; // refused, by why // Tool-session decisions, by name (/props pflash.keep_history). std::map pflash_keep_history_counts_; + // Unified decisions and events, by name (/props pflash.unified). + std::map pflash_unified_counts_; nlohmann::json pflash_failure_counters_json() const; nlohmann::json pflash_keep_history_json() const; + nlohmann::json pflash_unified_json() const; // Active client thread tracking. std::atomic active_clients_{0}; diff --git a/server/src/server/launch_profiles.h b/server/src/server/launch_profiles.h index a56e3ab2b..210ae97ec 100644 --- a/server/src/server/launch_profiles.h +++ b/server/src/server/launch_profiles.h @@ -345,6 +345,9 @@ inline std::vector pflash_release_env() { {"PFLASH_ADMISSION", "cache"}, {"PFLASH_ARRIVE", "1"}, {"PFLASH_CHAT_REBUILD_MIN_TOKENS", "65536"}, + // Chats and agent sessions share the unified view: only the newest + // turn's long content is paged, and recall is gated. + {"PFLASH_UNIFIED", "1"}, }; const bool segment_probe = set("PFLASH_SEGMENT_PROBE_GGUF"); if (segment_probe) env.push_back({"PFLASH_SELECT_SEGMENTS", "probe"}); diff --git a/server/src/server/pflash_unified.cpp b/server/src/server/pflash_unified.cpp new file mode 100644 index 000000000..0755d09bb --- /dev/null +++ b/server/src/server/pflash_unified.cpp @@ -0,0 +1,1224 @@ +#include "server/pflash_unified.h" + +#include "common/sha1.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pflash_unified { + +namespace { + +std::string trim(const std::string & s) { + size_t b = 0; + size_t e = s.size(); + while (b < e && std::isspace((unsigned char) s[b])) ++b; + while (e > b && std::isspace((unsigned char) s[e - 1])) --e; + return s.substr(b, e - b); +} + +bool json_ws(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r'; +} + + +uint64_t fnv1a(const std::string & s) { + uint64_t h = 1469598103934665603ull; + for (unsigned char c : s) { + h ^= c; + h *= 1099511628211ull; + } + return h == 0 ? 1 : h; +} + +} // namespace + +// ── Tool list ────────────────────────────────────────────────────────── + +bool ToolList::listed(const std::string & name) const { + if (all) return true; + return std::find(names.begin(), names.end(), name) != names.end(); +} + +bool parse_tool_list(const std::string & spec, ToolList & out, std::string & error) { + out = ToolList{}; + const std::string s = trim(spec); + if (s.empty()) return true; + std::vector names; + size_t at = 0; + while (true) { + const size_t comma = s.find(',', at); + const std::string entry = trim(s.substr(at, comma == std::string::npos ? std::string::npos : comma - at)); + if (entry.empty()) { + error = "PFLASH_REGION_TOOLS: empty entry in '" + s + "'"; + return false; + } + for (unsigned char c : entry) { + if (c < 0x21 || c == 0x7F || c == '"' || c == '[' || c == ']') { + error = "PFLASH_REGION_TOOLS: invalid character in tool name '" + entry + "'"; + return false; + } + } + names.push_back(entry); + if (comma == std::string::npos) break; + at = comma + 1; + } + const bool star = std::find(names.begin(), names.end(), "*") != names.end(); + if (star && names.size() > 1) { + error = "PFLASH_REGION_TOOLS: '*' must be the only entry"; + return false; + } + if (star) { + out.all = true; + out.identity = "*"; + return true; + } + std::sort(names.begin(), names.end()); + names.erase(std::unique(names.begin(), names.end()), names.end()); + out.names = names; + for (size_t i = 0; i < names.size(); ++i) { + if (i) out.identity += ','; + out.identity += names[i]; + } + return true; +} + +std::string call_brief(const std::string & name, const nlohmann::json & arguments, size_t max_args) { + std::string brief; + for (unsigned char c : name) brief += (c < 0x20 || c == 0x7F) ? ' ' : (char) c; + nlohmann::json args = arguments; + if (args.is_string()) { + const std::string raw = args.get(); + nlohmann::json parsed = nlohmann::json::parse(raw, nullptr, false); + if (!parsed.is_discarded()) args = std::move(parsed); + } + if (args.is_null()) return brief; + if (args.is_object() && args.empty()) return brief; + std::string text = args.dump(-1, ' ', true, nlohmann::json::error_handler_t::replace); + if (text.size() > max_args) text = text.substr(0, max_args) + "\xE2\x80\xA6"; + return brief + " " + text; +} + +// ── Line structure ───────────────────────────────────────────────────── + +Lines split_lines(const std::string & text, size_t begin, size_t end, bool escaped) { + Lines out; + out.escaped = escaped; + out.begin = begin; + out.end = std::min(end, text.size()); + if (out.begin >= out.end) return out; + out.starts.push_back(out.begin); + size_t i = out.begin; + while (i < out.end) { + size_t sep = std::string::npos; + size_t next = i + 1; + if (!escaped) { + if (text[i] == '\n') sep = i; + } else if (text[i] == '\\' && i + 1 < out.end) { + if (text[i + 1] == 'n') sep = i; + next = text[i + 1] == 'u' ? std::min(out.end, i + 6) : i + 2; + } + if (sep != std::string::npos) { + next = sep + out.sep_size(); + out.ends.push_back(sep); + if (next < out.end) out.starts.push_back(next); + } + i = next; + } + if (out.ends.size() < out.starts.size()) out.ends.push_back(out.end); + return out; +} + +std::vector> json_string_values(const std::string & text, size_t begin, size_t end) { + std::vector> out; + end = std::min(end, text.size()); + if (begin >= end) return out; + if (!nlohmann::json::accept(text.begin() + (std::ptrdiff_t) begin, text.begin() + (std::ptrdiff_t) end)) { + return out; + } + size_t i = begin; + while (i < end) { + if (text[i] != '"') { + ++i; + continue; + } + size_t j = i + 1; + while (j < end && text[j] != '"') j += text[j] == '\\' ? 2 : 1; + if (j >= end) break; + size_t k = j + 1; + while (k < end && json_ws(text[k])) ++k; + if (k >= end || text[k] != ':') out.emplace_back(i + 1, j); + i = j + 1; + } + return out; +} + +Lines detect_lines(const std::string & text, size_t begin, size_t end) { + Lines best = split_lines(text, begin, end, false); + for (const auto & v : json_string_values(text, begin, end)) { + Lines escaped = split_lines(text, v.first, v.second, true); + if (escaped.count() > best.count()) best = std::move(escaped); + } + return best; +} + +// ── Region layout ─────────────────────────────────────────────────────── + +size_t line_cut(const Lines & lines, const std::vector & common, int line) { + if (line <= 0 || (size_t) line >= lines.count()) return std::string::npos; + const size_t start = lines.starts[(size_t) line]; + if (std::binary_search(common.begin(), common.end(), start)) return start; + const size_t sep = lines.ends[(size_t) line - 1]; + if (sep > lines.starts[(size_t) line - 1] && std::binary_search(common.begin(), common.end(), sep)) return sep; + return std::string::npos; +} + +bool layout_region(const Lines & lines, const std::vector & common, + const std::vector & proposed, Layout & out, std::string & why) { + out = Layout{}; + const int n = (int) lines.count(); + if (n < 5) { + why = "fewer than five lines"; + return false; + } + int head_line = -1; + size_t head_cut = std::string::npos; + for (int l = 2; l <= n - 3; ++l) { + head_cut = line_cut(lines, common, l); + if (head_cut != std::string::npos) { + head_line = l; + break; + } + } + int tail_line = -1; + size_t tail_cut = std::string::npos; + for (int l = n - 2; l > std::max(head_line, 2); --l) { + tail_cut = line_cut(lines, common, l); + if (tail_cut != std::string::npos) { + tail_line = l; + break; + } + } + if (head_line < 0 || tail_line < 0) { + why = "no clean head or tail cut"; + return false; + } + std::vector open; + for (size_t p : proposed) { + if (p <= lines.begin || p >= lines.end) continue; + int li = (int) (std::upper_bound(lines.starts.begin(), lines.starts.end(), p) - lines.starts.begin()) - 1; + if (li < 0) continue; + if (p >= lines.ends[(size_t) li] && li + 1 < n) ++li; + if (li > head_line && li < tail_line) open.push_back(li); + } + std::sort(open.begin(), open.end()); + open.erase(std::unique(open.begin(), open.end()), open.end()); + out.head_end = head_cut; + out.tail_begin = tail_cut; + out.cuts.push_back(head_cut); + out.cut_line.push_back(head_line); + for (int li : open) { + const size_t c = line_cut(lines, common, li); + if (c == std::string::npos) continue; + out.cuts.push_back(c); + out.cut_line.push_back(li); + } + out.cuts.push_back(tail_cut); + out.cut_line.push_back(tail_line); + return true; +} + +// ── Text ──────────────────────────────────────────────────────────────── + +std::string json_unescape(const std::string & text, size_t begin, size_t end) { + std::string out; + out.reserve(end - begin); + auto hex4 = [&](size_t at, uint32_t & v) { + if (at + 4 > end) return false; + v = 0; + for (size_t k = at; k < at + 4; ++k) { + const char c = text[k]; + v <<= 4; + if (c >= '0' && c <= '9') v |= (uint32_t) (c - '0'); + else if (c >= 'a' && c <= 'f') v |= (uint32_t) (c - 'a' + 10); + else if (c >= 'A' && c <= 'F') v |= (uint32_t) (c - 'A' + 10); + else return false; + } + return true; + }; + auto put_utf8 = [&](uint32_t cp) { + if (cp < 0x80) { + out += (char) cp; + } else if (cp < 0x800) { + out += (char) (0xC0 | (cp >> 6)); + out += (char) (0x80 | (cp & 0x3F)); + } else if (cp < 0x10000) { + out += (char) (0xE0 | (cp >> 12)); + out += (char) (0x80 | ((cp >> 6) & 0x3F)); + out += (char) (0x80 | (cp & 0x3F)); + } else { + out += (char) (0xF0 | (cp >> 18)); + out += (char) (0x80 | ((cp >> 12) & 0x3F)); + out += (char) (0x80 | ((cp >> 6) & 0x3F)); + out += (char) (0x80 | (cp & 0x3F)); + } + }; + size_t i = begin; + while (i < end) { + if (text[i] != '\\' || i + 1 >= end) { + out += text[i++]; + continue; + } + const char c = text[i + 1]; + switch (c) { + case 'n': out += '\n'; i += 2; continue; + case 't': out += '\t'; i += 2; continue; + case 'r': out += '\r'; i += 2; continue; + case 'b': out += '\b'; i += 2; continue; + case 'f': out += '\f'; i += 2; continue; + case '"': case '\\': case '/': out += c; i += 2; continue; + case 'u': { + uint32_t v = 0; + if (!hex4(i + 2, v)) break; + size_t used = 6; + if (v >= 0xD800 && v < 0xDC00 && i + 12 <= end && text[i + 6] == '\\' && text[i + 7] == 'u') { + uint32_t lo = 0; + if (hex4(i + 8, lo) && lo >= 0xDC00 && lo < 0xE000) { + v = 0x10000 + ((v - 0xD800) << 10) + (lo - 0xDC00); + used = 12; + } + } + put_utf8(v); + i += used; + continue; + } + default: break; + } + out += text[i++]; + } + return out; +} + + +std::string json_escape(const std::string & text) { + static const char * hex = "0123456789abcdef"; + std::string out; + out.reserve(text.size() + 8); + for (unsigned char c : text) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + case '\b': out += "\\b"; break; + case '\f': out += "\\f"; break; + default: + if (c < 0x20) { + out += "\\u00"; + out += hex[c >> 4]; + out += hex[c & 0xF]; + } else { + out += (char) c; + } + } + } + return out; +} + +std::string marker_text(int lines_not_shown) { + return "[\xE2\x80\xA6 " + std::to_string(lines_not_shown) + (lines_not_shown == 1 ? " line" : " lines") + + " not shown \xE2\x80\xA6]"; +} + +std::string preamble_text(RegionKind kind, int shown_lines, int total_lines) { + const std::string counts = std::to_string(shown_lines) + " of " + std::to_string(total_lines) + + " lines are shown; each omitted range is marked inline."; + if (kind == RegionKind::ToolOutput) { + return "[Server note: this tool result was shortened: " + counts + + " Call the tool again if you need an omitted part.]"; + } + return "[Server note: this message was shortened: " + counts + "]"; +} + +std::string recall_label_text(RegionKind kind, const std::string & brief) { + if (kind == RegionKind::ToolOutput) return "[From the earlier result of " + brief + "]"; + return "[From an earlier user message]"; +} + +std::string inserted_line(const std::string & text, const Lines & lines, size_t at, size_t next, + const std::string & body) { + const std::string sep = lines.sep(); + const bool before = at > lines.begin && std::binary_search(lines.starts.begin(), lines.starts.end(), at); + bool after = false; + if (next < lines.end && next + sep.size() <= text.size() && text.compare(next, sep.size(), sep) == 0) { + after = std::binary_search(lines.ends.begin(), lines.ends.end(), next); + } + std::string out; + if (!before) out += sep; + out += lines.escaped ? json_escape(body) : body; + if (!after) out += sep; + return out; +} + +std::string render_region_text(const std::string & text, size_t content_begin, size_t content_end, + const Lines & lines, const Layout & layout, const std::vector & admitted, + RegionKind kind) { + const int n = layout.units(); + int dropped = 0; + for (int u = 0; u < n; ++u) { + if (!admitted[(size_t) u]) dropped += layout.line_count(u, u); + } + std::string out = text.substr(content_begin, layout.head_end - content_begin); + out += inserted_line(text, lines, layout.head_end, layout.head_end, + preamble_text(kind, (int) lines.count() - dropped, (int) lines.count())); + for (int u = 0; u < n;) { + if (admitted[(size_t) u]) { + out += text.substr(layout.cuts[(size_t) u], layout.cuts[(size_t) u + 1] - layout.cuts[(size_t) u]); + ++u; + continue; + } + int b = u; + while (b + 1 < n && !admitted[(size_t) b + 1]) ++b; + out += inserted_line(text, lines, layout.cuts[(size_t) u], layout.cuts[(size_t) b + 1], + marker_text(layout.line_count(u, b))); + u = b + 1; + } + out += text.substr(layout.tail_begin, content_end - layout.tail_begin); + return out; +} + +// ── Admission ────────────────────────────────────────────────────────── + +namespace { + +// The maximal run around ``u`` of units whose ``in`` equals ``in[u]``. +std::pair run_around(const std::vector & in, int u) { + int a = u; + int b = u; + while (a > 0 && in[(size_t) a - 1] == in[(size_t) u]) --a; + while (b + 1 < (int) in.size() && in[(size_t) b + 1] == in[(size_t) u]) ++b; + return {a, b}; +} + +} // namespace + +Admission admit_paging(const std::vector & unit_cost, const std::vector & locked, + const std::vector & order, int fixed, int cap, + const std::function & marker_cost) { + Admission out; + const int n = (int) unit_cost.size(); + out.admitted.assign((size_t) n, 0); + for (int u = 0; u < n; ++u) out.admitted[(size_t) u] = (u < (int) locked.size() && locked[(size_t) u]) ? 1 : 0; + out.cost = fixed; + for (int u = 0; u < n;) { + if (out.admitted[(size_t) u]) { + ++u; + continue; + } + const auto run = run_around(out.admitted, u); + out.cost += marker_cost(run.first, run.second); + u = run.second + 1; + } + if (out.cost > cap) return out; + out.ok = true; + for (int u : order) { + if (u < 0 || u >= n || out.admitted[(size_t) u]) continue; + const auto gap = run_around(out.admitted, u); + int delta = unit_cost[(size_t) u] - marker_cost(gap.first, gap.second); + if (gap.first <= u - 1) delta += marker_cost(gap.first, u - 1); + if (u + 1 <= gap.second) delta += marker_cost(u + 1, gap.second); + if (out.cost + delta > cap) continue; + out.admitted[(size_t) u] = 1; + out.cost += delta; + ++out.taken; + } + return out; +} + +namespace { + +// The cost change of taking unit ``u`` into ``admitted``: its tokens, one label for the run it forms, minus the labels +// of the runs it joins. +int recall_delta(const std::vector & admitted, const std::vector & unit_cost, int u, + const std::function & label_cost) { + const int n = (int) admitted.size(); + int a = u; + int b = u; + int delta = unit_cost[(size_t) u]; + if (u > 0 && admitted[(size_t) u - 1]) { + const auto run = run_around(admitted, u - 1); + a = run.first; + delta -= label_cost(run.first, run.second); + } + if (u + 1 < n && admitted[(size_t) u + 1]) { + const auto run = run_around(admitted, u + 1); + b = run.second; + delta -= label_cost(run.first, run.second); + } + return delta + label_cost(a, b); +} + +} // namespace + +Admission admit_recall(const std::vector & unit_cost, const std::vector & held, + const std::vector & order, int cap, + const std::function & label_cost) { + Admission out; + const int n = (int) unit_cost.size(); + out.admitted.assign((size_t) n, 0); + out.ok = cap >= 0; + if (!out.ok) return out; + for (int u : order) { + if (u < 0 || u >= n || out.admitted[(size_t) u]) continue; + if (u < (int) held.size() && held[(size_t) u]) continue; + const int delta = recall_delta(out.admitted, unit_cost, u, label_cost); + if (out.cost + delta > cap) continue; + out.admitted[(size_t) u] = 1; + out.cost += delta; + ++out.taken; + } + return out; +} + +GatedRecall admit_recall_gated(const std::vector & regions, double context_mass, double theta) { + GatedRecall out; + out.regions.resize(regions.size()); + struct Entry { + size_t region; + int unit; + double score; + }; + std::vector order; + double deficit = 0.0; + for (size_t r = 0; r < regions.size(); ++r) { + const RecallRegion & region = regions[r]; + const int n = (int) region.unit_cost.size(); + Admission & a = out.regions[r]; + a.admitted.assign((size_t) n, 0); + a.ok = region.cap >= 0; + for (int u = 0; u < n; ++u) { + if (u < (int) region.held.size() && region.held[(size_t) u]) continue; + const double mass = u < (int) region.mass.size() ? region.mass[(size_t) u] : 0.0; + if (std::isfinite(mass) && mass > 0.0) deficit += mass; + const double score = u < (int) region.score.size() ? region.score[(size_t) u] : NAN; + order.push_back({r, u, std::isnan(score) ? -INFINITY : score}); + } + } + out.deficit_before = deficit / context_mass; + std::stable_sort(order.begin(), order.end(), [] (const Entry & x, const Entry & y) { return x.score > y.score; }); + for (const Entry & e : order) { + if (deficit / context_mass < theta) break; + const RecallRegion & region = regions[e.region]; + Admission & a = out.regions[e.region]; + if (!a.ok) continue; + const auto label_cost = [&] (int, int) { return region.label_cost; }; + const int delta = recall_delta(a.admitted, region.unit_cost, e.unit, label_cost); + if (a.cost + delta > region.cap) continue; + a.admitted[(size_t) e.unit] = 1; + a.cost += delta; + ++a.taken; + const double mass = e.unit < (int) region.mass.size() ? region.mass[(size_t) e.unit] : 0.0; + if (std::isfinite(mass) && mass > 0.0) deficit -= mass; + } + out.deficit_after = (std::max)(0.0, deficit) / context_mass; + out.gated = out.deficit_before < theta; + return out; +} + +std::vector rank_units(const std::vector & score, const std::vector & skip) { + std::vector order; + for (int u = 0; u < (int) score.size(); ++u) { + if (u < (int) skip.size() && skip[(size_t) u]) continue; + order.push_back(u); + } + auto key = [&](int u) { + const double s = score[(size_t) u]; + return std::isnan(s) ? -INFINITY : s; + }; + std::stable_sort(order.begin(), order.end(), [&](int a, int b) { return key(a) > key(b); }); + return order; +} + +// ── Re-reads ─────────────────────────────────────────────────────────── + +std::vector line_hashes(const std::string & text, const Lines & lines) { + std::vector out; + out.reserve(lines.count()); + for (size_t i = 0; i < lines.count(); ++i) { + std::string line = lines.escaped ? json_unescape(text, lines.starts[i], lines.ends[i]) + : text.substr(lines.starts[i], lines.ends[i] - lines.starts[i]); + while (!line.empty() && line.back() == '\r') line.pop_back(); + const bool blank = std::all_of(line.begin(), line.end(), [](unsigned char c) { + return c == ' ' || c == '\t' || c == '\r' || c == '\f' || c == '\v'; + }); + out.push_back(blank ? 0 : fnv1a(line)); + } + return out; +} + +RereadMatch match_reread(const std::vector & candidate, const std::vector & region, + int min_run, int max_repeat) { + RereadMatch out; + out.region_covered.assign(region.size(), 0); + std::unordered_map freq; + for (uint64_t h : region) if (h) ++freq[h]; + auto informative = [&](uint64_t h) { + if (!h) return false; + const auto it = freq.find(h); + return it == freq.end() || it->second <= max_repeat; + }; + std::vector reg; // informative region lines + out.region_informative.assign(region.size(), 0); + for (int i = 0; i < (int) region.size(); ++i) { + if (!informative(region[(size_t) i])) continue; + reg.push_back(i); + out.region_informative[(size_t) i] = 1; + } + std::vector cand; // informative candidate lines + for (int j = 0; j < (int) candidate.size(); ++j) if (informative(candidate[(size_t) j])) cand.push_back(j); + out.candidate_informative = (int) cand.size(); + std::unordered_map> where; + for (int p = 0; p < (int) reg.size(); ++p) where[region[(size_t) reg[(size_t) p]]].push_back(p); + std::vector matched(cand.size(), 0); + std::unordered_map prev; + for (int j = 0; j < (int) cand.size(); ++j) { + std::unordered_map cur; + const auto it = where.find(candidate[(size_t) cand[(size_t) j]]); + if (it != where.end()) { + for (int p : it->second) { + const auto pr = prev.find(p - 1); + const int len = pr == prev.end() ? 1 : pr->second + 1; + cur[p] = len; + if (len < min_run) continue; + const int from = len == min_run ? len - 1 : 0; + for (int k = from; k >= 0; --k) { + matched[(size_t) (j - k)] = 1; + out.region_covered[(size_t) reg[(size_t) (p - k)]] = 1; + } + } + } + prev.swap(cur); + } + for (char m : matched) out.matched_candidate += m ? 1 : 0; + // Skipped region lines between two covered informative lines are covered. + for (size_t k = 0; k + 1 < reg.size(); ++k) { + if (!out.region_covered[(size_t) reg[k]] || !out.region_covered[(size_t) reg[k + 1]]) continue; + for (int i = reg[k] + 1; i < reg[k + 1]; ++i) out.region_covered[(size_t) i] = 1; + } + return out; +} + +bool lines_covered(const std::vector & covered, const std::vector & informative, int begin, int end) { + bool any_informative = false; + bool informative_covered = true; + bool all_covered = true; + for (int line = begin; line < end; ++line) { + const bool is_covered = covered[(size_t) line] != 0; + all_covered = all_covered && is_covered; + if (informative[(size_t) line]) { + any_informative = true; + informative_covered = informative_covered && is_covered; + } + } + return any_informative ? informative_covered : all_covered; +} + +// ── View ─────────────────────────────────────────────────────────────── + +size_t View::bytes() const { + size_t n = sizeof(View) + config.size(); + n += (raw_tokens.size() + drafter_ids.size() + view_tokens.size()) * sizeof(int32_t); + n += pieces.size() * sizeof(Piece); + for (const auto & p : pieces) n += p.kind.size(); + for (const auto & r : regions) { + n += sizeof(Region) + r.brief.size() + r.tool_call_id.size() + r.content_sha.size(); + n += (r.lines.starts.size() + r.lines.ends.size() + r.layout.cuts.size()) * sizeof(size_t); + n += r.layout.cut_line.size() * sizeof(int) + r.hashes.size() * sizeof(uint64_t); + n += (r.unit_tb.size() + r.unit_te.size() + r.unit_db.size() + r.unit_de.size()) * sizeof(int); + n += r.mandatory.size() + r.kept.size() + r.held.size() + r.reread.size(); + } + return n; +} + +namespace { + +bool prefix_of(const std::vector & stored, int gen_begin, const std::vector & request) { + const size_t n = gen_begin < 0 ? stored.size() : std::min(stored.size(), (size_t) gen_begin); + return request.size() >= n && std::equal(stored.begin(), stored.begin() + (std::ptrdiff_t) n, request.begin()); +} + +} // namespace + +bool ViewStore::find(const std::vector & raw_tokens, const std::vector & drafter_ids, + const std::string & config, View & out) { + std::lock_guard lock(mutex_); + int best = -1; + size_t best_len = 0; + for (int i = 0; i < (int) entries_.size(); ++i) { + const View & v = entries_[(size_t) i].view; + if (v.config != config) continue; + if (!prefix_of(v.raw_tokens, v.raw_gen_begin, raw_tokens)) continue; + if (!prefix_of(v.drafter_ids, v.drafter_gen_begin, drafter_ids)) continue; + const size_t len = v.raw_gen_begin < 0 ? v.raw_tokens.size() : (size_t) v.raw_gen_begin; + if (best < 0 || len > best_len || + (len == best_len && entries_[(size_t) i].used > entries_[(size_t) best].used)) { + best = i; + best_len = len; + } + } + if (best < 0) return false; + entries_[(size_t) best].used = ++clock_; + out = entries_[(size_t) best].view; + return true; +} + +std::vector ViewStore::remember(View view) { + std::lock_guard lock(mutex_); + std::vector evicted; + for (auto it = entries_.begin(); it != entries_.end(); ++it) { + const View & v = it->view; + if (v.config == view.config && v.raw_tokens == view.raw_tokens && v.drafter_ids == view.drafter_ids) { + entries_.erase(it); + break; + } + } + entries_.push_back(Entry{std::move(view), ++clock_}); + auto total = [&] { + size_t n = 0; + for (const auto & e : entries_) n += e.view.bytes(); + return n; + }; + while (entries_.size() > 1 && (entries_.size() > capacity_ || total() > byte_cap_)) { + size_t oldest = 0; + for (size_t i = 1; i + 1 < entries_.size(); ++i) { + if (entries_[i].used < entries_[oldest].used) oldest = i; + } + evicted.push_back(std::move(entries_[oldest].view)); + entries_.erase(entries_.begin() + (std::ptrdiff_t) oldest); + } + return evicted; +} + +std::vector ViewStore::all() const { + std::lock_guard lock(mutex_); + std::vector order; + for (const auto & e : entries_) order.push_back(&e); + std::sort(order.begin(), order.end(), [] (const Entry * a, const Entry * b) { return a->used > b->used; }); + std::vector out; + for (const Entry * e : order) out.push_back(e->view); + return out; +} + +void ViewStore::forget_lineage(uint64_t lineage) { + std::lock_guard lock(mutex_); + entries_.erase(std::remove_if(entries_.begin(), entries_.end(), + [&] (const Entry & e) { return e.view.lineage == lineage; }), + entries_.end()); +} + +std::vector ViewStore::lineages_continued_by(const std::vector & raw_tokens) const { + std::lock_guard lock(mutex_); + std::vector out; + for (const auto & e : entries_) { + if (prefix_of(e.view.raw_tokens, e.view.raw_gen_begin, raw_tokens)) out.push_back(e.view.lineage); + } + return out; +} + +size_t ViewStore::size() const { + std::lock_guard lock(mutex_); + return entries_.size(); +} + +size_t ViewStore::bytes() const { + std::lock_guard lock(mutex_); + size_t n = 0; + for (const auto & e : entries_) n += e.view.bytes(); + return n; +} + +namespace { + +std::string bits(const std::vector & v) { + std::string out; + for (char b : v) out += b ? '1' : '0'; + return out; +} + +bool unbits(const nlohmann::json & j, std::vector & out) { + const std::string s = j.get(); + out.clear(); + for (char ch : s) { + if (ch != '0' && ch != '1') return false; + out.push_back(ch == '1' ? 1 : 0); + } + return true; +} + +uint64_t token_hash(const std::vector & tokens, size_t n) { + uint64_t h = 1469598103934665603ULL; + const auto * p = reinterpret_cast(tokens.data()); + for (size_t i = 0; i < n * sizeof(int32_t); ++i) { + h ^= p[i]; + h *= 1099511628211ULL; + } + return h; +} + +// token_hash(tokens, n) for every n in ``lengths``, in one pass over the tokens. +std::unordered_map prefix_hashes(const std::vector & tokens, std::vector lengths) { + std::sort(lengths.begin(), lengths.end()); + std::unordered_map out; + uint64_t h = 1469598103934665603ULL; + const auto * p = reinterpret_cast(tokens.data()); + size_t at = 0; + for (size_t n : lengths) { + for (; at < n * sizeof(int32_t); ++at) { + h ^= p[at]; + h *= 1099511628211ULL; + } + out[n] = h; + } + return out; +} + +size_t gen_prefix(const std::vector & tokens, int gen_begin) { + return gen_begin < 0 ? tokens.size() : std::min(tokens.size(), (size_t) gen_begin); +} + +uint64_t now_ms() { + return (uint64_t) std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); +} + +// Everything serving indexes without a check: lines inside the content and in order, one hash and one re-read flag +// per line, at least one unit, cuts and their lines strictly ascending from the head end to the tail start, and +// per-unit token ranges that tile the content's tokens in order. +bool region_consistent(const Region & r, std::string & error) { + const auto fail = [&] (const char * what) { + error = std::string("region ") + what; + return false; + }; + const Lines & l = r.lines; + const size_t n = l.count(); + if (n == 0 || l.begin < r.cb || l.end > r.ce || l.begin > l.end) return fail("lines"); + for (size_t i = 0; i < n; ++i) { + if (l.starts[i] > l.ends[i] || l.starts[i] < l.begin || l.ends[i] > l.end || + (i > 0 && l.starts[i] < l.ends[i - 1])) { + return fail("lines"); + } + } + if (r.hashes.size() != n || r.reread.size() != n) return fail("per-line vectors"); + const Layout & y = r.layout; + if (y.cuts.size() < 2 || y.cuts.front() != y.head_end || y.cuts.back() != y.tail_begin) return fail("layout"); + for (size_t i = 0; i < y.cuts.size(); ++i) { + if (y.cuts[i] < l.begin || y.cuts[i] > l.end || y.cut_line[i] < 1 || (size_t) y.cut_line[i] >= n || + (i > 0 && (y.cuts[i] <= y.cuts[i - 1] || y.cut_line[i] <= y.cut_line[i - 1]))) { + return fail("layout"); + } + } + const auto tiles = [] (const std::vector & b, const std::vector & e, int lo, int hi) { + for (size_t u = 0; u < b.size(); ++u) { + if (b[u] < lo || b[u] > e[u] || e[u] > hi || (u > 0 && b[u] != e[u - 1])) return false; + } + return true; + }; + if (!tiles(r.unit_tb, r.unit_te, r.tb, r.te) || !tiles(r.unit_db, r.unit_de, r.db, r.de)) return fail("units"); + return true; +} + +} // namespace + +nlohmann::json view_to_json(const View & view) { + using nlohmann::json; + json regions = json::array(); + for (const auto & r : view.regions) { + regions.push_back({ + {"kind", r.kind == RegionKind::ToolOutput ? "tool_output" : "user_content"}, + {"brief", r.brief}, {"tool_call_id", r.tool_call_id}, {"content_sha", r.content_sha}, + {"cb", r.cb}, {"ce", r.ce}, {"tb", r.tb}, {"te", r.te}, {"db", r.db}, {"de", r.de}, + {"lines", {{"escaped", r.lines.escaped}, {"begin", r.lines.begin}, {"end", r.lines.end}, + {"starts", r.lines.starts}, {"ends", r.lines.ends}}}, + {"layout", {{"head_end", r.layout.head_end}, {"tail_begin", r.layout.tail_begin}, + {"cuts", r.layout.cuts}, {"cut_line", r.layout.cut_line}}}, + {"hashes", r.hashes}, + {"unit_tb", r.unit_tb}, {"unit_te", r.unit_te}, {"unit_db", r.unit_db}, {"unit_de", r.unit_de}, + {"mandatory", bits(r.mandatory)}, {"kept", bits(r.kept)}, {"held", bits(r.held)}, + {"reread", bits(r.reread)}, + {"cap", r.cap}, {"droppable", r.droppable}, {"recalled_tokens", r.recalled_tokens}, + }); + } + json pieces = json::array(); + for (const auto & p : view.pieces) pieces.push_back(json::array({p.begin, p.end, p.kind, p.region, p.turn})); + return {{"format", 1}, {"lineage", view.lineage}, {"version", view.version}, {"config", view.config}, + {"raw_tokens", view.raw_tokens}, {"raw_gen_begin", view.raw_gen_begin}, + {"drafter_ids", view.drafter_ids}, {"drafter_gen_begin", view.drafter_gen_begin}, + {"view_tokens", view.view_tokens}, {"view_gen_begin", view.view_gen_begin}, + {"regions", regions}, {"pieces", pieces}, {"turns", view.turns}}; +} + +bool view_from_json(const nlohmann::json & j, View & out, std::string & error) { + try { + if (j.at("format").get() != 1) { + error = "format"; + return false; + } + View v; + v.lineage = j.at("lineage").get(); + v.version = j.at("version").get(); + v.config = j.at("config").get(); + v.raw_tokens = j.at("raw_tokens").get>(); + v.raw_gen_begin = j.at("raw_gen_begin").get(); + v.drafter_ids = j.at("drafter_ids").get>(); + v.drafter_gen_begin = j.at("drafter_gen_begin").get(); + v.view_tokens = j.at("view_tokens").get>(); + v.view_gen_begin = j.at("view_gen_begin").get(); + v.turns = j.at("turns").get(); + const auto within = [] (int at, size_t size) { return at >= 0 && (size_t) at <= size; }; + if (!within(v.raw_gen_begin, v.raw_tokens.size()) || !within(v.drafter_gen_begin, v.drafter_ids.size()) || + v.view_gen_begin <= 0 || !within(v.view_gen_begin, v.view_tokens.size())) { + error = "generation offsets"; + return false; + } + for (const auto & jr : j.at("regions")) { + Region r; + const std::string kind = jr.at("kind").get(); + if (kind != "tool_output" && kind != "user_content") { + error = "region kind"; + return false; + } + r.kind = kind == "tool_output" ? RegionKind::ToolOutput : RegionKind::UserContent; + r.brief = jr.at("brief").get(); + r.tool_call_id = jr.at("tool_call_id").get(); + r.content_sha = jr.at("content_sha").get(); + r.cb = jr.at("cb").get(); + r.ce = jr.at("ce").get(); + r.tb = jr.at("tb").get(); + r.te = jr.at("te").get(); + r.db = jr.at("db").get(); + r.de = jr.at("de").get(); + const auto & jl = jr.at("lines"); + r.lines.escaped = jl.at("escaped").get(); + r.lines.begin = jl.at("begin").get(); + r.lines.end = jl.at("end").get(); + r.lines.starts = jl.at("starts").get>(); + r.lines.ends = jl.at("ends").get>(); + const auto & jy = jr.at("layout"); + r.layout.head_end = jy.at("head_end").get(); + r.layout.tail_begin = jy.at("tail_begin").get(); + r.layout.cuts = jy.at("cuts").get>(); + r.layout.cut_line = jy.at("cut_line").get>(); + r.hashes = jr.at("hashes").get>(); + r.unit_tb = jr.at("unit_tb").get>(); + r.unit_te = jr.at("unit_te").get>(); + r.unit_db = jr.at("unit_db").get>(); + r.unit_de = jr.at("unit_de").get>(); + if (!unbits(jr.at("mandatory"), r.mandatory) || !unbits(jr.at("kept"), r.kept) || + !unbits(jr.at("held"), r.held) || !unbits(jr.at("reread"), r.reread)) { + error = "region flags"; + return false; + } + r.cap = jr.at("cap").get(); + r.droppable = jr.at("droppable").get(); + r.recalled_tokens = jr.at("recalled_tokens").get(); + const size_t units = (size_t) r.layout.units(); + const size_t lines = r.lines.count(); + if (r.cb > r.ce || r.tb < 0 || r.tb > r.te || (size_t) r.te > v.raw_tokens.size() || r.db < 0 || + r.db > r.de || (size_t) r.de > v.drafter_ids.size() || r.lines.ends.size() != lines || + r.layout.cut_line.size() != r.layout.cuts.size() || r.unit_tb.size() != units || + r.unit_te.size() != units || r.unit_db.size() != units || r.unit_de.size() != units || + r.mandatory.size() != units || r.kept.size() != units || r.held.size() != units) { + error = "region shape"; + return false; + } + if (r.te > v.raw_gen_begin || r.de > v.drafter_gen_begin) { + error = "region after the generation prompt"; + return false; + } + if (!region_consistent(r, error)) return false; + v.regions.push_back(std::move(r)); + } + for (const auto & jp : j.at("pieces")) { + Piece p; + p.begin = jp.at(0).get(); + p.end = jp.at(1).get(); + p.kind = jp.at(2).get(); + p.region = jp.at(3).get(); + p.turn = jp.at(4).get(); + if (p.begin < 0 || p.begin > p.end || (size_t) p.end > v.view_tokens.size() || p.region < -1 || + p.region >= (int) v.regions.size()) { + error = "piece"; + return false; + } + v.pieces.push_back(std::move(p)); + } + out = std::move(v); + return true; + } catch (const std::exception & e) { + error = e.what(); + return false; + } +} + +bool ViewDiskStore::init(const std::string & dir, const std::string & identity, uint64_t ttl_seconds, size_t cap) { + namespace fs = std::filesystem; + std::lock_guard lock(mutex_); + enabled_ = false; + metas_.clear(); + last_written_ = 0; + std::error_code ec; + fs::create_directories(dir, ec); + if (ec || !fs::is_directory(dir, ec)) { + std::fprintf(stderr, "[pflash-unified] views cannot be kept on disk (%s): RAM only\n", dir.c_str()); + return false; + } + dir_ = dir; + identity_ = identity; + ttl_ = ttl_seconds; + cap_ = cap; + const uint64_t now = this->now(); + size_t foreign = 0; + for (const auto & entry : fs::directory_iterator(dir, ec)) { + const fs::path path = entry.path(); + if (path.extension() == ".tmp") { // a write the previous process did not finish + fs::remove(path, ec); + continue; + } + if (path.extension() != ".view") continue; + std::ifstream in(path); + std::string line; + Meta m; + std::string id; + bool ok = false; + if (std::getline(in, line)) { + try { + const auto h = nlohmann::json::parse(line); + id = h.at("identity").get(); + m.lineage = h.at("lineage").get(); + m.config = h.at("config").get(); + m.raw_len = h.at("raw_len").get(); + m.raw_hash = h.at("raw_hash").get(); + m.drafter_len = h.at("drafter_len").get(); + m.drafter_hash = h.at("drafter_hash").get(); + m.written = h.at("written").get(); + ok = true; + } catch (const std::exception &) { + } + } + if (!ok || (ttl_ > 0 && m.written + ttl_ * 1000 < now)) { + fs::remove(path, ec); + continue; + } + if (id != identity_) { // another model's or config's views: never read, deleted after the TTL + ++foreign; + continue; + } + m.path = path.string(); + metas_.push_back(std::move(m)); + } + // Oldest written first; later saves are stamped after the newest one here, so the cap drops the earliest. + std::sort(metas_.begin(), metas_.end(), [] (const Meta & a, const Meta & b) { return a.written < b.written; }); + last_written_ = metas_.empty() ? 0 : metas_.back().written; + enabled_ = true; + enforce_cap_locked(); + std::fprintf(stderr, "[pflash-unified] views on disk: %zu in %s (%zu of another identity)\n", metas_.size(), + dir_.c_str(), foreign); + return true; +} + +bool ViewDiskStore::enabled() const { + std::lock_guard lock(mutex_); + return enabled_; +} + +bool ViewDiskStore::save(const View & view) { + namespace fs = std::filesystem; + std::lock_guard lock(mutex_); + if (!enabled_) return false; + Meta m; + m.lineage = view.lineage; + m.config = view.config; + m.raw_len = gen_prefix(view.raw_tokens, view.raw_gen_begin); + m.raw_hash = token_hash(view.raw_tokens, m.raw_len); + m.drafter_len = gen_prefix(view.drafter_ids, view.drafter_gen_begin); + m.drafter_hash = token_hash(view.drafter_ids, m.drafter_len); + m.written = std::max(now(), last_written_ + 1); + last_written_ = m.written; + const nlohmann::json header = { + {"format", 1}, {"identity", identity_}, {"lineage", m.lineage}, {"config", m.config}, + {"raw_len", m.raw_len}, {"raw_hash", m.raw_hash}, {"drafter_len", m.drafter_len}, + {"drafter_hash", m.drafter_hash}, {"written", m.written}}; + char name[32]; + std::snprintf(name, sizeof(name), "%016llx.view", + (unsigned long long) fnv1a(m.config + ":" + std::to_string(m.raw_len) + ":" + + std::to_string(m.raw_hash) + ":" + std::to_string(m.drafter_len) + + ":" + std::to_string(m.drafter_hash))); + m.path = dir_ + "/" + name; + const std::string tmp = m.path + ".tmp"; + { + std::ofstream out(tmp, std::ios::binary | std::ios::trunc); + out << header.dump() << '\n' << view_to_json(view).dump() << '\n'; + out.flush(); + if (!out) { + std::error_code ec; + fs::remove(tmp, ec); + return false; + } + } + std::error_code ec; + fs::rename(tmp, m.path, ec); + if (ec) { + fs::remove(tmp, ec); + return false; + } + metas_.erase(std::remove_if(metas_.begin(), metas_.end(), [&] (const Meta & x) { return x.path == m.path; }), + metas_.end()); + metas_.push_back(std::move(m)); + enforce_cap_locked(); + return true; +} + +bool ViewDiskStore::find(const std::vector & raw_tokens, const std::vector & drafter_ids, + const std::string & config, size_t longer_than, View & out) { + std::lock_guard lock(mutex_); + if (!enabled_) return false; + // The TTL counts from a view's last write. A view read here returns to the RAM store and is written again when + // it leaves it, so a conversation in use stays on disk. + if (ttl_ > 0) { + const uint64_t t = now(); + for (size_t i = metas_.size(); i-- > 0;) { + if (metas_[i].written + ttl_ * 1000 < t) remove_locked(i); + } + } + std::vector candidates, raw_lengths, drafter_lengths; + for (size_t i = 0; i < metas_.size(); ++i) { + const Meta & m = metas_[i]; + if (m.config != config || m.raw_len <= longer_than || m.raw_len > raw_tokens.size() || + m.drafter_len > drafter_ids.size()) { + continue; + } + candidates.push_back(i); + raw_lengths.push_back(m.raw_len); + drafter_lengths.push_back(m.drafter_len); + } + if (candidates.empty()) return false; + // Each request prefix is hashed once, however many views are on disk. + const auto raw_hashes = prefix_hashes(raw_tokens, raw_lengths); + const auto drafter_hashes = prefix_hashes(drafter_ids, drafter_lengths); + std::vector order; + for (size_t i : candidates) { + if (raw_hashes.at(metas_[i].raw_len) == metas_[i].raw_hash && + drafter_hashes.at(metas_[i].drafter_len) == metas_[i].drafter_hash) { + order.push_back(i); + } + } + std::sort(order.begin(), order.end(), [&] (size_t a, size_t b) { + return metas_[a].raw_len != metas_[b].raw_len ? metas_[a].raw_len > metas_[b].raw_len + : metas_[a].written > metas_[b].written; + }); + for (size_t i : order) { + std::ifstream in(metas_[i].path); + std::string header, body; + View v; + std::string error = "unreadable"; + bool ok = false; + if (std::getline(in, header) && std::getline(in, body)) { + try { + ok = view_from_json(nlohmann::json::parse(body), v, error); + } catch (const std::exception & e) { + error = e.what(); + } + } + ok = ok && v.config == config && prefix_of(v.raw_tokens, v.raw_gen_begin, raw_tokens) && + prefix_of(v.drafter_ids, v.drafter_gen_begin, drafter_ids); + if (!ok) { + std::fprintf(stderr, "[pflash-unified] view file %s does not load (%s): deleted\n", + metas_[i].path.c_str(), error.c_str()); + remove_locked(i); + return false; + } + out = std::move(v); + return true; + } + return false; +} + +void ViewDiskStore::forget_lineage(uint64_t lineage) { + std::lock_guard lock(mutex_); + for (size_t i = metas_.size(); i-- > 0;) { + if (metas_[i].lineage == lineage) remove_locked(i); + } +} + +std::vector ViewDiskStore::lineages_continued_by(const std::vector & raw_tokens) const { + std::lock_guard lock(mutex_); + std::vector out; + if (!enabled_) return out; + std::vector lengths; + for (const auto & m : metas_) { + if (m.raw_len <= raw_tokens.size()) lengths.push_back(m.raw_len); + } + const auto hashes = prefix_hashes(raw_tokens, lengths); + for (const auto & m : metas_) { + if (m.raw_len <= raw_tokens.size() && hashes.at(m.raw_len) == m.raw_hash) out.push_back(m.lineage); + } + return out; +} + +size_t ViewDiskStore::size() const { + std::lock_guard lock(mutex_); + return metas_.size(); +} + +uint64_t ViewDiskStore::max_lineage() const { + std::lock_guard lock(mutex_); + uint64_t out = 0; + for (const auto & m : metas_) out = std::max(out, m.lineage); + return out; +} + +void ViewDiskStore::set_clock(std::function clock) { + std::lock_guard lock(mutex_); + clock_ = std::move(clock); +} + +uint64_t ViewDiskStore::now() const { + return clock_ ? clock_() : now_ms(); +} + +void ViewDiskStore::enforce_cap_locked() { + while (metas_.size() > cap_) { + size_t oldest = 0; + for (size_t i = 1; i < metas_.size(); ++i) { + if (metas_[i].written < metas_[oldest].written) oldest = i; + } + remove_locked(oldest); + } +} + +void ViewDiskStore::remove_locked(size_t index) { + std::error_code ec; + std::filesystem::remove(metas_[index].path, ec); + metas_.erase(metas_.begin() + (std::ptrdiff_t) index); +} + +std::string sha1_hex(const std::string & bytes) { + uint8_t digest[20]; + luce::common::sha1_hash(bytes.data(), bytes.size(), digest); + static const char * hex = "0123456789abcdef"; + std::string out; + for (uint8_t b : digest) { + out += hex[b >> 4]; + out += hex[b & 0xF]; + } + return out; +} + +} // namespace pflash_unified diff --git a/server/src/server/pflash_unified.h b/server/src/server/pflash_unified.h new file mode 100644 index 000000000..cf727f9c9 --- /dev/null +++ b/server/src/server/pflash_unified.h @@ -0,0 +1,340 @@ +#pragma once + +// PFLASH_UNIFIED=1: one view/recall mechanism for chat and agent sessions +// (server/README.md, PFlash). +// The pure parts: the tool list, line-structure detection, region layout, +// marker and label text, greedy admission of units under a cap, re-read +// matching and the view store. Nothing here knows a harness's output format, +// and nothing here tokenises: callers pass byte offsets of the rendered prompt +// text and token costs. + +#include + +#include +#include +#include +#include +#include +#include + +namespace pflash_unified { + +// ── Tool list (PFLASH_REGION_TOOLS) ─────────────────────────────────── + +// Which tool outputs may become regions: tools the agent can call again before the next user question (recall +// only runs on a user question). Names only; ``all`` for "*". +struct ToolList { + bool all = false; + std::vector names; + std::string identity; // canonical text (part of the config identity) + bool listed(const std::string & name) const; +}; + +// ``spec``: comma-separated tool names, or "*". Empty = none. False with ``error`` on an empty or malformed entry +// (the server refuses to start). +bool parse_tool_list(const std::string & spec, ToolList & out, std::string & error); + +// "NAME ARGS" for a recall label: the arguments as compact ASCII JSON (``arguments`` may be an object or a +// JSON-encoded string), cut at ``max_args`` bytes with "…". +std::string call_brief(const std::string & name, const nlohmann::json & arguments, size_t max_args = 120); + +// ── Line structure ───────────────────────────────────────────────────── + +// The lines of text[begin, end): plain ('\n' separates; a CR stays in its line) or escaped (inside a JSON string: +// the two bytes "\n" separate, other escapes are skipped whole). ``starts[i]`` is where line i begins, ``ends[i]`` +// where its separator begins (``end`` for the last line). A separator at ``end`` opens no empty last line. +struct Lines { + bool escaped = false; + size_t begin = 0; + size_t end = 0; + std::vector starts; + std::vector ends; + size_t count() const { return starts.size(); } + size_t sep_size() const { return escaped ? 2 : 1; } + const char * sep() const { return escaped ? "\\n" : "\n"; } +}; +Lines split_lines(const std::string & text, size_t begin, size_t end, bool escaped); + +// The raw byte ranges (between the quotes) of the string values of text[begin, end) when it is exactly one JSON +// value, surrounding whitespace allowed; keys are not values. Empty when it is not JSON. +std::vector> json_string_values(const std::string & text, size_t begin, size_t end); + +// The structure to page: the plain lines of the content, or the escaped lines of one of its JSON string +// values, whichever has the most lines; ties go to plain. +Lines detect_lines(const std::string & text, size_t begin, size_t end); + +// ── Region layout ─────────────────────────────────────────────────────── + +// A region's lines split into a pinned head (at least its first two lines), a pinned tail (at least its last two +// lines) and units between cuts. Every cut is a line_cut(): a position that starts a token in both tokenisations +// (``common``), at the start of the line it opens or of the separator before it. +struct Layout { + size_t head_end = 0; // pinned head [content begin, head_end) + size_t tail_begin = 0; // pinned tail [tail_begin, content end) + std::vector cuts; // cuts.front() == head_end, cuts.back() == tail_begin + std::vector cut_line; // the line each cut opens (cut_line.back(): the tail's first line) + int units() const { return cuts.empty() ? 0 : (int) cuts.size() - 1; } + int first_line(int unit) const { return cut_line[(size_t) unit]; } + int line_count(int first_unit, int last_unit) const { + return cut_line[(size_t) last_unit + 1] - cut_line[(size_t) first_unit]; + } +}; + +// The cut opening ``line`` (> 0), so that kept lines stay whole: the line's start when that is a common boundary, +// else the start of the separator before it when that is one and lies after the previous line's start; npos +// otherwise (the line opens no unit). +size_t line_cut(const Lines & lines, const std::vector & common, int line); + +// ``common``: ascending byte offsets that start a token in both tokenisations. ``proposed``: candidate cut offsets +// (the drafter's candidate starts: probe segments or fixed chunks); each opens the line it falls in and +// is kept when that line has a cut; a boundary at a separator opens the next line. The head ends at the first line +// >= 2 with a cut, the tail starts at the last line <= count - 2 with one. False with ``why`` when there are fewer +// than five lines, no head/tail cut, or no unit between them. +bool layout_region(const Lines & lines, const std::vector & common, + const std::vector & proposed, Layout & out, std::string & why); + +// ── Text ──────────────────────────────────────────────────────────────── + +enum class RegionKind { UserContent, ToolOutput }; + +// ``text`` escaped for the inside of a JSON string. +std::string json_escape(const std::string & text); +// The decoded bytes of text[begin, end) read as the inside of a JSON string; malformed escapes stay as written. +std::string json_unescape(const std::string & text, size_t begin, size_t end); + +std::string marker_text(int lines_not_shown); +std::string preamble_text(RegionKind kind, int shown_lines, int total_lines); +std::string recall_label_text(RegionKind kind, const std::string & brief); +inline const char * recall_open() { return "[Earlier in this conversation]\n"; } +inline const char * recall_close() { return "\n[End of earlier excerpts]\n\n"; } + +// ``body`` as a line of its own at a cut in ``lines``' structure: a separator before it unless the text before +// ``at`` ends with one, a separator after it unless the text from ``next`` starts with one; JSON-escaped in +// escaped mode. +std::string inserted_line(const std::string & text, const Lines & lines, size_t at, size_t next, + const std::string & body); + +// The served text of a paged region's content text[content_begin, content_end): the pinned head, the preamble, +// each unit raw when ``admitted`` and one marker per run of dropped units (counting their lines), the pinned tail. +// The server serves the same pieces as token slices; this is their text, for parity checks and tests. +std::string render_region_text(const std::string & text, size_t content_begin, size_t content_end, + const Lines & lines, const Layout & layout, const std::vector & admitted, + RegionKind kind); + +// ── Admission ────────────────────────────────────────────────────────── + +struct Admission { + bool ok = false; + std::vector admitted; // per unit (locked units included) + int cost = 0; // fixed + optional units + markers (or labels) + int taken = 0; // optional units admitted +}; + +// Paging: ``locked`` units are kept whatever the cap (mandatory, served on top). The rest start dropped and +// ``fixed`` + Σ marker_cost(first, last) over the gaps is the starting cost. Units are taken in ``order`` (best +// first) while the cost after taking one -- its tokens plus the change in marker cost of the gap it splits -- stays +// within ``cap``. ``ok`` false when even the starting cost exceeds the cap. +Admission admit_paging(const std::vector & unit_cost, const std::vector & locked, + const std::vector & order, int fixed, int cap, + const std::function & marker_cost); + +// Recall: ``held`` units are never taken; the others in ``order`` while Σ unit tokens + Σ label_cost(first, last) +// over the runs of consecutive taken units stays within ``cap``. +Admission admit_recall(const std::vector & unit_cost, const std::vector & held, + const std::vector & order, int cap, + const std::function & label_cost); + +// PFLASH_RECALL_GATE: recall what the question needs. One region's candidates: +// per unit its token cost, held flag, score (mean lift: the ranking key) and head mass (its share of the head's +// mass on the whole prompt); the region's cap and the label cost of one excerpt run. +struct RecallRegion { + std::vector unit_cost; + std::vector held; + std::vector score; + std::vector mass; + int cap = 0; + int label_cost = 0; +}; + +struct GatedRecall { + std::vector regions; // per region, as admit_recall's + double deficit_before = 0.0; // D / S before any unit: the head mass on unheld units over S + double deficit_after = 0.0; // D / S after the admitted units + bool gated = false; // deficit_before < theta: nothing taken, the present context suffices +}; + +// ``context_mass`` = S (> 0): the head mass on every candidate that is not pinned (history, held and unheld units). +// D = the mass of the unheld units. When D / S < ``theta`` nothing is taken. Else unheld units are taken best first +// ACROSS regions (one order by score; region then document order on ties), each under its own region's cap with +// admit_recall's run and label accounting (a unit that does not fit is skipped), and the taking stops as soon as the +// mass still unheld is below theta x S. The caps bound it: the worst case is admit_recall's fill. +GatedRecall admit_recall_gated(const std::vector & regions, double context_mass, double theta); + +// Units best first: higher score first, document order on ties; ``skip`` units are left out. +std::vector rank_units(const std::vector & score, const std::vector & skip); + +// ── Re-reads ────────────────────────────────────────────────────────── + +// Line fingerprints of a structure (CR stripped; blank lines 0). +std::vector line_hashes(const std::string & text, const Lines & lines); + +// Lines of ``region`` matched by ``candidate``. Informative lines: non-blank and occurring at most ``max_repeat`` +// times in the region (blank and over-frequent lines such as "}" are skipped on both sides, never breaking a run). +// A match is a run of >= ``min_run`` consecutive informative candidate lines equal to consecutive informative +// region lines. A skipped region line counts as covered when the informative lines on both sides of it are. +struct RereadMatch { + std::vector region_covered; // per region line + std::vector region_informative; // per region line + int matched_candidate = 0; // informative candidate lines inside a match + int candidate_informative = 0; + bool is_reread() const { return candidate_informative > 0 && 2 * matched_candidate >= candidate_informative; } +}; +RereadMatch match_reread(const std::vector & candidate, const std::vector & region, + int min_run = 3, int max_repeat = 32); + +// Whether lines [begin, end) are covered as a whole: every informative line is (blank and over-frequent lines at +// the edges do not count), or, when none is informative, every line is. +bool lines_covered(const std::vector & covered, const std::vector & informative, int begin, int end); + +// ── View ─────────────────────────────────────────────────────────────── + +struct Region { + RegionKind kind = RegionKind::UserContent; + std::string brief; // call_brief() of a tool output ("" for user content) + std::string tool_call_id; + std::string content_sha; // hex digest of the raw content bytes + size_t cb = 0, ce = 0; // content bytes in the rendered prompt text + int tb = 0, te = 0; // target tokens of the content + int db = 0, de = 0; // drafter tokens of the content + Lines lines; + Layout layout; + std::vector hashes; // line_hashes(lines) + std::vector unit_tb, unit_te; // per unit, target tokens + std::vector unit_db, unit_de; // per unit, drafter tokens + std::vector mandatory; // per unit + std::vector kept; // served when paged + std::vector held; // kept, recalled or re-read + std::vector reread; // per line, covered by a re-read + int cap = 0; // optional served tokens at paging, before stage-2 widening (target) + int droppable = 0; // target tokens of the optional units + int recalled_tokens = 0; // cumulative appended by recall (target) +}; + +struct Piece { + int begin = 0; // served target tokens [begin, end) + int end = 0; + std::string kind; // raw, paged_text, preamble, marker, recall_wrapper, recall_label, recall_text, generation + int region = -1; + int turn = -1; +}; + +struct View { + uint64_t lineage = 0; + int version = 0; + std::string config; + std::vector raw_tokens; // target, raw prompt + int raw_gen_begin = -1; + std::vector drafter_ids; + int drafter_gen_begin = -1; + std::vector view_tokens; // target, served + int view_gen_begin = -1; + std::vector regions; + std::vector pieces; + int turns = 0; + size_t bytes() const; +}; + +// Views by raw prefix. ``find``: the longest stored view whose raw tokens up to its generation prompt prefix this +// request's (target and drafter alike) and whose config identity matches; newest on ties; a copy. ``remember`` +// adds a view and evicts the least recently used views over ``capacity`` or ``byte_cap``, never the newest one; +// the view it continued stays, so a fork or an edited request continues it. +class ViewStore { +public: + ViewStore(size_t capacity = 16, size_t byte_cap = (size_t) 256 << 20) + : capacity_(capacity), byte_cap_(byte_cap) {} + bool find(const std::vector & raw_tokens, const std::vector & drafter_ids, + const std::string & config, View & out); + // Returns the views it evicted (the server writes them to disk when the disk session cache is on). + std::vector remember(View view); + // Every stored view, most recently used first (the shutdown flush). + std::vector all() const; + // Drops every view of ``lineage`` (each turn's view is its own entry, so an ancestor would match next). + void forget_lineage(uint64_t lineage); + // The lineages of the views a request with these raw tokens could continue (raw prefix up to the generation + // prompt, any config). + std::vector lineages_continued_by(const std::vector & raw_tokens) const; + size_t size() const; + size_t bytes() const; + +private: + struct Entry { + View view; + uint64_t used = 0; + }; + mutable std::mutex mutex_; + size_t capacity_; + size_t byte_cap_; + uint64_t clock_ = 0; + std::vector entries_; +}; + +// ── Views on disk (the SSD session cache) ───────────────────────────── + +// A view as JSON, every field (its disk form), and back. ``view_from_json`` fails closed: false with ``error`` for +// anything that is not a complete view whose offsets and per-unit and per-line vectors agree. +nlohmann::json view_to_json(const View & view); +bool view_from_json(const nlohmann::json & j, View & view, std::string & error); + +// Views of shortened conversations kept on disk next to the KV cache, so a conversation survives a restart or the +// RAM store's eviction: written when a view leaves the RAM store and at shutdown, read on a RAM miss. One file per +// view: a one-line header (identity, config, prefix lengths and hashes) that ``init`` indexes, then the view. A view +// written under another ``identity`` (the disk cache's salt: model, template, context) is never read; any file older +// than ``ttl_seconds`` (0: never) is deleted at ``init``; over ``cap`` views the least recently written go. +class ViewDiskStore { +public: + bool init(const std::string & dir, const std::string & identity, uint64_t ttl_seconds, size_t cap); + bool enabled() const; + // Writes ``view`` (a temporary file, then a rename), replacing the file of the same prefixes and config. + bool save(const View & view); + // ViewStore::find over the views on disk whose raw prefix is longer than ``longer_than`` tokens. The file is read + // and both prefixes compared token by token before it is returned; a file that does not load is deleted. + bool find(const std::vector & raw_tokens, const std::vector & drafter_ids, + const std::string & config, size_t longer_than, View & out); + void forget_lineage(uint64_t lineage); + // ViewStore::lineages_continued_by over the views on disk, from their headers (no file is read). + std::vector lineages_continued_by(const std::vector & raw_tokens) const; + size_t size() const; + // The largest lineage on disk: a restarted server numbers its new conversations after it. + uint64_t max_lineage() const; + // Milliseconds since the epoch for write stamps and the TTL (tests inject one). + void set_clock(std::function clock); + +private: + struct Meta { + std::string path; + uint64_t lineage = 0; + std::string config; + size_t raw_len = 0; + size_t drafter_len = 0; + uint64_t raw_hash = 0; + uint64_t drafter_hash = 0; + uint64_t written = 0; // ms since the epoch + }; + void enforce_cap_locked(); + void remove_locked(size_t index); + mutable std::mutex mutex_; + bool enabled_ = false; + std::string dir_; + std::string identity_; + uint64_t ttl_ = 0; + size_t cap_ = 0; + uint64_t last_written_ = 0; // writes are strictly ordered, also within one millisecond + std::function clock_; + uint64_t now() const; + std::vector metas_; +}; + +// Hex SHA-1 of ``bytes`` (region identity). +std::string sha1_hex(const std::string & bytes); + +} // namespace pflash_unified diff --git a/server/test/pflash/test_pflash_query_candidates.cpp b/server/test/pflash/test_pflash_query_candidates.cpp index 9b0d95ec4..301d82fa5 100644 --- a/server/test/pflash/test_pflash_query_candidates.cpp +++ b/server/test/pflash/test_pflash_query_candidates.cpp @@ -162,6 +162,7 @@ TEST_CASE(PFlashQueryCandidatesFixture, release_profile_is_the_default_with_a_tr luce_test::ScopedEnvVar head{"PFLASH_SCORING_HEAD_GGUF", "head.gguf"}; REQUIRE(has("PFLASH_SELECT_MODE") && !has("PFLASH_SELECT_SEGMENTS") && !has("PFLASH_SELECT_QUERY_PROBE")); + REQUIRE(has("PFLASH_UNIFIED")); { Vars vars; install(vars); diff --git a/server/test/pflash/test_pflash_unified.cpp b/server/test/pflash/test_pflash_unified.cpp new file mode 100644 index 000000000..ba5ce00b0 --- /dev/null +++ b/server/test/pflash/test_pflash_unified.cpp @@ -0,0 +1,671 @@ +#include "CppUnitTestFramework.hpp" + +#include "server/pflash_unified.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include + +using namespace pflash_unified; + +namespace { + +struct PFlashUnifiedFixture : CppUnitTestFramework::CommonFixture { + using CppUnitTestFramework::CommonFixture::CommonFixture; +}; + +// "line 1" .. "line n", '\n'-joined, with a trailing newline. +std::string numbered_text(int n, const char * prefix = "line ") { + std::string out; + for (int i = 1; i <= n; ++i) out += prefix + std::to_string(i) + "\n"; + return out; +} + +// Every line start and separator start: both tokenisations agree everywhere. +std::vector all_boundaries(const Lines & lines) { + std::vector out; + for (size_t i = 0; i < lines.count(); ++i) { + out.push_back(lines.starts[i]); + out.push_back(lines.ends[i]); + } + std::sort(out.begin(), out.end()); + out.erase(std::unique(out.begin(), out.end()), out.end()); + return out; +} + +std::string line_at(const std::string & text, const Lines & lines, size_t i) { + return text.substr(lines.starts[i], lines.ends[i] - lines.starts[i]); +} + +} // namespace + +TEST_CASE(PFlashUnifiedFixture, test_unified_region_tools_parse) { + ToolList t; + std::string err; + REQUIRE(parse_tool_list("", t, err)); + REQUIRE(!t.all); + REQUIRE(t.names.empty()); + REQUIRE(!t.listed("read_file")); + + REQUIRE(parse_tool_list(" search_files , read_file,read_file ", t, err)); + REQUIRE(t.names == std::vector({"read_file", "search_files"})); + REQUIRE(t.identity == "read_file,search_files"); + REQUIRE(t.listed("read_file")); + REQUIRE(!t.listed("terminal")); + + REQUIRE(parse_tool_list("*", t, err)); + REQUIRE(t.all); + REQUIRE(t.identity == "*"); + REQUIRE(t.listed("anything")); + + REQUIRE(!parse_tool_list("*,read_file", t, err)); + REQUIRE(!parse_tool_list("a,,b", t, err)); + REQUIRE(!parse_tool_list("read file", t, err)); + REQUIRE(!parse_tool_list("a[0]", t, err)); +} + +TEST_CASE(PFlashUnifiedFixture, test_unified_labels_name_and_arguments) { + const nlohmann::json obj = {{"path", "docs/a.md"}, {"offset", 10}}; + REQUIRE(call_brief("read_file", obj) == "read_file {\"offset\":10,\"path\":\"docs/a.md\"}"); + // JSON-encoded arguments (the OpenAI wire form) read the same. + REQUIRE(call_brief("read_file", nlohmann::json(obj.dump())) == call_brief("read_file", obj)); + // Newlines and non-ASCII never reach the label raw; long arguments are cut. + const std::string brief = call_brief("run\ncmd", {{"cmd", "echo a\nb \xC3\xA9"}}); + REQUIRE(brief.find('\n') == std::string::npos); + REQUIRE(brief.find("\\n") != std::string::npos); + REQUIRE(brief.find("\\u00e9") != std::string::npos); + const std::string long_brief = call_brief("t", {{"q", std::string(500, 'x')}}, 40); + REQUIRE(long_brief.size() == 2 + 40 + 3); + REQUIRE(long_brief.substr(long_brief.size() - 3) == "\xE2\x80\xA6"); + REQUIRE(call_brief("list", nlohmann::json::object()) == "list"); + REQUIRE(call_brief("list", nullptr) == "list"); + + REQUIRE(recall_label_text(RegionKind::ToolOutput, "read_file {\"path\":\"a\"}") == + "[From the earlier result of read_file {\"path\":\"a\"}]"); + REQUIRE(recall_label_text(RegionKind::UserContent, "") == "[From an earlier user message]"); +} + +TEST_CASE(PFlashUnifiedFixture, unified_split_lines_plain_and_escaped) { + const std::string plain = "a\r\nbb\n\nc\n"; + const Lines p = split_lines(plain, 0, plain.size(), false); + REQUIRE(p.count() == 4); + REQUIRE(line_at(plain, p, 0) == "a\r"); + REQUIRE(line_at(plain, p, 1) == "bb"); + REQUIRE(line_at(plain, p, 2) == ""); + REQUIRE(line_at(plain, p, 3) == "c"); + REQUIRE(split_lines(plain, 0, plain.size() - 1, false).count() == 4); + REQUIRE(split_lines(plain, 0, 0, false).count() == 0); + + // Inside a JSON string: "\n" separates, an escaped backslash followed by n does not, \u escapes are skipped. + const std::string esc = "1|a\\n2|b\\\\n still 2\\n3|\\u005cn"; + const Lines e = split_lines(esc, 0, esc.size(), true); + REQUIRE(e.count() == 3); + REQUIRE(line_at(esc, e, 1) == "2|b\\\\n still 2"); + REQUIRE(line_at(esc, e, 2) == "3|\\u005cn"); + REQUIRE(std::string(e.sep()) == "\\n"); +} + +TEST_CASE(PFlashUnifiedFixture, test_unified_line_model_json_string) { + // A JSON-wrapped file read (the shape Hermes and many MCP tools return): the lines live in a string value. + std::string body; + for (int i = 1; i <= 40; ++i) body += std::to_string(i) + "|text " + std::to_string(i) + "\n"; + const nlohmann::json result = {{"content", body}, {"total_lines", 40}, {"_hint", "a\nb"}}; + const std::string text = result.dump(); + const Lines lines = detect_lines(text, 0, text.size()); + REQUIRE(lines.escaped); + REQUIRE(lines.count() == 40); + REQUIRE(line_at(text, lines, 0) == "1|text 1"); + REQUIRE(line_at(text, lines, 39) == "40|text 40"); + + // The same lines as plain text are found as plain lines; a pretty-printed JSON whose plain lines outnumber + // its string's lines stays plain. + const Lines plain = detect_lines(body, 0, body.size()); + REQUIRE(!plain.escaped); + REQUIRE(plain.count() == 40); + nlohmann::json many = nlohmann::json::array(); + for (int i = 0; i < 30; ++i) many.push_back({{"k", i}, {"v", "x\ny"}}); + const std::string pretty = many.dump(2); + REQUIRE(!detect_lines(pretty, 0, pretty.size()).escaped); + REQUIRE(json_string_values("not json {", 0, 10).empty()); + const std::string keys = "{\"a\\nb\\nc\\nd\\ne\\nf\": 1}"; + REQUIRE(json_string_values(keys, 0, keys.size()).empty()); + + // Paging inside the string keeps the result valid JSON with every other field unchanged. + std::vector common = all_boundaries(lines); + std::vector proposed; + for (int l = 5; l < 40; l += 5) proposed.push_back(lines.starts[(size_t) l]); + Layout layout; + std::string why; + REQUIRE(layout_region(lines, common, proposed, layout, why)); + std::vector admitted((size_t) layout.units(), 0); + admitted[1] = 1; + const std::string served = render_region_text(text, 0, text.size(), lines, layout, admitted, + RegionKind::ToolOutput); + const nlohmann::json parsed = nlohmann::json::parse(served, nullptr, false); + REQUIRE(!parsed.is_discarded()); + REQUIRE(parsed["total_lines"] == 40); + REQUIRE(parsed["_hint"] == "a\nb"); + const std::string content = parsed["content"].get(); + REQUIRE(content.find("1|text 1\n2|text 2\n[Server note: this tool result was shortened:") == 0); + REQUIRE(content.find("[\xE2\x80\xA6 3 lines not shown \xE2\x80\xA6]\n6|text 6\n") != std::string::npos); + REQUIRE(content.find("\n39|text 39\n40|text 40\n") != std::string::npos); +} + +TEST_CASE(PFlashUnifiedFixture, test_unified_markers_count_lines) { + // CRLF content without a trailing newline: served bytes keep CRLF; markers count the dropped lines. + std::string text; + for (int i = 1; i <= 12; ++i) text += "row " + std::to_string(i) + (i < 12 ? "\r\n" : ""); + const Lines lines = detect_lines(text, 0, text.size()); + REQUIRE(lines.count() == 12); + std::vector proposed; + for (int l = 3; l < 12; l += 2) proposed.push_back(lines.starts[(size_t) l] + 1); // mid-line: opens that line + Layout layout; + std::string why; + REQUIRE(layout_region(lines, all_boundaries(lines), proposed, layout, why)); + REQUIRE(layout.cut_line == std::vector({2, 3, 5, 7, 9, 10})); + std::vector admitted = {1, 0, 0, 1, 0}; + const std::string served = render_region_text(text, 0, text.size(), lines, layout, admitted, + RegionKind::UserContent); + const std::string expect = + "row 1\r\nrow 2\r\n" + "[Server note: this message was shortened: 7 of 12 lines are shown; each omitted range is marked inline.]\n" + "row 3\r\n" + "[\xE2\x80\xA6 4 lines not shown \xE2\x80\xA6]\n" + "row 8\r\nrow 9\r\n" + "[\xE2\x80\xA6 1 line not shown \xE2\x80\xA6]\n" + "row 11\r\nrow 12"; + REQUIRE(served == expect); + REQUIRE(marker_text(2) == "[\xE2\x80\xA6 2 lines not shown \xE2\x80\xA6]"); +} + +TEST_CASE(PFlashUnifiedFixture, unified_layout_cuts_only_at_clean_line_boundaries) { + const std::string text = numbered_text(20); + const Lines lines = detect_lines(text, 0, text.size()); + // Separator starts only (a tokenizer that glues the newline to the next word): cuts sit before the separator. + std::vector seps; + for (size_t i = 0; i < lines.count(); ++i) seps.push_back(lines.ends[i]); + Layout layout; + std::string why; + REQUIRE(layout_region(lines, seps, {lines.starts[10]}, layout, why)); + REQUIRE(layout.cuts.front() == lines.ends[1]); + REQUIRE(layout.cuts[1] == lines.ends[9]); + REQUIRE(layout.cuts.back() == lines.ends[17]); + std::vector admitted = {0, 1}; + const std::string served = render_region_text(text, 0, text.size(), lines, layout, admitted, + RegionKind::UserContent); + REQUIRE(served.find("line 2\n[Server note:") != std::string::npos); + REQUIRE(served.find("inline.]\n[\xE2\x80\xA6 8 lines not shown \xE2\x80\xA6]\nline 11\n") != std::string::npos); + + // A line with no clean boundary opens no unit; the edges move inward to the first clean line. + std::vector sparse; + for (int l : {3, 8, 16}) sparse.push_back(lines.starts[(size_t) l]); + REQUIRE(layout_region(lines, sparse, {lines.starts[5], lines.starts[8]}, layout, why)); + REQUIRE(layout.cut_line == std::vector({3, 8, 16})); + + // Too few lines, or no clean cut at all: never a region. + const std::string four = numbered_text(4); + const Lines four_lines = detect_lines(four, 0, four.size()); + REQUIRE(!layout_region(four_lines, all_boundaries(four_lines), {}, layout, why)); + REQUIRE(!layout_region(lines, {}, {}, layout, why)); + const std::string one_line(5000, 'x'); + REQUIRE(detect_lines(one_line, 0, one_line.size()).count() == 1); +} + +TEST_CASE(PFlashUnifiedFixture, unified_admit_paging_counts_markers_and_respects_cap) { + // Five units of 10 tokens, marker cost 3 per gap, preamble 4. + const std::vector cost = {10, 10, 10, 10, 10}; + auto marker = [](int, int) { return 3; }; + const std::vector order = {2, 0, 4, 1, 3}; + // Start: one gap (3) + preamble (4) = 7. Unit 2 splits the gap: +10 +3 = 20. Unit 0: +10 +0 = 30. Unit 4: + // +10 + 0 = 40 > 35, skipped; unit 1 joins [0..2]: +10 -3 = 37 > 35, skipped; unit 3: +10 -3 +3 = 40, skipped. + Admission a = admit_paging(cost, {}, order, 4, 35, marker); + REQUIRE(a.ok); + REQUIRE(a.admitted == std::vector({1, 0, 1, 0, 0})); + REQUIRE(a.cost == 30); + REQUIRE(a.taken == 2); + // Locked units are kept and cost nothing against the cap. + a = admit_paging(cost, {0, 0, 0, 0, 1}, order, 4, 7, marker); + REQUIRE(a.ok); + REQUIRE(a.admitted == std::vector({0, 0, 0, 0, 1})); + REQUIRE(a.cost == 7); + // A cap below the preamble plus one marker cannot page. + REQUIRE(!admit_paging(cost, {}, order, 4, 6, marker).ok); +} + +TEST_CASE(PFlashUnifiedFixture, unified_admit_recall_skips_held_and_charges_labels_per_run) { + const std::vector cost = {10, 10, 10, 10, 10}; + auto label = [](int, int) { return 5; }; + // Held 1; order 2, 3, 0, 4: unit 2 = 15; unit 3 joins the run: +10 = 25; unit 0 new run: +15 = 40 > 30. + Admission a = admit_recall(cost, {0, 1, 0, 0, 0}, {2, 3, 0, 4}, 30, label); + REQUIRE(a.admitted == std::vector({0, 0, 1, 1, 0})); + REQUIRE(a.cost == 25); + // Unit 4 fits after unit 0 is skipped. + a = admit_recall(cost, {0, 1, 0, 0, 0}, {2, 3, 0, 4}, 35, label); + REQUIRE(a.admitted == std::vector({0, 0, 1, 1, 1})); + REQUIRE(a.cost == 35); + // Bridging two runs merges their labels. + a = admit_recall(cost, {}, {0, 2, 1}, 1000, label); + REQUIRE(a.cost == 35); + // Worst case: every unheld unit comes back once, nothing more. + a = admit_recall(cost, {1, 0, 1, 0, 0}, {0, 1, 2, 3, 4}, 1000, label); + REQUIRE(a.admitted == std::vector({0, 1, 0, 1, 1})); +} + +TEST_CASE(PFlashUnifiedFixture, unified_recall_gate_appends_nothing_when_the_present_context_suffices) { + RecallRegion r; + r.unit_cost = {10, 10, 10, 10}; + r.held = {1, 0, 0, 1}; + r.score = {9.0, 2.0, 1.0, 9.0}; + r.mass = {0.40, 0.03, 0.01, 0.30}; // held units carry most of the mass; D = 0.04 + r.cap = 1000; + r.label_cost = 5; + GatedRecall g = admit_recall_gated({r}, 1.0, 0.05); + REQUIRE(g.gated); + REQUIRE(g.regions[0].taken == 0); + REQUIRE(g.regions[0].admitted == std::vector({0, 0, 0, 0})); + REQUIRE(std::fabs(g.deficit_before - 0.04) < 1e-12); + REQUIRE(std::fabs(g.deficit_after - 0.04) < 1e-12); + // Held units never count towards D, whatever their mass. + r.mass = {5.0, 0.03, 0.01, 5.0}; + REQUIRE(std::fabs(admit_recall_gated({r}, 1.0, 0.05).deficit_before - 0.04) < 1e-12); +} + +TEST_CASE(PFlashUnifiedFixture, unified_recall_gate_stops_once_the_deficit_is_covered) { + RecallRegion r; + r.unit_cost = {10, 10, 10, 10, 10}; + r.held = {0, 0, 0, 0, 0}; + r.score = {1.0, 8.0, 2.0, 0.5, 0.4}; + r.mass = {0.02, 0.30, 0.05, 0.01, 0.01}; // D = 0.39 of S = 1 + r.cap = 1000; + r.label_cost = 5; + // theta .10: unit 1 leaves .09 < .10: one unit, nothing more. + GatedRecall g = admit_recall_gated({r}, 1.0, 0.10); + REQUIRE(!g.gated); + REQUIRE(g.regions[0].admitted == std::vector({0, 1, 0, 0, 0})); + REQUIRE(g.regions[0].cost == 15); + REQUIRE(std::fabs(g.deficit_after - 0.09) < 1e-12); + // theta .05: unit 1 (.09 left), unit 2 (.04 left): stop. + g = admit_recall_gated({r}, 1.0, 0.05); + REQUIRE(g.regions[0].admitted == std::vector({0, 1, 1, 0, 0})); + REQUIRE(g.regions[0].cost == 25); + // S scales the share: the same units against S = 4 are under .10 at the start. + REQUIRE(admit_recall_gated({r}, 4.0, 0.10).gated); +} + +TEST_CASE(PFlashUnifiedFixture, unified_recall_gate_at_zero_fills_like_admit_recall_and_never_past_the_cap) { + RecallRegion r; + r.unit_cost = {10, 10, 10, 10, 10}; + r.held = {0, 1, 0, 0, 0}; + r.score = {3.0, 9.0, 5.0, 4.0, 1.0}; + r.mass = {0.1, 0.5, 0.1, 0.1, 0.1}; + r.cap = 30; + r.label_cost = 5; + const auto label = [](int, int) { return 5; }; + const Admission fill = admit_recall(r.unit_cost, r.held, rank_units(r.score, r.held), r.cap, label); + const GatedRecall g = admit_recall_gated({r}, 1.0, 0.0); + REQUIRE(g.regions[0].admitted == fill.admitted); + REQUIRE(g.regions[0].cost == fill.cost); + REQUIRE(g.regions[0].cost <= r.cap); + // Diffuse mass under a strict theta: still bounded by the cap. + REQUIRE(admit_recall_gated({r}, 1.0, 0.01).regions[0].cost <= r.cap); +} + +TEST_CASE(PFlashUnifiedFixture, unified_recall_gate_orders_units_across_regions_under_their_own_caps) { + RecallRegion a; + a.unit_cost = {10, 10}; + a.held = {0, 0}; + a.score = {1.0, 6.0}; + a.mass = {0.05, 0.10}; + a.cap = 15; + a.label_cost = 5; + RecallRegion b; + b.unit_cost = {10, 10}; + b.held = {0, 0}; + b.score = {7.0, 2.0}; + b.mass = {0.20, 0.05}; + b.cap = 1000; + b.label_cost = 5; + // D = .40. Order: b0 (7), a1 (6), b1 (2), a0 (1). theta .12: b0 -> .20, a1 -> .10 < .12: stop. + GatedRecall g = admit_recall_gated({a, b}, 1.0, 0.12); + REQUIRE(g.regions[0].admitted == std::vector({0, 1})); + REQUIRE(g.regions[1].admitted == std::vector({1, 0})); + // theta .01: a0 does not fit a's cap (15 + 10 + label change) and is skipped; b1 is taken. + g = admit_recall_gated({a, b}, 1.0, 0.01); + REQUIRE(g.regions[0].admitted == std::vector({0, 1})); + REQUIRE(g.regions[0].cost <= a.cap); + REQUIRE(g.regions[1].admitted == std::vector({1, 1})); +} + +TEST_CASE(PFlashUnifiedFixture, unified_rank_units_orders_by_score_then_document) { + REQUIRE(rank_units({0.5, 2.0, 0.5, NAN, 3.0}, {}) == std::vector({4, 1, 0, 2, 3})); + REQUIRE(rank_units({0.5, 2.0, 0.5}, {0, 1, 0}) == std::vector({0, 2})); +} + +TEST_CASE(PFlashUnifiedFixture, test_unified_reread_by_content) { + std::string file; + for (int i = 1; i <= 100; ++i) file += "def f" + std::to_string(i) + "():\n return " + std::to_string(i) + "\n}\n\n"; + const Lines region_lines = detect_lines(file, 0, file.size()); + const auto region = line_hashes(file, region_lines); + + // A partial re-read of the same file, delivered JSON-wrapped: decoded lines compare equal. + std::string part; + for (int i = 40; i <= 60; ++i) part += "def f" + std::to_string(i) + "():\n return " + std::to_string(i) + "\n}\n\n"; + const std::string wrapped = nlohmann::json({{"content", part}}).dump(); + const Lines part_lines = detect_lines(wrapped, 0, wrapped.size()); + REQUIRE(part_lines.escaped); + RereadMatch m = match_reread(line_hashes(wrapped, part_lines), region); + REQUIRE(m.is_reread()); + REQUIRE(m.matched_candidate == m.candidate_informative); + REQUIRE(m.candidate_informative == 21 * 2); // "}" (100 copies) and blank lines are skipped + int covered = 0; + for (char c : m.region_covered) covered += c; + REQUIRE(covered == 21 * 2 + 20 * 2); // + the "}" and blank lines between matched blocks + REQUIRE(!m.region_covered[(size_t) (59 * 4 + 2)]); // the "}" after the last block: next line unmatched + REQUIRE(m.region_covered[(size_t) (39 * 4)]); // "def f40():" + REQUIRE(!m.region_covered[(size_t) (38 * 4)]); // "def f39():" + REQUIRE(m.region_informative[(size_t) (39 * 4)]); + REQUIRE(!m.region_informative[(size_t) (39 * 4 + 2)]); // "}" + REQUIRE(!m.region_informative[(size_t) (39 * 4 + 3)]); // blank + // A unit ending in the last block's "}" and blank line is covered: its informative lines are; one reaching + // into f61 is not. + REQUIRE(lines_covered(m.region_covered, m.region_informative, 59 * 4, 60 * 4)); + REQUIRE(!lines_covered(m.region_covered, m.region_informative, 59 * 4, 60 * 4 + 1)); + // Lines with nothing informative count only when covered themselves (between two covered blocks). + REQUIRE(lines_covered(m.region_covered, m.region_informative, 39 * 4 + 2, 40 * 4)); + REQUIRE(!lines_covered(m.region_covered, m.region_informative, 59 * 4 + 2, 60 * 4)); + + // A changed file matches only where it is unchanged; "}" lines (more than 32 copies) never match alone. + std::string changed = part; + for (char & c : changed) if (c == 'r') c = 'R'; + m = match_reread(line_hashes(changed, detect_lines(changed, 0, changed.size())), region); + REQUIRE(m.matched_candidate == 0); + REQUIRE(!m.is_reread()); + const std::string unrelated = numbered_text(30, "other "); + m = match_reread(line_hashes(unrelated, detect_lines(unrelated, 0, unrelated.size())), region); + REQUIRE(!m.is_reread()); +} + +TEST_CASE(PFlashUnifiedFixture, test_unified_reread_multiple_regions) { + const std::string v1 = numbered_text(50, "alpha "); + std::string v2 = v1; + v2.replace(v2.find("alpha 20\n"), 9, "ALPHA 20\n"); + const auto h1 = line_hashes(v1, detect_lines(v1, 0, v1.size())); + const auto h2 = line_hashes(v2, detect_lines(v2, 0, v2.size())); + const std::string reread = numbered_text(25, "alpha "); + const auto hr = line_hashes(reread, detect_lines(reread, 0, reread.size())); + const RereadMatch m1 = match_reread(hr, h1); + const RereadMatch m2 = match_reread(hr, h2); + REQUIRE(m1.region_covered[19]); + REQUIRE(!m2.region_covered[19]); + REQUIRE(m2.region_covered[18]); + REQUIRE(m2.region_covered[20]); +} + +TEST_CASE(PFlashUnifiedFixture, unified_view_store_finds_longest_prefix_and_keeps_ancestors) { + ViewStore store(3, (size_t) 1 << 30); + auto make = [](std::vector raw, int gen, const std::string & cfg) { + View v; + v.raw_tokens = raw; + v.raw_gen_begin = gen; + v.drafter_ids = raw; + v.drafter_gen_begin = gen; + v.config = cfg; + return v; + }; + store.remember(make({1, 2, 3, 9}, 3, "c")); + store.remember(make({1, 2, 3, 4, 5, 9}, 5, "c")); + View out; + REQUIRE(store.find({1, 2, 3, 4, 5, 6, 7}, {1, 2, 3, 4, 5, 6, 7}, "c", out)); + REQUIRE(out.raw_gen_begin == 5); + REQUIRE(store.find({1, 2, 3, 7}, {1, 2, 3, 7}, "c", out)); // a fork continues the ancestor + REQUIRE(out.raw_gen_begin == 3); + REQUIRE(!store.find({1, 2, 3, 4, 5, 6}, {1, 2, 3, 4, 5, 6}, "other", out)); + REQUIRE(!store.find({1, 2, 3, 4, 5, 6}, {1, 2, 9, 4, 5, 6}, "c", out)); // drafter ids must match too + // A retry replaces its own entry; capacity evicts the least recently used, never the newest. + store.remember(make({1, 2, 3, 4, 5, 9}, 5, "c")); + REQUIRE(store.size() == 2); + store.remember(make({7, 7}, 1, "c")); + store.remember(make({8, 8}, 1, "c")); + REQUIRE(store.size() == 3); + REQUIRE(!store.find({1, 2, 3, 0}, {1, 2, 3, 0}, "c", out)); + REQUIRE(store.find({8, 8, 1}, {8, 8, 1}, "c", out)); +} + +// A small view with one region whose per-unit and per-line vectors agree. +static View disk_test_view(const std::vector & raw, int gen, const std::string & config, uint64_t lineage) { + View v; + v.lineage = lineage; + v.version = 2; + v.config = config; + v.raw_tokens = raw; + v.raw_gen_begin = gen; + v.drafter_ids = raw; + v.drafter_gen_begin = gen; + v.view_tokens = {raw.front(), 99, raw.back()}; + v.view_gen_begin = 2; + v.turns = 1; + Region r; + r.kind = RegionKind::ToolOutput; + r.brief = "read_file(a.md)"; + r.tool_call_id = "call_1"; + r.content_sha = "ab"; + r.cb = 3; + r.ce = 30; + r.te = 2; + r.de = 2; + r.lines.escaped = true; + r.lines.begin = 3; + r.lines.end = 30; + r.lines.starts = {3, 12, 21}; + r.lines.ends = {10, 19, 30}; + r.layout.head_end = 12; + r.layout.tail_begin = 21; + r.layout.cuts = {12, 21}; + r.layout.cut_line = {1, 2}; + r.hashes = {0, 18446744073709551615ULL, 7}; + r.unit_tb = {0}; + r.unit_te = {1}; + r.unit_db = {0}; + r.unit_de = {1}; + r.mandatory = {0}; + r.kept = {1}; + r.held = {1}; + r.reread = {0, 1, 0}; + r.cap = 5; + r.droppable = 9; + r.recalled_tokens = 2; + v.regions.push_back(r); + v.pieces.push_back({0, 1, "raw", -1, 0}); + v.pieces.push_back({1, 2, "paged_text", 0, 0}); + return v; +} + +TEST_CASE(PFlashUnifiedFixture, unified_view_json_round_trip_fails_closed) { + const View v = disk_test_view({1, 2, 3, 4, 9}, 4, "c", 42); + View back; + std::string error; + REQUIRE(view_from_json(view_to_json(v), back, error)); + REQUIRE(view_to_json(back) == view_to_json(v)); + REQUIRE(back.regions[0].hashes[1] == 18446744073709551615ULL); + REQUIRE(back.lineage == 42 && back.regions[0].kind == RegionKind::ToolOutput && back.regions[0].lines.escaped); + const auto broken = [&] (const std::function & edit) { + auto j = view_to_json(v); + edit(j); + View out; + std::string why; + return !view_from_json(j, out, why) && !why.empty(); + }; + REQUIRE(broken([] (nlohmann::json & j) { j["format"] = 2; })); + REQUIRE(broken([] (nlohmann::json & j) { j.erase("raw_tokens"); })); + REQUIRE(broken([] (nlohmann::json & j) { j["view_gen_begin"] = 9; })); + REQUIRE(broken([] (nlohmann::json & j) { j["regions"][0]["kept"] = "11"; })); // one unit + REQUIRE(broken([] (nlohmann::json & j) { j["regions"][0]["held"] = "x"; })); + REQUIRE(broken([] (nlohmann::json & j) { j["regions"][0]["te"] = 50; })); // past the raw prompt + REQUIRE(broken([] (nlohmann::json & j) { j["regions"][0]["kind"] = "other"; })); + REQUIRE(broken([] (nlohmann::json & j) { j["pieces"][0][1] = 50; })); // past the view + REQUIRE(broken([] (nlohmann::json & j) { j["pieces"][1][3] = 3; })); // no such region + REQUIRE(broken([] (nlohmann::json & j) { j["pieces"][1][3] = -2; })); + // Vectors serving indexes per line or per unit, and the offsets it walks, must agree. + REQUIRE(broken([] (nlohmann::json & j) { j["regions"][0]["reread"] = ""; })); + REQUIRE(broken([] (nlohmann::json & j) { j["regions"][0]["hashes"] = nlohmann::json::array(); })); + REQUIRE(broken([] (nlohmann::json & j) { j["regions"][0]["lines"]["starts"] = {3, 2, 21}; })); + REQUIRE(broken([] (nlohmann::json & j) { j["regions"][0]["lines"]["ends"] = {10, 19, 40}; })); // past the content + REQUIRE(broken([] (nlohmann::json & j) { j["regions"][0]["layout"]["cuts"] = {21, 12}; })); + REQUIRE(broken([] (nlohmann::json & j) { j["regions"][0]["layout"]["cut_line"] = {1, 3}; })); // no line 3 + REQUIRE(broken([] (nlohmann::json & j) { j["regions"][0]["layout"]["head_end"] = 11; })); + REQUIRE(broken([] (nlohmann::json & j) { j["regions"][0]["unit_te"] = {3}; })); // past the content's tokens + REQUIRE(broken([] (nlohmann::json & j) { j["regions"][0]["unit_db"] = {2}; })); + REQUIRE(broken([] (nlohmann::json & j) { j["raw_gen_begin"] = 1; })); // region after the generation +} + +TEST_CASE(PFlashUnifiedFixture, unified_view_stores_name_the_lineages_a_prompt_continues) { + namespace fs = std::filesystem; + ViewStore store; + store.remember(disk_test_view({1, 2, 3, 9}, 3, "c", 1)); + store.remember(disk_test_view({1, 2, 3, 4, 5, 9}, 5, "other", 2)); + store.remember(disk_test_view({7, 7, 9}, 2, "c", 3)); + auto lineages = store.lineages_continued_by({1, 2, 3, 4, 5, 6}); + std::sort(lineages.begin(), lineages.end()); + REQUIRE(lineages == std::vector({1, 2})); // any config: none of them may continue + REQUIRE(store.lineages_continued_by({1, 2, 4}).empty()); + + const fs::path dir = fs::temp_directory_path() / ("luce_test_view_lineages_" + std::to_string(getpid())); + fs::remove_all(dir); + ViewDiskStore disk; + REQUIRE(disk.lineages_continued_by({1, 2, 3, 4}).empty()); // not initialised + REQUIRE(disk.init(dir.string(), "id-a", 0, 8)); + REQUIRE(disk.save(disk_test_view({1, 2, 3, 9}, 3, "c", 1))); + REQUIRE(disk.save(disk_test_view({7, 7, 9}, 2, "c", 3))); + REQUIRE(disk.lineages_continued_by({1, 2, 3, 4}) == std::vector({1})); + REQUIRE(disk.lineages_continued_by({1, 2}).empty()); + fs::remove_all(dir); +} + +TEST_CASE(PFlashUnifiedFixture, unified_view_disk_store_applies_the_ttl_at_lookup) { + namespace fs = std::filesystem; + const fs::path dir = fs::temp_directory_path() / ("luce_test_view_ttl_" + std::to_string(getpid())); + fs::remove_all(dir); + uint64_t now = 1'000'000'000; + ViewDiskStore store; + store.set_clock([&] { return now; }); + REQUIRE(store.init(dir.string(), "id-a", 3600, 8)); + REQUIRE(store.save(disk_test_view({1, 2, 3, 9}, 3, "c", 1))); + View out; + now += 3600 * 1000; + REQUIRE(store.find({1, 2, 3, 7}, {1, 2, 3, 7}, "c", 0, out)); // at the TTL: still there + now += 1; + REQUIRE(!store.find({1, 2, 3, 7}, {1, 2, 3, 7}, "c", 0, out)); // past it: gone, file and all + REQUIRE(store.size() == 0); + REQUIRE(fs::is_empty(dir)); + fs::remove_all(dir); +} + +TEST_CASE(PFlashUnifiedFixture, unified_view_disk_store_restart_identity_cap_and_ttl) { + namespace fs = std::filesystem; + const fs::path dir = fs::temp_directory_path() / ("luce_test_view_disk_" + std::to_string(getpid())); + fs::remove_all(dir); + const auto view_files = [&] { + std::vector out; + for (const auto & e : fs::directory_iterator(dir)) { + if (e.path().extension() == ".view") out.push_back(e.path()); + } + return out; + }; + { + ViewDiskStore store; + REQUIRE(store.init(dir.string(), "id-a", 0, 3)); + REQUIRE(store.save(disk_test_view({1, 2, 3, 9}, 3, "c", 1))); + REQUIRE(store.save(disk_test_view({1, 2, 3, 4, 5, 9}, 5, "c", 1))); + REQUIRE(store.save(disk_test_view({1, 2, 3, 4, 5, 9}, 5, "c", 1))); // the same prefixes: replaced + REQUIRE(store.size() == 2 && view_files().size() == 2); + } + { + // A new process rebuilds the index from the headers and reads a view only when it matches. + ViewDiskStore store; + REQUIRE(store.init(dir.string(), "id-a", 0, 3)); + REQUIRE(store.size() == 2); + REQUIRE(store.max_lineage() == 1); + View out; + REQUIRE(store.find({1, 2, 3, 4, 5, 6, 7}, {1, 2, 3, 4, 5, 6, 7}, "c", 0, out)); + REQUIRE(view_to_json(out) == view_to_json(disk_test_view({1, 2, 3, 4, 5, 9}, 5, "c", 1))); + REQUIRE(store.find({1, 2, 3, 7}, {1, 2, 3, 7}, "c", 0, out)); // a fork continues the ancestor + REQUIRE(out.raw_gen_begin == 3); + REQUIRE(!store.find({1, 2, 3, 4, 5, 6}, {1, 2, 3, 4, 5, 6}, "c", 5, out)); // RAM holds as long a view + REQUIRE(!store.find({1, 2, 3, 4, 5, 6}, {1, 2, 3, 4, 5, 6}, "other", 0, out)); + REQUIRE(!store.find({1, 2, 3, 4, 5, 6}, {1, 2, 9, 4, 5, 6}, "c", 0, out)); // drafter ids must match too + // Over the cap the least recently written goes. + REQUIRE(store.save(disk_test_view({7, 7, 9}, 2, "c", 2))); + REQUIRE(store.save(disk_test_view({8, 8, 9}, 2, "c", 3))); + REQUIRE(store.size() == 3 && view_files().size() == 3); + REQUIRE(!store.find({1, 2, 3, 7}, {1, 2, 3, 7}, "c", 0, out)); + REQUIRE(store.find({1, 2, 3, 4, 5, 6, 7}, {1, 2, 3, 4, 5, 6, 7}, "c", 0, out)); + // A conversation the chat view took over leaves the disk too. + store.forget_lineage(1); + REQUIRE(store.size() == 2 && view_files().size() == 2); + REQUIRE(!store.find({1, 2, 3, 4, 5, 6, 7}, {1, 2, 3, 4, 5, 6, 7}, "c", 0, out)); + } + { + // Another identity (model, template, context) never reads these views. + ViewDiskStore other; + REQUIRE(other.init(dir.string(), "id-b", 0, 3)); + REQUIRE(other.size() == 0); + View out; + REQUIRE(!other.find({7, 7, 1}, {7, 7, 1}, "c", 0, out)); + REQUIRE(view_files().size() == 2); + } + { + // A view whose body does not load is deleted when read; the request then has no view. + for (const auto & file : view_files()) { + std::ifstream in(file); + std::string header; + std::getline(in, header); + in.close(); + if (header.find("\"lineage\":2") == std::string::npos) continue; + std::ofstream(file, std::ios::trunc) << header << "\n{\"format\":1}\n"; + } + ViewDiskStore store; + REQUIRE(store.init(dir.string(), "id-a", 0, 3)); + View out; + REQUIRE(!store.find({7, 7, 1}, {7, 7, 1}, "c", 0, out)); + REQUIRE(store.size() == 1 && view_files().size() == 1); + REQUIRE(store.find({8, 8, 1}, {8, 8, 1}, "c", 0, out)); + } + { + // At startup: an unfinished write is removed, and so is a view older than the TTL. + std::ofstream(dir / "x.view.tmp") << "partial"; + std::ofstream(dir / "old.view") << nlohmann::json({{"format", 1}, {"identity", "id-a"}, {"lineage", 9}, + {"config", "c"}, {"raw_len", 1}, {"raw_hash", 1}, + {"drafter_len", 1}, {"drafter_hash", 1}, + {"written", 1000}}).dump() << "\n{}\n"; + ViewDiskStore store; + REQUIRE(store.init(dir.string(), "id-a", 3600, 3)); + REQUIRE(store.size() == 1); + REQUIRE(!fs::exists(dir / "x.view.tmp") && !fs::exists(dir / "old.view")); + } + fs::remove_all(dir); +} + +TEST_CASE(PFlashUnifiedFixture, unified_view_store_reports_evicted_views) { + ViewStore store(2, (size_t) 1 << 30); + REQUIRE(store.remember(disk_test_view({1, 9}, 1, "c", 1)).empty()); + REQUIRE(store.remember(disk_test_view({2, 9}, 1, "c", 2)).empty()); + const auto evicted = store.remember(disk_test_view({3, 9}, 1, "c", 3)); + REQUIRE(evicted.size() == 1 && evicted[0].lineage == 1); + const auto all = store.all(); + REQUIRE(all.size() == 2 && all[0].lineage == 3 && all[1].lineage == 2); // most recently used first +} + +TEST_CASE(PFlashUnifiedFixture, unified_text_helpers) { + REQUIRE(json_escape("a\"b\\c\nd\te\x01 \xC3\xA9") == "a\\\"b\\\\c\\nd\\te\\u0001 \xC3\xA9"); + REQUIRE(sha1_hex("abc") == "a9993e364706816aba3e25717850c26c9cd0d89d"); + REQUIRE(preamble_text(RegionKind::ToolOutput, 3, 10) == + "[Server note: this tool result was shortened: 3 of 10 lines are shown; each omitted range is marked " + "inline. Call the tool again if you need an omitted part.]"); +} diff --git a/server/test/unit/test_server_unit.cpp b/server/test/unit/test_server_unit.cpp index 7852484ee..ead7521d9 100644 --- a/server/test/unit/test_server_unit.cpp +++ b/server/test/unit/test_server_unit.cpp @@ -288,6 +288,9 @@ struct HttpServerTestAccess { static size_t pflash_view_count(const HttpServer & server) { return server.pflash_views_.size(); } + static json pflash_unified(const HttpServer & server) { + return server.pflash_unified_json(); + } // A committed inline prefix-cache entry: ids[0, cut) restorable from // ``slot`` (the backend reports the snapshot). static void seed_prefix_cache(HttpServer & server, int slot, int cut, @@ -17872,3 +17875,1613 @@ TEST_CASE(ServerUnitFixture, test_pflash_raw_base_target_fails_closed) { tokenizer, markers, ids, tokenizer, ids, parsed.turns, 99) == -1); unlink(path.c_str()); } + +// ── PFLASH_UNIFIED=1 ─────────────────────────────────────────────────── + +// Scores every candidate (a line, a paragraph ending in a blank line, or ``chunk`` tokens) with +// ``score(segment text, prompt text)`` and returns the candidate scores the unified path admits units by; the +// selection itself is ignored there. +// ``Base`` MockBackend, or DiskPayloadBackend for a server with the disk session cache. +template +struct MockUnifiedBackendT : Base { + const Tokenizer * tokenizer = nullptr; + int chunk = 0; + bool paragraphs = false; + bool widen = false; + bool keep_scored_only = false; // kept_spans = the candidates scored above 1 (the selection keeps only them) + std::string probe_pin_text; // non-empty: the request probe pins its last occurrence in the input + bool fail = false; // the drafter pass fails + std::function score; + int compress_calls = 0; + ModelBackend::CompressRequest last_request; + + ModelBackend::CompressResult compress(const ModelBackend::CompressRequest & request) override { + ++compress_calls; + last_request = request; + const auto & ids = request.input_ids; + const int n = (int) ids.size(); + std::vector pieces((size_t) n); + std::string whole; + for (int i = 0; i < n; ++i) { + pieces[(size_t) i] = tokenizer->decode(std::vector{ids[(size_t) i]}); + whole += pieces[(size_t) i]; + } + ModelBackend::CompressResult result; + if (fail) return result; + if (!probe_pin_text.empty()) { + const size_t at = whole.rfind(probe_pin_text); + if (at != std::string::npos) { + int b = -1, e = -1; + size_t offset = 0; + for (int i = 0; i < n; ++i) { + if (b < 0 && offset + pieces[(size_t) i].size() > at) b = i; + offset += pieces[(size_t) i].size(); + if (e < 0 && offset >= at + probe_pin_text.size()) e = i + 1; + } + result.probe_pins.push_back({b, e}); + } + } + int begin = 0; + std::string segment; + for (int i = 0; i < n; ++i) { + segment += pieces[(size_t) i]; + const bool cut = chunk > 0 ? i + 1 - begin >= chunk + : paragraphs ? segment.size() >= 2 && segment.compare(segment.size() - 2, 2, "\n\n") == 0 + : pieces[(size_t) i].find('\n') != std::string::npos; + if (!cut && i + 1 < n) continue; + result.candidate_lifts.push_back({{begin, i + 1}, score ? score(segment, whole) : 1.0}); + segment.clear(); + begin = i + 1; + } + result.compressed_ids = ids; + result.kept_spans = {{0, n}}; + if (keep_scored_only) { + result.kept_spans.clear(); + for (const auto & lift : result.candidate_lifts) { + if (lift.second > 1.0) result.kept_spans.push_back(lift.first); + } + } + result.budget_widened = widen; + result.ok = true; + return result; + } +}; +using MockUnifiedBackend = MockUnifiedBackendT; + +struct UnifiedRun { + std::vector prepared; + std::vector> ids; + std::vector served_text; + std::vector raw_text; + std::vector calls_after; + std::vector view_records; + json unified; + std::vector> target_starts; // per turn, byte offsets where a raw target token starts + std::vector> drafter_starts; +}; + +using UnifiedScore = std::function; + +struct UnifiedOptions { + std::vector max_outputs; // per turn; 16 when absent + const char * select_mode = "budget_only"; // nullptr: no strict-selection environment + std::vector target_words; // whole pre-tokeniser pieces the target has as one token + bool distinct_drafter = false; // a drafter tokenizer of its own, with ``drafter_words`` + std::vector drafter_words; + bool paragraphs = false; // drafter candidates end at blank lines + bool widen = false; // the drafter pass reports a stage-2 widened budget + bool allow_errors = false; // a turn may be refused (context overflow) + const char * recall_gate = "off"; // PFLASH_RECALL_GATE (nullptr: unset, the 0.075 default) + const char * trace_units = nullptr; // PFLASH_RECALL_TRACE_UNITS + bool keep_scored_only = false; // the drafter selection keeps only candidates scored above 1 + bool unified = true; // false: PFLASH_UNIFIED unset (the chat view serves every turn) + std::string disk_dir; // non-empty: the disk session cache in this directory + bool disk_seed_history = false; // first, a server restart: turn 0 up to its newest user message on disk + DiskPrefixCacheMode disk_mode = DiskPrefixCacheMode::Full; // the disk scope of the requests + int restart_at = -1; // a server restart (shutdown flush, a new process) before this turn + std::string probe_pin_text; // the request probe pins this text's last occurrence + bool drafter_fails = false; // every drafter pass fails + const char * scorer = nullptr; // PFLASH_SELECT_SCORER + const char * fail_closed = nullptr; // PFLASH_FAIL_CLOSED + std::vector> required; // per turn, pflash_required + std::function before_turn; // runs before each turn (after a restart) +}; + +// One server with PFLASH_UNIFIED=1 (unset when ``options.unified`` is false), the turns in order. By default one fixture tokenizer is target and drafter, so +// every token boundary is shared; ``distinct_drafter`` gives the drafter its own merges. ``region_tools`` nullptr +// lists no tool (PFLASH_REGION_TOOLS set empty; unset would be the "*" default). +static UnifiedRun run_unified_session(const std::vector & turns, const json & tools, const char * region_tools, + int threshold, const UnifiedScore & score, int chunk = 0, + float keep_ratio = 0.3f, const char * recall = nullptr, + int max_ctx = 65536, const UnifiedOptions & options = UnifiedOptions()) { + luce_test::ScopedEnvVar unified{"PFLASH_UNIFIED", options.unified ? "1" : nullptr}; + luce_test::ScopedEnvVar region{"PFLASH_REGION_TOOLS", region_tools ? region_tools : ""}; + luce_test::ScopedEnvVar recall_env{"PFLASH_RECALL", recall}; + luce_test::ScopedEnvVar recall_gate{"PFLASH_RECALL_GATE", options.recall_gate}; + luce_test::ScopedEnvVar trace_units{"PFLASH_RECALL_TRACE_UNITS", options.trace_units}; + luce_test::ScopedEnvVar mode{"PFLASH_SELECT_MODE", options.select_mode}; + luce_test::ScopedEnvVar admission{"PFLASH_ADMISSION", "cache"}; + luce_test::ScopedEnvVar scorer{"PFLASH_SELECT_SCORER", options.scorer}; + luce_test::ScopedEnvVar fail_closed{"PFLASH_FAIL_CLOSED", options.fail_closed}; + char trace_path[] = "/tmp/pflash_unified_view_XXXXXX"; + const int trace_fd = mkstemp(trace_path); + TEST_ASSERT(trace_fd >= 0); + close(trace_fd); + luce_test::ScopedEnvVar view_trace{"PFLASH_VIEW_TRACE_PATH", trace_path}; + std::vector rendered; + std::string cover; + for (const auto & messages : turns) { + ToolMemory tool_memory; + const auto chat = normalize_chat_messages(messages, ApiFormat::OPENAI_CHAT, tool_memory); + rendered.push_back(render_chat_template( + chat, ChatFormat::QWEN3, /*add_generation_prompt=*/true, /*enable_thinking=*/true, + tools.is_array() && !tools.empty() ? tools.dump() : "")); + cover += rendered.back(); + } + cover += "[]\xE2\x80\xA6" "0123456789:;,.-|\\\"" "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ "; + const auto vocabulary = [] (const std::vector & words) { + std::vector out{"user", "assistant", "system", "\n", "tool"}; + out.insert(out.end(), words.begin(), words.end()); + return out; + }; + const std::string path = write_pflash_bpe_tokenizer_fixture(vocabulary(options.target_words), cover); + Tokenizer tokenizer; + TEST_ASSERT(tokenizer.load_from_gguf(path.c_str())); + std::string drafter_path; + Tokenizer drafter_tokenizer; + if (options.distinct_drafter) { + drafter_path = write_pflash_bpe_tokenizer_fixture(vocabulary(options.drafter_words), cover); + TEST_ASSERT(drafter_tokenizer.load_from_gguf(drafter_path.c_str())); + } + Tokenizer & drafter = options.distinct_drafter ? drafter_tokenizer : tokenizer; + UnifiedRun run; + const auto starts = [] (const Tokenizer & tok, const std::vector & ids) { + std::vector out; + size_t at = 0; + for (int32_t id : ids) { + out.push_back(at); + at += tok.decode(std::vector{id}).size(); + } + return out; + }; + for (const auto & text : rendered) { + run.ids.push_back(tokenizer.encode(text)); + run.raw_text.push_back(text); + run.target_starts.push_back(starts(tokenizer, run.ids.back())); + run.drafter_starts.push_back(starts(drafter, drafter.encode(text))); + } + std::unique_ptr backend_owner; + std::function compress_calls; + const auto setup = [&] (auto backend) { + backend->tokenizer = &drafter; + backend->chunk = chunk; + backend->paragraphs = options.paragraphs; + backend->widen = options.widen; + backend->keep_scored_only = options.keep_scored_only; + backend->probe_pin_text = options.probe_pin_text; + backend->fail = options.drafter_fails; + backend->score = score; + compress_calls = [mock = backend.get()] { return mock->compress_calls; }; + backend_owner = std::move(backend); + }; + if (options.disk_dir.empty()) { + setup(std::make_unique()); + } else { + setup(std::make_unique>()); + } + LuceEngine engine(std::move(backend_owner)); + ServerConfig config; + config.pflash_mode = ServerConfig::PflashMode::AUTO; + config.pflash_threshold = threshold; + config.pflash_keep_ratio = keep_ratio; + config.max_ctx = max_ctx; + config.prefix_cache_cap = 0; + config.prefill_cache_cap = 0; + if (!options.disk_dir.empty()) { + config.prefix_cache_cap = 8; + config.prefix_cache_max_bytes = 0; + config.disk_cache_dir = options.disk_dir; + config.disk_cache_min_tokens = 1; + } + if (options.disk_seed_history) { + // The previous process: its session (turn 0 before the newest user message) leaves RAM at shutdown. + const size_t at = run.raw_text[0].rfind("<|im_start|>user"); + const auto & starts0 = run.target_starts[0]; + const auto it = std::find(starts0.begin(), starts0.end(), at); + TEST_ASSERT(at != std::string::npos && it != starts0.end()); + const std::vector history(run.ids[0].begin(), run.ids[0].begin() + (it - starts0.begin())); + auto seed_owner = std::make_unique(); + DiskPayloadBackend & seed_backend = *seed_owner; + LuceEngine seed_engine(std::move(seed_owner)); + HttpServer seed(seed_engine, tokenizer, config); + seed_backend.put(0, (int) history.size()); + SchedulerTestHarness::prefix_cache(seed).confirm_inline_snap(0, (int) history.size(), history); + SchedulerTestHarness::set_slot_tokens(seed, 0, history); + SchedulerTestHarness::flush_sessions(seed); + TEST_ASSERT(SchedulerTestHarness::disk_cache(seed).longest_prefix(history).len == (int) history.size()); + } + config.disk_cache_policy.mode = options.disk_mode; + { + auto start = [&] { + auto server = std::make_unique(engine, tokenizer, config); + server->set_drafter_tokenizer(&drafter); + return server; + }; + auto owned = start(); + for (size_t index = 0; index < turns.size(); ++index) { + if ((int) index == options.restart_at) { + SchedulerTestHarness::flush_sessions(*owned); + owned.reset(); + owned = start(); + } + if (options.before_turn) options.before_turn(index); + HttpServer & server = *owned; + ParsedRequest request; + request.format = ApiFormat::OPENAI_CHAT; + request.messages = turns[index]; + if (tools.is_array() && !tools.empty()) request.tools = tools; + request.prompt_tokens = run.ids[index]; + request.max_output = index < options.max_outputs.size() ? options.max_outputs[index] : 16; + if (index < options.required.size()) request.pflash_required = options.required[index]; + request.disk_cache_policy = config.disk_cache_policy; // as the request parser sets it + run.prepared.push_back(HttpServerTestAccess::prepare_prompt(server, request)); + if (!options.allow_errors) TEST_ASSERT_MSG(run.prepared.back().error.empty(), run.prepared.back().error); + run.served_text.push_back(tokenizer.decode(run.prepared.back().tokens)); + run.calls_after.push_back(compress_calls()); + } + run.unified = HttpServerTestAccess::pflash_unified(*owned); + } + std::ifstream in(trace_path); + for (std::string line; std::getline(in, line);) { + if (!line.empty()) run.view_records.push_back(json::parse(line)); + } + unlink(trace_path); + unlink(path.c_str()); + if (!drafter_path.empty()) unlink(drafter_path.c_str()); + return run; +} + +// "Line i: plain filler text here[ mark]\n" for i in 1..lines. +static std::string unified_doc(int lines, const std::map & marks, + const std::string & prefix = "Line ") { + std::string out; + for (int i = 1; i <= lines; ++i) { + out += prefix + std::to_string(i) + ": plain filler text here"; + const auto it = marks.find(i); + if (it != marks.end()) out += " " + it->second; + out += "\n"; + } + return out; +} + +static size_t unified_occurrences(const std::string & text, const std::string & needle) { + size_t n = 0; + for (size_t at = text.find(needle); at != std::string::npos; at = text.find(needle, at + needle.size())) ++n; + return n; +} + +// Wants the kiwi line when the prompt asks KIWI?, the mango line when it asks MANGO?. +static double unified_fruit_score(const std::string & segment, const std::string & prompt) { + double score = 1.0; + if (prompt.find("KIWI?") != std::string::npos && segment.find("kiwi") != std::string::npos) score += 50.0; + if (prompt.find("MANGO?") != std::string::npos && segment.find("mango") != std::string::npos) score += 50.0; + return score; +} + +static json unified_tool_call(const std::string & id, const std::string & name, const json & arguments) { + return {{"id", id}, {"type", "function"}, {"function", {{"name", name}, {"arguments", arguments.dump()}}}}; +} + +TEST_CASE(ServerUnitFixture, test_unified_paste_pages_then_recall_appends_and_repeats) { + const std::string doc = unified_doc(80, {{7, "kiwi"}, {60, "mango"}}); + const json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc + "Which line mentions KIWI?"}}, + }); + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", "Line 7."}}); + turn2.push_back({{"role", "user"}, {"content", "Which line mentions MANGO?"}}); + const auto run = run_unified_session({turn1, turn2, turn2}, nullptr, nullptr, 64, unified_fruit_score); + + // Turn 1: the paste is paged; the system prompt, the pinned edges, the question and the kiwi line stay whole. + const auto & p1 = run.prepared[0]; + TEST_ASSERT(p1.compressed); + TEST_ASSERT_MSG(p1.pflash_stats["unified"].value("decision", "") == "region", p1.pflash_stats.dump()); + const std::string & s1 = run.served_text[0]; + TEST_ASSERT(s1.find("You are helpful.") != std::string::npos); + TEST_ASSERT(s1.find("Line 1: plain filler text here\nLine 2: plain filler text here\n" + "[Server note: this message was shortened:") != std::string::npos); + TEST_ASSERT(s1.find("Line 7: plain filler text here kiwi\n") != std::string::npos); + TEST_ASSERT(s1.find("mango") == std::string::npos); + TEST_ASSERT(s1.find("lines not shown \xE2\x80\xA6]\n") != std::string::npos); + TEST_ASSERT(s1.find("Line 80: plain filler text here\nWhich line mentions KIWI?") != std::string::npos); + TEST_ASSERT(run.prepared[0].tokens.size() < run.ids[0].size()); + + // Turn 2: turn 1's served prompt is reused token for token; the mango line comes back in a labelled block at + // the start of the question; nothing turn 1 kept is appended again. + const auto & p2 = run.prepared[1]; + const int vg1 = p1.snapshot_cut; + TEST_ASSERT(vg1 > 0); + TEST_ASSERT(std::equal(p1.tokens.begin(), p1.tokens.begin() + vg1, p2.tokens.begin())); + const std::string & s2 = run.served_text[1]; + const size_t answer = s2.find("Line 7.<|im_end|>"); + const size_t block = s2.find("[Earlier in this conversation]\n[From an earlier user message]\n"); + const size_t mango = s2.find("Line 60: plain filler text here mango\n"); + const size_t close = s2.find("mango\n\n[End of earlier excerpts]\n\nWhich line mentions MANGO?"); + TEST_ASSERT(answer != std::string::npos); + TEST_ASSERT_MSG(block != std::string::npos && block > answer, s2.substr(answer)); + TEST_ASSERT_MSG(mango != std::string::npos && mango > block && close != std::string::npos && close > mango, + s2.substr(block)); + TEST_ASSERT(s2.find("here\n\n[From an earlier user message]\nLine 60:", block) != std::string::npos); + for (int line = 1; line <= 80; ++line) { + TEST_ASSERT_MSG(unified_occurrences(s2, "Line " + std::to_string(line) + ":") <= 1, std::to_string(line)); + } + TEST_ASSERT(p2.pflash_stats["unified"].value("decision", "") == "recall"); + + // Turn 3, a retry: the same bytes, no drafter pass. + TEST_ASSERT(run.prepared[2].tokens == p2.tokens); + TEST_ASSERT(run.prepared[2].pflash_stats["unified"].value("decision", "") == "repeat"); + TEST_ASSERT(run.calls_after == std::vector({1, 2, 2})); + TEST_ASSERT(run.unified["counts"].value("serve.region", 0) == 1); + TEST_ASSERT(run.unified["counts"].value("recall.fired", 0) == 1); + TEST_ASSERT(run.view_records.size() == 3); + for (const auto & record : run.view_records) TEST_ASSERT(record.value("schema_version", 0) == 3); +} + +TEST_CASE(ServerUnitFixture, test_unified_paging_keeps_the_request_probes_pins) { + // The question sits mid-paste and scores low; only the request probe's pin keeps it. + const std::string question = "Line 40: plain filler text here QUESTION-X"; + const std::string doc = unified_doc(80, {{7, "kiwi"}, {40, "QUESTION-X"}}); + const json turn = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc + "Which line mentions KIWI?"}}, + }); + const UnifiedScore score = [] (const std::string & segment, const std::string & prompt) { + return segment.find("QUESTION-X") != std::string::npos ? 0.1 : unified_fruit_score(segment, prompt); + }; + const auto unpinned = run_unified_session({turn}, nullptr, nullptr, 64, score); + TEST_ASSERT(unpinned.prepared[0].compressed); + TEST_ASSERT(unpinned.served_text[0].find(question) == std::string::npos); + UnifiedOptions options; + options.probe_pin_text = "QUESTION-X"; + const auto pinned = run_unified_session({turn}, nullptr, nullptr, 64, score, 0, 0.3f, nullptr, 65536, options); + TEST_ASSERT(pinned.prepared[0].compressed); + TEST_ASSERT_MSG(pinned.served_text[0].find(question + "\n") != std::string::npos, pinned.served_text[0]); + TEST_ASSERT(pinned.served_text[0].find("Line 7: plain filler text here kiwi\n") != std::string::npos); +} + +TEST_CASE(ServerUnitFixture, test_unified_raw_fallback_drops_the_conversations_views) { + const std::string doc = unified_doc(80, {{7, "kiwi"}, {60, "mango"}}); + const json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc + "Which line mentions KIWI?"}}, + }); + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", "Line 7."}}); + turn2.push_back({{"role", "user"}, {"content", "Which line mentions MANGO?"}}); + json turn3 = turn2; + turn3.push_back({{"role", "assistant"}, {"content", "Line 60."}}); + turn3.push_back({{"role", "user"}, {"content", "Which line mentions KIWI?"}}); + // Turn 2 fails its PFlash mapping (a pflash_required text it does not hold) and is served raw. + UnifiedOptions options; + options.required = {{}, {"text that is not in the message"}, {}}; + const auto run = run_unified_session({turn1, turn2, turn3}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, + nullptr, 65536, options); + TEST_ASSERT(run.prepared[0].compressed); + TEST_ASSERT(!run.prepared[1].compressed && run.prepared[1].tokens == run.ids[1]); + const json & fallback = run.prepared[1].pflash_stats["pflash_fallback"]; + TEST_ASSERT_MSG(fallback.value("unified_lineages_forgotten", 0) == 1, run.prepared[1].pflash_stats.dump()); + // Turn 3 does not continue turn 1's view (its history was served raw): the chat view takes it. + TEST_ASSERT_MSG(run.prepared[2].unified_handback == "uncached", run.prepared[2].pflash_stats.dump()); + TEST_ASSERT(run.unified["counts"].value("serve.region", 0) == 1); +} + +TEST_CASE(ServerUnitFixture, test_unified_needs_candidate_scores) { + using luce::pflash::PFlashScorer; + using luce::pflash::PFlashSelectStrategy; + luce::pflash::PFlashSelectionConfig selection; + TEST_ASSERT(http_detail::pflash_selection_gives_lifts(selection)); + selection.scorer = PFlashScorer::Legacy; + TEST_ASSERT(!http_detail::pflash_selection_gives_lifts(selection)); + selection.scorer = PFlashScorer::Split; + TEST_ASSERT(!http_detail::pflash_selection_gives_lifts(selection)); + selection.scorer = PFlashScorer::Head; + selection.strategy = PFlashSelectStrategy::Fold; + TEST_ASSERT(!http_detail::pflash_selection_gives_lifts(selection)); + selection.strategy = PFlashSelectStrategy::Segments; + { + luce_test::ScopedEnvVar legacy{"PFLASH_QWEN35_LEGACY_SCORER", "1"}; + TEST_ASSERT(!http_detail::pflash_selection_gives_lifts(selection)); + } + + const std::string doc = unified_doc(80, {{7, "kiwi"}}); + const json turn = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc + "Which line mentions KIWI?"}}, + }); + // A scorer that gives no candidate scores: the chat view serves the request. + UnifiedOptions legacy; + legacy.scorer = "legacy"; + const auto handed = run_unified_session({turn}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, nullptr, + 65536, legacy); + TEST_ASSERT_MSG(handed.prepared[0].unified_handback == "unscored_selection", + handed.prepared[0].pflash_stats.dump()); + // A failed drafter pass: the newest content is served verbatim, or with PFLASH_FAIL_CLOSED=1 refused. + UnifiedOptions failing; + failing.drafter_fails = true; + const auto verbatim = run_unified_session({turn}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, nullptr, + 65536, failing); + TEST_ASSERT(verbatim.served_text[0] == verbatim.raw_text[0]); + TEST_ASSERT(verbatim.unified["counts"].value("drafter.failed", 0) == 1); + failing.fail_closed = "1"; + failing.allow_errors = true; + const auto refused = run_unified_session({turn}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, nullptr, + 65536, failing); + TEST_ASSERT_MSG(!refused.prepared[0].error.empty(), refused.prepared[0].pflash_stats.dump()); +} + +TEST_CASE(ServerUnitFixture, test_unified_repeat_with_a_new_pin_serves_it) { + // A retry of the same messages that now pins (pflash_required) a line the view left out is not answered with + // the stored view: it is paged again with that line kept. + const std::string pear = "Line 30: plain filler text here pear"; + const json turn = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", unified_doc(80, {{7, "kiwi"}, {30, "pear"}}) + "Which line mentions KIWI?"}}, + }); + const UnifiedScore score = [] (const std::string & segment, const std::string & prompt) { + return segment.find("pear") != std::string::npos ? 0.1 : unified_fruit_score(segment, prompt); + }; + UnifiedOptions options; + options.required = {{}, {pear}}; + const auto run = run_unified_session({turn, turn}, nullptr, nullptr, 64, score, 0, 0.3f, nullptr, 65536, options); + TEST_ASSERT(run.prepared[0].compressed && run.served_text[0].find(pear) == std::string::npos); + TEST_ASSERT_MSG(run.served_text[1].find(pear + "\n") != std::string::npos, run.prepared[1].pflash_stats.dump()); + TEST_ASSERT(run.unified["counts"].value("repeat.pins_unserved", 0) == 1); + TEST_ASSERT(run.unified["counts"].value("repeat", 0) == 0); +} + +TEST_CASE(ServerUnitFixture, test_unified_disk_view_that_does_not_match_the_prompt_is_dropped) { + // The restarted server finds turn 1's view on disk, but the file's region digest no longer matches the + // conversation's bytes: the view is deleted and the follow-up goes to the chat view as a cold history. + const json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", unified_doc(80, {{7, "kiwi"}, {60, "mango"}}) + "Which line mentions KIWI?"}}, + }); + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", "Line 7."}}); + turn2.push_back({{"role", "user"}, {"content", "Which line mentions MANGO?"}}); + const std::string dir = test_tmp_path("luce_test_unified_view_mismatch").string(); + rm_rf(dir); + UnifiedOptions disk; + disk.disk_dir = dir; + disk.restart_at = 1; + int edited = 0; + disk.before_turn = [&] (size_t index) { + if (index != 1) return; + for (const auto & entry : std::filesystem::directory_iterator(dir + "/.pflash-views")) { + if (entry.path().extension() != ".view") continue; + std::ifstream in(entry.path()); + std::string header, body; + std::getline(in, header); + std::getline(in, body); + in.close(); + json view = json::parse(body); + view["regions"][0]["content_sha"] = "0000"; + std::ofstream(entry.path(), std::ios::trunc) << header << "\n" << view.dump() << "\n"; + ++edited; + } + }; + const auto run = run_unified_session({turn1, turn2}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, nullptr, + 65536, disk); + TEST_ASSERT(edited == 1); + TEST_ASSERT(run.prepared[0].compressed && run.prepared[0].unified_view); + TEST_ASSERT_MSG(run.prepared[1].unified_handback == "uncached", run.prepared[1].pflash_stats.dump()); + TEST_ASSERT(run.unified["counts"].value("disk_view_rejected", 0) == 1); + TEST_ASSERT(run.unified["counts"].value("disk_view", 0) == 0); + rm_rf(dir); +} + +TEST_CASE(ServerUnitFixture, test_unified_tool_output_needs_listing) { + const json tools = json::array({ + {{"type", "function"}, {"function", {{"name", "read_file"}, {"description", "Read a file"}}}}, + {{"type", "function"}, {"function", {{"name", "terminal"}, {"description", "Run a command"}}}}, + }); + const json turn = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", "Read both files. Which line mentions KIWI?"}}, + {{"role", "assistant"}, {"content", ""}, {"tool_calls", json::array({ + unified_tool_call("call_1", "read_file", {{"path", "docs/a.md"}}), + unified_tool_call("call_2", "terminal", {{"cmd", "cat docs/b.md"}}), + unified_tool_call("call_3", "read_file", {{"path", "docs/c.md"}}), + })}}, + {{"role", "tool"}, {"tool_call_id", "call_1"}, {"content", unified_doc(80, {{9, "kiwi"}}, "A")}}, + {{"role", "tool"}, {"tool_call_id", "call_2"}, {"content", unified_doc(80, {}, "B")}}, + {{"role", "tool"}, {"tool_call_id", "call_3"}, {"content", unified_doc(1, {}, "C")}}, + }); + // The fixture tokenizer has one token per character, so the history before the tool results (the question and + // three calls, ~350 tokens) would already be a 64-token threshold's worth of uncached history and the request + // would go to the chat view; 600 keeps it a fresh agent turn whose 80-line outputs are still large. + const int threshold = 600; + const auto listed = run_unified_session({turn}, tools, "read_file", threshold, unified_fruit_score); + const json & admission = listed.prepared[0].pflash_stats["unified"]["admission"]; + TEST_ASSERT_MSG(admission.size() == 3, admission.dump()); + TEST_ASSERT(admission[0].value("decision", "") == "region"); + TEST_ASSERT(admission[1].value("decision", "") == "unlisted"); + TEST_ASSERT(admission[2].value("decision", "") == "small"); + const std::string & served = listed.served_text[0]; + TEST_ASSERT(served.find("A9: plain filler text here kiwi\n") != std::string::npos); + TEST_ASSERT(served.find("[Server note: this tool result was shortened:") != std::string::npos); + for (int line = 1; line <= 80; ++line) { + TEST_ASSERT(served.find("B" + std::to_string(line) + ": plain filler text here\n") != std::string::npos); + } + // "*": every tool output may be paged; the one-line one is still too small. + const auto all = run_unified_session({turn}, tools, "*", threshold, unified_fruit_score); + const json & all_admission = all.prepared[0].pflash_stats["unified"]["admission"]; + TEST_ASSERT(all_admission[0].value("decision", "") == "region"); + TEST_ASSERT(all_admission[1].value("decision", "") == "region"); + TEST_ASSERT(all_admission[2].value("decision", "") == "small"); + // Nothing listed: the tool turn is served raw, without a drafter pass. + const auto none = run_unified_session({turn}, tools, nullptr, threshold, unified_fruit_score); + TEST_ASSERT(!none.prepared[0].compressed); + TEST_ASSERT(none.prepared[0].tokens == none.ids[0]); + TEST_ASSERT(none.calls_after[0] == 0); + TEST_ASSERT(none.prepared[0].pflash_stats["unified"].value("reason", "") == "no_region"); +} + +TEST_CASE(ServerUnitFixture, test_unified_line_model_json_string_served) { + // A JSON-wrapped read (the shape some harnesses return): numbered lines inside a string field. + std::string body; + for (int i = 1; i <= 80; ++i) { + body += std::to_string(i) + "|row " + std::to_string(i) + (i == 33 ? " kiwi" : "") + "\n"; + } + const std::string wrapped = json({{"content", body}, {"total_lines", 80}, {"truncated", false}}).dump(); + const json tools = json::array({{{"type", "function"}, {"function", {{"name", "read_file"}}}}}); + const json turn = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", "Which row mentions KIWI?"}}, + {{"role", "assistant"}, {"content", ""}, {"tool_calls", json::array({ + unified_tool_call("call_1", "read_file", {{"path", "rows.txt"}})})}}, + {{"role", "tool"}, {"tool_call_id", "call_1"}, {"content", wrapped}}, + }); + const auto run = run_unified_session({turn}, tools, "read_file", 64, unified_fruit_score, /*chunk=*/7); + const auto & prepared = run.prepared[0]; + TEST_ASSERT_MSG(prepared.compressed, prepared.pflash_stats.dump()); + const json & admission = prepared.pflash_stats["unified"]["admission"][0]; + TEST_ASSERT(admission.value("escaped_lines", false)); + const std::string & served = run.served_text[0]; + const size_t open = served.rfind("\n"); + const size_t close = served.find("\n", open); + TEST_ASSERT(open != std::string::npos && close != std::string::npos); + const json result = json::parse(served.substr(open + 16, close - open - 16), nullptr, false); + TEST_ASSERT_MSG(!result.is_discarded(), served.substr(open, 400)); + TEST_ASSERT(result.value("total_lines", 0) == 80); + TEST_ASSERT(result.value("truncated", true) == false); + const std::string content = result.value("content", ""); + TEST_ASSERT(content.find("1|row 1\n2|row 2\n[Server note: this tool result was shortened:") == 0); + TEST_ASSERT(content.find("33|row 33 kiwi\n") != std::string::npos); + TEST_ASSERT(content.find(" lines not shown \xE2\x80\xA6]\n") != std::string::npos); + TEST_ASSERT(content.find("\n79|row 79\n80|row 80\n") != std::string::npos); +} + +TEST_CASE(ServerUnitFixture, test_unified_recall_user_question_only) { + const json tools = json::array({{{"type", "function"}, {"function", {{"name", "read_file"}}}}}); + json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", "Which line mentions KIWI?"}}, + {{"role", "assistant"}, {"content", ""}, {"tool_calls", json::array({ + unified_tool_call("call_1", "read_file", {{"path", "doc.md"}})})}}, + {{"role", "tool"}, {"tool_call_id", "call_1"}, + {"content", unified_doc(80, {{7, "kiwi"}, {60, "mango"}})}}, + }); + // An agent step: another call and a small result. No recall, no drafter pass; the view continues. + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", ""}, {"tool_calls", json::array({ + unified_tool_call("call_2", "read_file", {{"path", "note.md"}})})}}); + turn2.push_back({{"role", "tool"}, {"tool_call_id", "call_2"}, {"content", "short note"}}); + // The answer, then a new user question: recall at the question. + json turn3 = turn2; + turn3.push_back({{"role", "assistant"}, {"content", "Line 7."}}); + turn3.push_back({{"role", "user"}, {"content", "Which line mentions MANGO?"}}); + const auto run = run_unified_session({turn1, turn2, turn3}, tools, "read_file", 64, unified_fruit_score); + TEST_ASSERT(run.calls_after == std::vector({1, 1, 2})); + TEST_ASSERT(run.prepared[1].pflash_stats["unified"].value("decision", "") == "continue"); + TEST_ASSERT(run.prepared[2].pflash_stats["unified"].value("decision", "") == "recall"); + for (size_t turn = 1; turn < 3; ++turn) { + const int vg = run.prepared[turn - 1].snapshot_cut; + TEST_ASSERT(std::equal(run.prepared[turn - 1].tokens.begin(), run.prepared[turn - 1].tokens.begin() + vg, + run.prepared[turn].tokens.begin())); + } + const std::string & s3 = run.served_text[2]; + const size_t block = s3.find("[Earlier in this conversation]\n[From the earlier result of read_file " + "{\"path\":\"doc.md\"}]\n"); + TEST_ASSERT_MSG(block != std::string::npos, s3); + TEST_ASSERT(s3.find("Line 60: plain filler text here mango\n", block) != std::string::npos); + TEST_ASSERT(s3.find("short note") < block); +} + +TEST_CASE(ServerUnitFixture, test_unified_cold_history_hands_back_to_the_chat_view) { + // A cold transcript: a large paste in turn 1 and a large listed tool output in turn 2, the newest turn a short + // question. The unified view never pages anything before the newest turn and would prefill that history raw, so + // it hands the request to the chat view, which serves a tool session raw (keep_history): no drafter pass. + const json tools = json::array({{{"type", "function"}, {"function", {{"name", "read_file"}}}}}); + const json turn = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", unified_doc(80, {{7, "kiwi"}}) + "Summarise this."}}, + {{"role", "assistant"}, {"content", ""}, {"tool_calls", json::array({ + unified_tool_call("call_1", "read_file", {{"path", "doc.md"}})})}}, + {{"role", "tool"}, {"tool_call_id", "call_1"}, {"content", unified_doc(80, {}, "T")}}, + {{"role", "assistant"}, {"content", "Done."}}, + {{"role", "user"}, {"content", "Which line mentions KIWI?"}}, + }); + const auto run = run_unified_session({turn}, tools, "read_file", 64, unified_fruit_score); + TEST_ASSERT_MSG(run.prepared[0].unified_handback == "uncached", run.prepared[0].pflash_stats.dump()); + TEST_ASSERT(run.prepared[0].pflash_stats.find("unified") == run.prepared[0].pflash_stats.end()); + TEST_ASSERT(!run.prepared[0].compressed); + TEST_ASSERT(run.prepared[0].tokens == run.ids[0]); + TEST_ASSERT(run.calls_after[0] == 0); + TEST_ASSERT(run.unified["counts"].value("handback.uncached", 0) == 1); + TEST_ASSERT_MSG(run.view_records.size() == 1 && + run.view_records[0]["pflash"]["unified"].value("decision", "") == "handback" && + run.view_records[0]["pflash"]["unified"].value("reason", "") == "uncached", + run.view_records.empty() ? "no view record" : run.view_records[0].dump()); +} + +TEST_CASE(ServerUnitFixture, test_unified_cold_chat_history_goes_to_the_chat_view) { + // The same without tools: a chat whose first request already holds a large earlier paste. The chat view takes + // it (its cache admission runs the drafter on the uncached history) and serves it exactly as without the flag; + // the follow-up stays with the chat view (the unified view holds no view of this conversation). + const json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", unified_doc(80, {{7, "kiwi"}, {60, "mango"}}) + "Summarise this."}}, + {{"role", "assistant"}, {"content", "A list of lines."}}, + {{"role", "user"}, {"content", "Which line mentions KIWI?"}}, + }); + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", "Line 7."}}); + turn2.push_back({{"role", "user"}, {"content", "Which line mentions MANGO?"}}); + const auto run = run_unified_session({turn1, turn2}, nullptr, nullptr, 64, unified_fruit_score); + UnifiedOptions off; + off.unified = false; + const auto base = run_unified_session({turn1, turn2}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, nullptr, + 65536, off); + for (size_t t = 0; t < 2; ++t) { + TEST_ASSERT_MSG(run.prepared[t].unified_handback == "uncached", run.prepared[t].pflash_stats.dump()); + TEST_ASSERT(run.prepared[t].pflash_stats.find("unified") == run.prepared[t].pflash_stats.end()); + TEST_ASSERT(run.prepared[t].tokens == base.prepared[t].tokens); + TEST_ASSERT(run.prepared[t].compressed == base.prepared[t].compressed); + } + TEST_ASSERT(run.calls_after == base.calls_after); + TEST_ASSERT(run.calls_after[0] == 1); + TEST_ASSERT(run.unified["counts"].value("handback.uncached", 0) == 2); + TEST_ASSERT(run.unified.value("views", 1) == 0); +} + +TEST_CASE(ServerUnitFixture, test_pflash_restorable_prefix_counts_disk_only_when_on) { + using http_detail::pflash_restorable_prefix; + const auto is = [](const http_detail::PflashRestorablePrefix & got, int tokens, const char * source) { + return got.tokens == tokens && std::string(got.source) == source; + }; + // Disk cache off: the RAM prefix, whatever the disk would hold. + TEST_ASSERT(is(pflash_restorable_prefix(0, false, 0), 0, "none")); + TEST_ASSERT(is(pflash_restorable_prefix(0, false, 40000), 0, "none")); + TEST_ASSERT(is(pflash_restorable_prefix(9000, false, 40000), 9000, "ram")); + // Disk cache on: the deeper of the two; ties stay in RAM (no file read). + TEST_ASSERT(is(pflash_restorable_prefix(0, true, 0), 0, "none")); + TEST_ASSERT(is(pflash_restorable_prefix(0, true, 40000), 40000, "disk")); + TEST_ASSERT(is(pflash_restorable_prefix(9000, true, 40000), 40000, "disk")); + TEST_ASSERT(is(pflash_restorable_prefix(40000, true, 40000), 40000, "ram")); + TEST_ASSERT(is(pflash_restorable_prefix(41000, true, 40000), 41000, "ram")); + TEST_ASSERT(is(pflash_restorable_prefix(-1, true, -5), 0, "none")); +} + +TEST_CASE(ServerUnitFixture, test_unified_restart_resumes_a_disk_chat_and_pages_only_the_new_paste) { + // A server restart: the chat's history (an earlier paste and its answer) is on disk, RAM is empty. The unified + // view counts the disk entry as cached, as the chat view's admission does: it keeps the conversation instead of + // handing a cold history to the chat view, serves the history as it is (the generation path restores it from + // disk) and pages only the new paste. Without the entry, or outside the Full disk scope, the chat view takes it. + const json turn = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", unified_doc(80, {{7, "mango"}}) + "Summarise this."}}, + {{"role", "assistant"}, {"content", "A list of lines."}}, + {{"role", "user"}, {"content", unified_doc(80, {{33, "kiwi"}}, "Row ") + "Which row mentions KIWI?"}}, + }); + const std::string dir = test_tmp_path("luce_test_unified_disk_resume").string(); + rm_rf(dir); + UnifiedOptions disk; + disk.disk_dir = dir; + disk.disk_seed_history = true; + const auto run = run_unified_session({turn}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, nullptr, 65536, + disk); + const auto & stats = run.prepared[0].pflash_stats; + TEST_ASSERT_MSG(run.prepared[0].unified_handback.empty(), stats.dump()); + TEST_ASSERT_MSG(stats.contains("unified") && stats["unified"].contains("cached") && + stats["unified"]["cached"].value("source", "") == "disk" && + stats["unified"]["cached"].value("tokens", 0) > 0, stats.dump()); + TEST_ASSERT(run.prepared[0].compressed); + TEST_ASSERT(run.calls_after[0] == 1); + TEST_ASSERT(run.served_text[0].find(unified_doc(80, {{7, "mango"}}) + "Summarise this.") != std::string::npos); + TEST_ASSERT(run.served_text[0].find("Row 33: plain filler text here kiwi") != std::string::npos); + TEST_ASSERT(run.served_text[0].size() < run.raw_text[0].size()); + TEST_ASSERT(run.unified["counts"].value("handback.uncached", 0) == 0); + + // The disk is looked up only in the Full scope, so only there does it count. (A fresh directory: the first run's + // shutdown wrote its view there, and a view on disk continues the conversation whatever the scope.) + rm_rf(dir); + UnifiedOptions off = disk; + off.disk_mode = DiskPrefixCacheMode::Off; + const auto scoped = run_unified_session({turn}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, nullptr, + 65536, off); + TEST_ASSERT_MSG(scoped.prepared[0].unified_handback == "uncached", scoped.prepared[0].pflash_stats.dump()); + rm_rf(dir); + // No disk entry: the cold history goes to the chat view, as without the disk cache. + UnifiedOptions empty; + empty.disk_dir = dir; + const auto cold = run_unified_session({turn}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, nullptr, + 65536, empty); + TEST_ASSERT_MSG(cold.prepared[0].unified_handback == "uncached", cold.prepared[0].pflash_stats.dump()); + rm_rf(dir); +} + +TEST_CASE(ServerUnitFixture, test_unified_restart_serves_a_disk_chat_raw_for_a_short_question) { + // The same restart with a short follow-up question: nothing new to page, so the whole prompt is served raw for + // the disk restore -- no drafter pass and no compression of the history. Without the disk cache the chat view + // compresses the cold history (test_unified_cold_chat_history_goes_to_the_chat_view). + const json turn = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", unified_doc(80, {{7, "kiwi"}, {60, "mango"}}) + "Summarise this."}}, + {{"role", "assistant"}, {"content", "A list of lines."}}, + {{"role", "user"}, {"content", "Which line mentions KIWI?"}}, + }); + const std::string dir = test_tmp_path("luce_test_unified_disk_short").string(); + rm_rf(dir); + UnifiedOptions disk; + disk.disk_dir = dir; + disk.disk_seed_history = true; + const auto run = run_unified_session({turn}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, nullptr, 65536, + disk); + TEST_ASSERT_MSG(run.prepared[0].unified_handback.empty(), run.prepared[0].pflash_stats.dump()); + TEST_ASSERT(!run.prepared[0].compressed); + TEST_ASSERT(run.prepared[0].tokens == run.ids[0]); + TEST_ASSERT(run.calls_after[0] == 0); + + // The chat view alone (PFLASH_UNIFIED unset) counts the disk the same way: its cache admission keeps the + // history raw instead of compressing it, and records where the cached prefix is. + UnifiedOptions chat = disk; + chat.unified = false; + const auto warm = run_unified_session({turn}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, nullptr, 65536, + chat); + TEST_ASSERT(warm.calls_after[0] == 0); + TEST_ASSERT(!warm.prepared[0].compressed); + TEST_ASSERT(warm.prepared[0].tokens == warm.ids[0]); + const json warm_admission = warm.prepared[0].pflash_stats.value("admission", json::object()); + TEST_ASSERT_MSG(warm_admission.value("cached_source", "") == "disk", warm.prepared[0].pflash_stats.dump()); + rm_rf(dir); + // Disk cache off: the admission compresses the cold history, and nothing records a source. + UnifiedOptions plain; + plain.unified = false; + const auto cold = run_unified_session({turn}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, nullptr, 65536, + plain); + TEST_ASSERT(cold.calls_after[0] == 1); + TEST_ASSERT_MSG(cold.prepared[0].pflash_stats.dump().find("cached_source") == std::string::npos, + cold.prepared[0].pflash_stats.dump()); +} + +TEST_CASE(ServerUnitFixture, test_unified_shortened_chat_survives_a_restart_through_its_view_on_disk) { + // Turn 1 pages a long paste. The server restarts (shutdown flush): its view is written to disk and read back on + // the next turn, so the follow-up continues the shortened conversation -- the same served prefix, nothing handed + // back to the chat view -- instead of arriving cold. Without the disk cache the restart loses the view, and the + // follow-up goes to the chat view as a cold history. + const json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", unified_doc(80, {{7, "kiwi"}, {60, "mango"}}) + "Which line mentions KIWI?"}}, + }); + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", "Line 7."}}); + turn2.push_back({{"role", "user"}, {"content", "Which line mentions MANGO?"}}); + const std::string dir = test_tmp_path("luce_test_unified_view_restart").string(); + rm_rf(dir); + UnifiedOptions disk; + disk.disk_dir = dir; + disk.restart_at = 1; + const auto run = run_unified_session({turn1, turn2}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, nullptr, + 65536, disk); + TEST_ASSERT_MSG(run.prepared[0].compressed && run.prepared[0].unified_view, run.prepared[0].pflash_stats.dump()); + const auto & stats = run.prepared[1].pflash_stats; + TEST_ASSERT_MSG(run.prepared[1].unified_handback.empty() && stats.contains("unified") && + stats["unified"].value("disk_view", false), stats.dump()); + TEST_ASSERT(run.prepared[1].compressed && run.prepared[1].unified_view); + TEST_ASSERT(run.unified["counts"].value("disk_view", 0) == 1); + // The follow-up's served prompt starts with what turn 1 served, up to its generation prompt. + const int cut = stats["unified"].value("reused_tokens", -1); + TEST_ASSERT_MSG(cut > 0 && cut == run.prepared[0].snapshot_cut && run.prepared[1].tokens.size() > (size_t) cut && + std::equal(run.prepared[0].tokens.begin(), run.prepared[0].tokens.begin() + cut, + run.prepared[1].tokens.begin()), stats.dump()); + TEST_ASSERT(run.served_text[1].find("Line 60: plain filler text here mango") != std::string::npos); + + // The same without the disk cache: the view is gone after the restart. + rm_rf(dir); + UnifiedOptions ram; + ram.restart_at = 1; + const auto lost = run_unified_session({turn1, turn2}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, + nullptr, 65536, ram); + TEST_ASSERT_MSG(lost.prepared[1].unified_handback == "uncached", lost.prepared[1].pflash_stats.dump()); +} + +TEST_CASE(ServerUnitFixture, test_unified_unsplittable_paste_goes_to_the_chat_view) { + // A large paste the view cannot cut into lines (one long line) would be prefilled raw: the chat view takes it. + std::string paste; + for (int i = 0; i < 300; ++i) paste += "word" + std::to_string(i % 10) + " "; + const json turn = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", paste + "\nWhat is this?"}}, + }); + const auto run = run_unified_session({turn}, nullptr, nullptr, 64, unified_fruit_score); + UnifiedOptions off; + off.unified = false; + const auto base = run_unified_session({turn}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, nullptr, 65536, + off); + TEST_ASSERT_MSG(run.prepared[0].unified_handback == "uncached", run.prepared[0].pflash_stats.dump()); + TEST_ASSERT(run.prepared[0].tokens == base.prepared[0].tokens); + TEST_ASSERT(run.calls_after == base.calls_after); + TEST_ASSERT(run.unified["counts"].value("handback.uncached", 0) == 1); +} + +TEST_CASE(ServerUnitFixture, test_unified_reread_never_paged_and_holds_units) { + const json tools = json::array({{{"type", "function"}, {"function", {{"name", "read_file"}}}}}); + const std::string doc = unified_doc(80, {{7, "kiwi"}, {60, "mango"}}); + json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", "Which line mentions KIWI?"}}, + {{"role", "assistant"}, {"content", ""}, {"tool_calls", json::array({ + unified_tool_call("call_1", "read_file", {{"path", "doc.md"}})})}}, + {{"role", "tool"}, {"tool_call_id", "call_1"}, {"content", doc}}, + }); + // The agent reads the whole file again: a re-read, served verbatim. + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", ""}, {"tool_calls", json::array({ + unified_tool_call("call_2", "read_file", {{"path", "doc.md"}, {"offset", 1}})})}}); + turn2.push_back({{"role", "tool"}, {"tool_call_id", "call_2"}, {"content", doc}}); + json turn3 = turn2; + turn3.push_back({{"role", "assistant"}, {"content", "Line 7."}}); + turn3.push_back({{"role", "user"}, {"content", "Which line mentions MANGO?"}}); + const auto run = run_unified_session({turn1, turn2, turn3}, tools, "read_file", 64, unified_fruit_score); + const json & admission = run.prepared[1].pflash_stats["unified"]["admission"]; + TEST_ASSERT_MSG(admission[0].value("decision", "") == "reread", admission.dump()); + TEST_ASSERT(run.prepared[1].pflash_stats["unified"].value("reread_held_units", 0) > 0); + TEST_ASSERT(run.calls_after[1] == 1); + // Everything was re-read: the question finds nothing to recall. + TEST_ASSERT(run.served_text[2].find("[Earlier in this conversation]") == std::string::npos); + TEST_ASSERT(run.unified["counts"].value("recall.empty", 0) == 1); +} + +TEST_CASE(ServerUnitFixture, test_unified_recall_coverage_bound_worst_case) { + // A scorer that ranks every unit: questions recall until the whole region is held, then nothing more. + const std::string doc = unified_doc(80, {}); + json turn = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc + "Question 0?"}}, + }); + std::vector turns{turn}; + for (int q = 1; q <= 8; ++q) { + turn.push_back({{"role", "assistant"}, {"content", "Ok."}}); + turn.push_back({{"role", "user"}, {"content", "Question " + std::to_string(q) + "?"}}); + turns.push_back(turn); + } + const auto run = run_unified_session(turns, nullptr, nullptr, 64, nullptr); + const std::string & last = run.served_text.back(); + for (int line = 1; line <= 80; ++line) { + TEST_ASSERT_MSG(unified_occurrences(last, "Line " + std::to_string(line) + ":") == 1, std::to_string(line)); + } + // Once everything is back, a question appends nothing: the view grows by the new turn's raw tokens only. + TEST_ASSERT(run.unified["counts"].value("recall.empty", 0) >= 1); + const size_t n = run.prepared.size(); + TEST_ASSERT(run.prepared[n - 1].pflash_stats["unified"].value("decision", "") == "continue"); + const int grown = (int) run.prepared[n - 1].tokens.size() - (int) run.prepared[n - 2].tokens.size(); + const int raw_grown = (int) run.ids[n - 1].size() - (int) run.ids[n - 2].size(); + TEST_ASSERT(grown == raw_grown); + // Cumulative appended by recall never exceeds what paging dropped plus labels and wrappers. + int recalled = 0; + for (size_t turn = 1; turn < n; ++turn) { + const json & recall = run.prepared[turn].pflash_stats["unified"]["recall"]; + if (recall.is_array()) for (const auto & region : recall) recalled = region.value("cumulative_tokens", 0); + } + const json & paged = run.prepared[0].pflash_stats["unified"]["admission"][0]; + TEST_ASSERT(recalled > 0); + TEST_ASSERT(recalled <= paged.value("region_tokens", 0) - paged.value("kept_tokens", 0) + 400); +} + +TEST_CASE(ServerUnitFixture, test_unified_recall_off_flag) { + const std::string doc = unified_doc(80, {{7, "kiwi"}, {60, "mango"}}); + const json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc + "Which line mentions KIWI?"}}, + }); + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", "Line 7."}}); + turn2.push_back({{"role", "user"}, {"content", "Which line mentions MANGO?"}}); + const auto run = run_unified_session({turn1, turn2}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, "0"); + TEST_ASSERT(run.calls_after == std::vector({1, 1})); + TEST_ASSERT(run.served_text[1].find("[Earlier in this conversation]") == std::string::npos); + TEST_ASSERT(run.prepared[1].pflash_stats["unified"].value("decision", "") == "continue"); +} + +// Like unified_fruit_score, with filler near zero: the head's mass sits on the line the question asks for. +static double unified_sharp_fruit_score(const std::string & segment, const std::string & prompt) { + double score = 0.01; + if (prompt.find("KIWI?") != std::string::npos && segment.find("kiwi") != std::string::npos) score += 50.0; + if (prompt.find("MANGO?") != std::string::npos && segment.find("mango") != std::string::npos) score += 50.0; + return score; +} + +static std::vector unified_fruit_turns(const char * second_question) { + const std::string doc = unified_doc(80, {{7, "kiwi"}, {60, "mango"}}); + const json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc + "Which line mentions KIWI?"}}, + }); + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", "Line 7."}}); + turn2.push_back({{"role", "user"}, {"content", second_question}}); + return {turn1, turn2}; +} + +TEST_CASE(ServerUnitFixture, test_unified_pages_a_document_sent_before_its_question) { + // The documents arrive as their own user message and the question as the next one: both are the newest turn, + // so the documents are paged exactly as when they share one message with the question. + const std::string doc = unified_doc(80, {{7, "kiwi"}, {60, "mango"}}); + const json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc}}, + {{"role", "user"}, {"content", "Which line mentions KIWI?"}}, + }); + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", "Line 7."}}); + turn2.push_back({{"role", "user"}, {"content", "Which line mentions MANGO?"}}); + const auto run = run_unified_session({turn1, turn2}, nullptr, nullptr, 64, unified_sharp_fruit_score); + const json & u0 = run.prepared[0].pflash_stats["unified"]; + TEST_ASSERT_MSG(u0.value("decision", "") == "region", u0.dump()); + const json & adm = u0["admission"]; + TEST_ASSERT_MSG(adm.size() == 2, adm.dump()); + TEST_ASSERT(adm[0].value("message", 0) == 1 && adm[0].value("decision", "") == "region"); + TEST_ASSERT(adm[1].value("message", 0) == 2 && adm[1].value("decision", "") != "region"); + TEST_ASSERT(run.prepared[0].tokens.size() < run.ids[0].size()); + // The question is served verbatim, after the paged documents. + const std::string & s0 = run.served_text[0]; + TEST_ASSERT(s0.find("Which line mentions KIWI?") != std::string::npos); + TEST_ASSERT(s0.find("Line 7: plain filler text here kiwi\n") != std::string::npos); + TEST_ASSERT(s0.find("Line 60: plain filler text here mango\n") == std::string::npos); + // The next question continues the view and recalls the dropped mango line; the served prefix is untouched (I1). + const json & u1 = run.prepared[1].pflash_stats["unified"]; + TEST_ASSERT_MSG(u1.value("decision", "") == "recall", u1.dump()); + TEST_ASSERT(run.served_text[1].find("Line 60: plain filler text here mango\n") != std::string::npos); + const int vg1 = run.prepared[0].snapshot_cut; + TEST_ASSERT(std::equal(run.prepared[0].tokens.begin(), run.prepared[0].tokens.begin() + vg1, + run.prepared[1].tokens.begin())); + // Same paging as the one-message layout (where the question line is one more unit of the region): the same + // document lines are kept. + const auto one = run_unified_session(unified_fruit_turns("Which line mentions MANGO?"), nullptr, nullptr, 64, + unified_sharp_fruit_score); + const json & k_split = adm[0]; + const json & k_one = one.prepared[0].pflash_stats["unified"]["admission"][0]; + TEST_ASSERT_MSG(k_split.value("units", -1) + 1 == k_one.value("units", -2) && + k_split.value("kept_units", -1) == k_one.value("kept_units", -2) && + k_split.value("kept_tokens", -1) == k_one.value("kept_tokens", -2), k_split.dump() + k_one.dump()); +} + +TEST_CASE(ServerUnitFixture, test_unified_recall_gate_takes_only_what_the_question_needs) { + UnifiedOptions gate; + gate.recall_gate = "0.05"; + const auto turns = unified_fruit_turns("Which line mentions MANGO?"); + const auto run = run_unified_session(turns, nullptr, nullptr, 64, unified_sharp_fruit_score, 0, 0.3f, nullptr, + 65536, gate); + const auto ungated = run_unified_session(turns, nullptr, nullptr, 64, unified_sharp_fruit_score); + // Turn 1 is paging only: the gate changes nothing there. + TEST_ASSERT(run.prepared[0].tokens == ungated.prepared[0].tokens); + const json & u = run.prepared[1].pflash_stats["unified"]; + TEST_ASSERT_MSG(u.value("decision", "") == "recall", u.dump()); + TEST_ASSERT(u["recall_gate"].value("decision", "") == "admitted"); + // The mock scores against the whole prompt: the history's KIWI? lifts the held kiwi line as much as MANGO? + // lifts the dropped mango line, so the deficit starts near one half and one unit covers it. + const double before = u["recall_gate"].value("deficit_before", 0.0); + TEST_ASSERT_MSG(before > 0.4 && before < 0.6, u["recall_gate"].dump()); + TEST_ASSERT(u["recall_gate"].value("deficit_after", 1.0) < 0.05); + // Nothing was trimmed at rendering: the served deficit is the planned one. + TEST_ASSERT(u["recall_gate"].value("trimmed_units", -1) == 0); + TEST_ASSERT(std::fabs(u["recall_gate"].value("deficit_after", 1.0) - u["recall_gate"].value("deficit_planned", 0.0)) < + 1e-12); + // The mango line comes back; the ungated fill also brings back filler, the gate does not. + const std::string & s = run.served_text[1]; + TEST_ASSERT(s.find("Line 60: plain filler text here mango\n") != std::string::npos); + const json & recall = u["recall"]; + const json & full = ungated.prepared[1].pflash_stats["unified"]["recall"]; + TEST_ASSERT(recall.is_array() && full.is_array()); + TEST_ASSERT(recall[0].value("units", 0) == 1); + TEST_ASSERT_MSG(full[0].value("units", 0) > 1, full.dump()); + TEST_ASSERT(run.prepared[1].tokens.size() < ungated.prepared[1].tokens.size()); + // The served prefix is untouched (I1). + const int vg1 = run.prepared[0].snapshot_cut; + TEST_ASSERT(std::equal(run.prepared[0].tokens.begin(), run.prepared[0].tokens.begin() + vg1, + run.prepared[1].tokens.begin())); + TEST_ASSERT(run.unified["counts"].value("recall.gate_admitted", 0) == 1); + TEST_ASSERT(run.unified.value("recall_gate", 0.0) == 0.05); +} + +TEST_CASE(ServerUnitFixture, test_unified_recall_gate_appends_nothing_when_the_view_answers) { + UnifiedOptions gate; + gate.recall_gate = "0.05"; + // The second question asks for the kiwi line again: it is held (kept at paging). + const auto run = run_unified_session(unified_fruit_turns("Which line mentions KIWI? Once more."), nullptr, + nullptr, 64, unified_sharp_fruit_score, 0, 0.3f, nullptr, 65536, gate); + const json & u = run.prepared[1].pflash_stats["unified"]; + TEST_ASSERT_MSG(u.value("decision", "") == "continue", u.dump()); + TEST_ASSERT(u["recall_gate"].value("decision", "") == "present"); + TEST_ASSERT(u["recall_gate"].value("deficit_before", 1.0) < 0.05); + TEST_ASSERT(run.served_text[1].find("[Earlier in this conversation]") == std::string::npos); + // The view grows by the new turn's raw tokens only. + const int grown = (int) run.prepared[1].tokens.size() - (int) run.prepared[0].tokens.size(); + TEST_ASSERT(grown == (int) run.ids[1].size() - (int) run.ids[0].size()); + TEST_ASSERT(run.calls_after == std::vector({1, 2})); + TEST_ASSERT(run.unified["counts"].value("recall.gate_present", 0) == 1); +} + +TEST_CASE(ServerUnitFixture, test_unified_recall_gate_fails_closed_on_a_non_finite_lift) { + // One history candidate scores NaN at the second question (finite at paging): the deficit cannot be computed, + // so nothing is appended -- never a gate decision over a silently smaller mass. + const auto score = [] (const std::string & segment, const std::string & prompt) { + if (prompt.find("MANGO?") != std::string::npos && segment.find("Line 30:") != std::string::npos) return (double) NAN; + return unified_sharp_fruit_score(segment, prompt); + }; + UnifiedOptions gate; + gate.recall_gate = "0.05"; + const auto run = run_unified_session(unified_fruit_turns("Which line mentions MANGO?"), nullptr, nullptr, 64, + score, 0, 0.3f, nullptr, 65536, gate); + const json & u = run.prepared[1].pflash_stats["unified"]; + TEST_ASSERT_MSG(u.value("decision", "") == "continue", u.dump()); + TEST_ASSERT(u["recall_gate"].value("decision", "") == "unscorable"); + TEST_ASSERT(run.served_text[1].find("[Earlier in this conversation]") == std::string::npos); + TEST_ASSERT(run.unified["counts"].value("recall.unscorable", 0) == 1); +} + +TEST_CASE(ServerUnitFixture, test_unified_recall_gate_leaves_newly_dropped_content_out_of_the_context) { + // The second question arrives with a new document that has a mango mark on every other line: paging keeps some + // of them and drops the rest. That dropped mass is neither present nor recallable, so the gate's S excludes it + // and the old region's missing mango line is measured against what the view could hold. + std::map marks; + for (int i = 2; i <= 80; i += 2) marks[i] = "mango"; + const std::string fresh_doc = unified_doc(80, marks, "Row "); + const auto turns = unified_fruit_turns((fresh_doc + "Which line mentions MANGO?").c_str()); + UnifiedOptions gate; + gate.recall_gate = "0.01"; + gate.trace_units = "1"; + const auto run = run_unified_session(turns, nullptr, nullptr, 64, unified_sharp_fruit_score, 0, 0.3f, nullptr, + 65536, gate); + const json & u = run.prepared[1].pflash_stats["unified"]; + TEST_ASSERT_MSG(u.value("decision", "") == "region_recall", u.dump()); + const json & g = u["recall_gate"]; + const json & q = u["recall_units"]; + const double dropped = g.value("new_dropped_mass", 0.0); + TEST_ASSERT_MSG(dropped > 0.0, g.dump()); + TEST_ASSERT(q.value("new_dropped_mass", 0.0) == dropped); + const double s = q.value("context_mass", 0.0) - dropped; + TEST_ASSERT(std::fabs(g.value("context_mass", 0.0) - s) < 1e-12); + double missing = 0.0; + const json & r = q["regions"][0]; + for (size_t k = 0; k < r["held"].size(); ++k) { + if (!r["held"][k].get()) missing += r["mass"][k].get(); + } + TEST_ASSERT_MSG(std::fabs(g.value("deficit_before", 0.0) - missing / s) < 1e-9, g.dump()); + // The old region's mango line comes back at the start of the question. + TEST_ASSERT(g.value("decision", "") == "admitted"); + TEST_ASSERT(run.served_text[1].find("Line 60: plain filler text here mango\n") != std::string::npos); +} + +TEST_CASE(ServerUnitFixture, test_unified_recall_gate_skips_the_drafter_once_everything_is_held) { + // A flat scorer: the deficit stays above a tiny theta, so questions recall to the cap until the region is + // complete; after that a question runs no drafter pass at all. + const std::string doc = unified_doc(80, {}); + json turn = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc + "Question 0?"}}, + }); + std::vector turns{turn}; + for (int q = 1; q <= 8; ++q) { + turn.push_back({{"role", "assistant"}, {"content", "Ok."}}); + turn.push_back({{"role", "user"}, {"content", "Question " + std::to_string(q) + "?"}}); + turns.push_back(turn); + } + UnifiedOptions gate; + gate.recall_gate = "0.000001"; + const auto run = run_unified_session(turns, nullptr, nullptr, 64, nullptr, 0, 0.3f, nullptr, 65536, gate); + const size_t n = run.prepared.size(); + for (int line = 1; line <= 80; ++line) { + TEST_ASSERT_MSG(unified_occurrences(run.served_text.back(), "Line " + std::to_string(line) + ":") == 1, + std::to_string(line)); + } + TEST_ASSERT(run.unified["counts"].value("recall.complete", 0) >= 1); + TEST_ASSERT(run.calls_after[n - 1] == run.calls_after[n - 2]); + const json & last = run.prepared[n - 1].pflash_stats["unified"]; + TEST_ASSERT_MSG(last.value("decision", "") == "continue", last.dump()); + TEST_ASSERT(last["recall_gate"].value("decision", "") == "complete"); +} + +TEST_CASE(ServerUnitFixture, test_unified_recall_trace_units_records_without_changing_bytes) { + UnifiedOptions traced; + traced.trace_units = "1"; + const auto turns = unified_fruit_turns("Which line mentions MANGO?"); + const auto run = run_unified_session(turns, nullptr, nullptr, 64, unified_sharp_fruit_score, 0, 0.3f, nullptr, + 65536, traced); + const auto plain = run_unified_session(turns, nullptr, nullptr, 64, unified_sharp_fruit_score); + for (size_t i = 0; i < run.prepared.size(); ++i) TEST_ASSERT(run.prepared[i].tokens == plain.prepared[i].tokens); + // Paging: the region's cuts relative to its content, and the units it kept. + const json & paged = run.prepared[0].pflash_stats["unified"]["admission"][0]["trace_units"]; + const int units = run.prepared[0].pflash_stats["unified"]["admission"][0].value("units", 0); + TEST_ASSERT(paged["cuts"].size() == (size_t) units + 1); + TEST_ASSERT(paged["kept"].size() == (size_t) units); + // A question: every unit's held flag, mass, cost and score, and the context mass. + const json & q = run.prepared[1].pflash_stats["unified"]["recall_units"]; + TEST_ASSERT(q.value("context_mass", 0.0) > 0.0); + const json & r = q["regions"][0]; + TEST_ASSERT(r["held"] == paged["kept"]); + TEST_ASSERT(r["mass"].size() == (size_t) units && r["cost"].size() == (size_t) units); + double total = 0.0; + for (const auto & m : r["mass"]) total += m.get(); + TEST_ASSERT(total > 0.0 && total <= q.value("context_mass", 0.0) + 1e-9); + TEST_ASSERT(plain.prepared[1].pflash_stats["unified"].find("recall_units") == + plain.prepared[1].pflash_stats["unified"].end()); +} + +TEST_CASE(ServerUnitFixture, test_unified_recall_gate_flags_fail_startup_on_bad_values) { + const std::string path = write_pflash_bpe_tokenizer_fixture({"a"}, "abc"); + Tokenizer tokenizer; + TEST_ASSERT(tokenizer.load_from_gguf(path.c_str())); + ServerConfig config; + config.prefix_cache_cap = 0; + config.prefill_cache_cap = 0; + luce_test::ScopedEnvVar unified{"PFLASH_UNIFIED", "1"}; + const auto refused = [&] (const char * gate, const char * source, const char * recall) { + luce_test::ScopedEnvVar g{"PFLASH_RECALL_GATE", gate}; + luce_test::ScopedEnvVar s{"PFLASH_RECALL_SOURCE", source}; + luce_test::ScopedEnvVar r{"PFLASH_RECALL", recall}; + LuceEngine engine(std::make_unique()); + try { + HttpServer server(engine, tokenizer, config); + } catch (const std::runtime_error &) { + return true; + } + return false; + }; + TEST_ASSERT(refused("0", nullptr, nullptr)); + TEST_ASSERT(refused("1", nullptr, nullptr)); + TEST_ASSERT(refused("1.5", nullptr, nullptr)); + TEST_ASSERT(refused("0.1x", nullptr, nullptr)); + TEST_ASSERT(refused("nan", nullptr, nullptr)); + TEST_ASSERT(refused("Off", nullptr, nullptr)); + TEST_ASSERT(refused("0.1", nullptr, "0")); + // PFLASH_RECALL_SOURCE was removed: any value refuses to start. + TEST_ASSERT(refused(nullptr, "fresh", nullptr)); + TEST_ASSERT(refused(nullptr, "kept", nullptr)); + { + luce_test::ScopedEnvVar trace{"PFLASH_RECALL_TRACE_UNITS", "yes"}; + TEST_ASSERT(refused(nullptr, nullptr, nullptr)); + } + TEST_ASSERT(!refused("0.1", nullptr, nullptr)); + TEST_ASSERT(!refused("off", nullptr, nullptr)); + TEST_ASSERT(!refused("off", nullptr, "0")); + TEST_ASSERT(!refused(nullptr, nullptr, "0")); + TEST_ASSERT(!refused(nullptr, nullptr, nullptr)); + unlink(path.c_str()); +} + +TEST_CASE(ServerUnitFixture, test_unified_defaults_gate_and_every_tool) { + // Under PFLASH_UNIFIED=1 alone: recall is gated at theta 0.075 and every tool's output may be paged. "off" and + // an empty tool list opt out; PFLASH_RECALL=0 leaves no gate. + const std::string path = write_pflash_bpe_tokenizer_fixture({"a"}, "abc"); + Tokenizer tokenizer; + TEST_ASSERT(tokenizer.load_from_gguf(path.c_str())); + ServerConfig config; + config.prefix_cache_cap = 0; + config.prefill_cache_cap = 0; + luce_test::ScopedEnvVar unified{"PFLASH_UNIFIED", "1"}; + const auto settings = [&] (const char * gate, const char * tools, const char * recall) { + luce_test::ScopedEnvVar g{"PFLASH_RECALL_GATE", gate}; + luce_test::ScopedEnvVar t{"PFLASH_REGION_TOOLS", tools}; + luce_test::ScopedEnvVar r{"PFLASH_RECALL", recall}; + LuceEngine engine(std::make_unique()); + HttpServer server(engine, tokenizer, config); + return HttpServerTestAccess::pflash_unified(server); + }; + const json defaults = settings(nullptr, nullptr, nullptr); + TEST_ASSERT_MSG(defaults.value("region_tools", "") == "*", defaults.dump()); + TEST_ASSERT(std::fabs(defaults["recall_gate"].get() - 0.075) < 1e-12); + const json opted_out = settings("off", "", nullptr); + TEST_ASSERT_MSG(opted_out.value("region_tools", "x").empty(), opted_out.dump()); + TEST_ASSERT(opted_out["recall_gate"].is_null()); + const json listed = settings("0.2", "read_file,grep", nullptr); + TEST_ASSERT(std::fabs(listed["recall_gate"].get() - 0.2) < 1e-12); + TEST_ASSERT(settings(nullptr, nullptr, "0")["recall_gate"].is_null()); + unlink(path.c_str()); +} + +TEST_CASE(ServerUnitFixture, test_unified_experimental_flags_fail_startup) { + const std::string path = write_pflash_bpe_tokenizer_fixture({"a"}, "abc"); + Tokenizer tokenizer; + TEST_ASSERT(tokenizer.load_from_gguf(path.c_str())); + ServerConfig config; + config.prefix_cache_cap = 0; + config.prefill_cache_cap = 0; + luce_test::ScopedEnvVar unified{"PFLASH_UNIFIED", "1"}; + { + luce_test::ScopedEnvVar arm{"PFLASH_UNIFIED_DEFRAG", "1"}; + LuceEngine engine(std::make_unique()); + bool threw = false; + try { + HttpServer server(engine, tokenizer, config); + } catch (const std::runtime_error &) { + threw = true; + } + TEST_ASSERT(threw); + } + { + luce_test::ScopedEnvVar tools{"PFLASH_REGION_TOOLS", "read_file,,grep"}; + LuceEngine engine(std::make_unique()); + bool threw = false; + try { + HttpServer server(engine, tokenizer, config); + } catch (const std::runtime_error &) { + threw = true; + } + TEST_ASSERT(threw); + } + unlink(path.c_str()); +} + +// The four lines of paragraph ``p`` of unified_paragraph_doc(). +static std::string unified_paragraph_lines(int p, const std::map & marks) { + std::string out; + for (int i = 1; i <= 4; ++i) { + out += "P" + std::to_string(p) + " line " + std::to_string(i) + ": filler text"; + const auto it = marks.find(p); + if (i == 3 && it != marks.end()) out += " " + it->second; + out += "\n"; + } + return out; +} + +// Paragraphs of four lines "P

line : filler text[ mark]\n", each followed by a blank line; a mark goes on line +// 3 of its paragraph, and paragraph ``gap`` gets a second blank line. +static std::string unified_paragraph_doc(int paragraphs, const std::map & marks, int gap = -1) { + std::string out; + for (int p = 1; p <= paragraphs; ++p) { + out += unified_paragraph_lines(p, marks); + out += p == gap ? "\n\n" : "\n"; + } + return out; +} + +// ``served`` is a paged form of ``raw``: every served line is the next raw line, a marker for the raw lines it +// hides, or the one preamble, whose counts agree. "" when it holds, else what broke. +static std::string unified_check_paged(const std::string & served, const std::string & raw) { + const auto split = [] (const std::string & text) { + std::vector out; + size_t at = 0; + for (;;) { + const size_t nl = text.find('\n', at); + out.push_back(text.substr(at, nl == std::string::npos ? std::string::npos : nl - at)); + if (nl == std::string::npos) break; + at = nl + 1; + } + if (!text.empty() && text.back() == '\n') out.pop_back(); + return out; + }; + const auto raw_lines = split(raw); + const std::string open = "[\xE2\x80\xA6 "; + const std::string close = " not shown \xE2\x80\xA6]"; + size_t next = 0; + int shown = 0; + int preambles = 0; + int said_shown = -1; + int said_total = -1; + for (const auto & line : split(served)) { + if (line.rfind("[Server note: this ", 0) == 0) { + ++preambles; + const size_t counts = line.find("shortened: "); + if (counts == std::string::npos || + std::sscanf(line.c_str() + counts + 11, "%d of %d", &said_shown, &said_total) != 2) { + return "preamble: " + line; + } + continue; + } + if (line.rfind(open, 0) == 0 && line.size() > close.size() && + line.compare(line.size() - close.size(), close.size(), close) == 0) { + next += (size_t) std::atoi(line.c_str() + open.size()); + continue; + } + if (next >= raw_lines.size() || line != raw_lines[next]) { + return "served line '" + line + "' at raw line " + std::to_string(next); + } + ++next; + ++shown; + } + if (next != raw_lines.size()) return "ended at raw line " + std::to_string(next) + " of " + + std::to_string(raw_lines.size()); + if (preambles != 1 || said_shown != shown || said_total != (int) raw_lines.size()) { + return "preamble says " + std::to_string(said_shown) + " of " + std::to_string(said_total) + ", served " + + std::to_string(shown) + " of " + std::to_string(raw_lines.size()); + } + return ""; +} + +// The content of the ``nth`` (0-based) message of ``role`` in a rendered Qwen prompt. +static std::string unified_rendered_content(const std::string & text, const std::string & role, int nth) { + const std::string open = "<|im_start|>" + role + "\n"; + size_t at = 0; + for (int k = 0; k <= nth; ++k) { + at = text.find(open, k == 0 ? 0 : at + 1); + if (at == std::string::npos) return ""; + } + const size_t begin = at + open.size(); + const size_t end = text.find("<|im_end|>", begin); + return end == std::string::npos ? "" : text.substr(begin, end - begin); +} + +TEST_CASE(ServerUnitFixture, test_unified_hands_back_without_selection_env) { + // PFLASH_UNIFIED=1 without a strict-selection environment: the view cannot score and never sees a request; the + // chat view serves every turn exactly as without the flag (a continuation takes the FlowKV path there). + const std::string doc = unified_doc(80, {{7, "kiwi"}, {60, "mango"}}); + const json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc + "Which line mentions KIWI?"}}, + }); + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", "Line 7."}}); + turn2.push_back({{"role", "user"}, {"content", "Which line mentions MANGO?"}}); + UnifiedOptions options; + options.select_mode = nullptr; + const auto run = run_unified_session({turn1, turn2}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, + nullptr, 65536, options); + UnifiedOptions off = options; + off.unified = false; + const auto base = run_unified_session({turn1, turn2}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, + nullptr, 65536, off); + for (size_t turn = 0; turn < 2; ++turn) { + TEST_ASSERT_MSG(run.prepared[turn].pflash_stats.find("unified") == run.prepared[turn].pflash_stats.end(), + run.prepared[turn].pflash_stats.dump()); + TEST_ASSERT(run.prepared[turn].tokens == base.prepared[turn].tokens); + TEST_ASSERT(run.prepared[turn].compressed == base.prepared[turn].compressed); + } + TEST_ASSERT(run.calls_after == base.calls_after); + TEST_ASSERT_MSG(run.unified.value("views", 1) == 0, run.unified.dump()); +} + +TEST_CASE(ServerUnitFixture, test_unified_reread_holds_unit_ending_in_blank_line) { + // Units are paragraphs ending in a blank line. A short user message that pastes paragraph 10 again covers its + // four lines; the blank line after it, next to an uncovered line, does not keep the unit from being held, so + // the question recalls other paragraphs, not that one again. + const std::map marks{{3, "kiwi"}, {10, "mango"}}; + const std::string doc = unified_paragraph_doc(16, marks); + const json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc + "Which line mentions KIWI?"}}, + }); + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", "P3 line 3."}}); + turn2.push_back({{"role", "user"}, {"content", "Here it is again:\n" + unified_paragraph_lines(10, marks) + + "\nWhich line mentions MANGO?"}}); + UnifiedOptions options; + options.paragraphs = true; + const auto run = run_unified_session({turn1, turn2}, nullptr, nullptr, 400, unified_fruit_score, 0, 0.3f, + nullptr, 65536, options); + TEST_ASSERT_MSG(run.prepared[0].pflash_stats["unified"].value("decision", "") == "region", + run.prepared[0].pflash_stats.dump()); + TEST_ASSERT(run.served_text[0].find("mango") == std::string::npos); + const json & u2 = run.prepared[1].pflash_stats["unified"]; + TEST_ASSERT_MSG(u2["admission"][0].value("decision", "") == "small", u2.dump()); + TEST_ASSERT_MSG(u2.value("reread_held_units", 0) == 1, u2.dump()); + TEST_ASSERT_MSG(u2.value("decision", "") == "recall", u2.dump()); + TEST_ASSERT_MSG(unified_occurrences(run.served_text[1], "mango") == 1, run.served_text[1]); +} + +TEST_CASE(ServerUnitFixture, test_unified_reread_coverage_counts_fallen_back_messages) { + // A big user message is a region candidate, but its cap does not cover the preamble and marker, so it falls + // back to verbatim while paging. It still repeats paragraph 10: that unit is held, never recalled again. + const std::map marks{{10, "mango"}}; + const std::string doc = unified_paragraph_doc(16, marks); + const json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc + "Summarise this."}}, + }); + std::string message = "Here it is again:\n" + unified_paragraph_lines(10, marks); + for (int i = 1; i <= 20; ++i) message += "Extra note " + std::to_string(i) + " about the plan\n"; + message += "Which line mentions MANGO?"; + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", "A list of paragraphs."}}); + turn2.push_back({{"role", "user"}, {"content", message}}); + UnifiedOptions options; + options.paragraphs = true; + const auto run = run_unified_session({turn1, turn2}, nullptr, nullptr, 400, unified_fruit_score, 0, 0.15f, + nullptr, 65536, options); + TEST_ASSERT_MSG(run.prepared[0].pflash_stats["unified"].value("decision", "") == "region", + run.prepared[0].pflash_stats.dump()); + TEST_ASSERT(run.served_text[0].find("mango") == std::string::npos); + const json & u2 = run.prepared[1].pflash_stats["unified"]; + TEST_ASSERT_MSG(u2["admission"][0].value("decision", "") == "cap_below_overhead", u2.dump()); + TEST_ASSERT_MSG(u2.value("decision", "") == "recall", u2.dump()); + TEST_ASSERT_MSG(u2.value("reread_held_units", 0) == 1, u2.dump()); + TEST_ASSERT_MSG(unified_occurrences(run.served_text[1], "mango") == 1, run.served_text[1]); +} + +TEST_CASE(ServerUnitFixture, test_unified_reset_hands_back_to_the_chat_view) { + // Recall overheads can grow a view past the raw prompt. A turn whose view no longer fits goes back to admission + // on the raw prompt; that history is uncached, so the chat view takes the turn instead of a raw prefill. + const std::string doc = unified_doc(80, {}); + json turn = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc + "Question 0?"}}, + }); + std::vector turns{turn}; + for (int q = 1; q <= 8; ++q) { + turn.push_back({{"role", "assistant"}, {"content", "Ok."}}); + turn.push_back({{"role", "user"}, {"content", "Question " + std::to_string(q) + "?"}}); + turns.push_back(turn); + } + const auto probe = run_unified_session(turns, nullptr, nullptr, 64, nullptr); + size_t over = 0; + for (size_t t = 1; t < probe.prepared.size() && over == 0; ++t) { + if (probe.prepared[t].tokens.size() > probe.ids[t].size()) over = t; + } + TEST_ASSERT_MSG(over > 0 && over + 1 < turns.size(), "no turn whose view is larger than its raw prompt"); + const std::vector session(turns.begin(), turns.begin() + (long) over + 2); + + // The view of turn ``over`` does not fit with a long output allowance; its raw prompt does. The next turn, with a + // short allowance again, must not continue the view the chat view replaced (nor an ancestor of it). + UnifiedOptions options; + options.max_outputs.assign(session.size(), 16); + options.max_outputs[over] = 4096; + const int fits_raw = (int) probe.ids[over].size() + 4096; + const auto run = run_unified_session(session, nullptr, nullptr, 64, nullptr, 0, 0.3f, nullptr, fits_raw, options); + TEST_ASSERT_MSG(run.prepared[over].unified_handback == "reset_uncached", run.prepared[over].pflash_stats.dump()); + TEST_ASSERT(run.prepared[over].error.empty()); + TEST_ASSERT(run.unified["counts"].value("reset.context", 0) == 1); + TEST_ASSERT(run.unified["counts"].value("raw.reset", 0) == 0); + TEST_ASSERT_MSG(run.prepared[over + 1].unified_handback == "uncached", run.prepared[over + 1].pflash_stats.dump()); + TEST_ASSERT_MSG(run.unified.value("views", -1) == 0, run.unified.dump()); + + // Nothing fits (the output allowance alone is the context): the chat view answers as it does without the flag. + const int fits_view = (int) probe.prepared[over].tokens.size() + 16; + UnifiedOptions tight; + tight.max_outputs.assign(session.size(), 16); + tight.max_outputs[over] = fits_view; + tight.allow_errors = true; + const auto again = run_unified_session(session, nullptr, nullptr, 64, nullptr, 0, 0.3f, nullptr, fits_view, tight); + UnifiedOptions tight_off = tight; + tight_off.unified = false; + const auto base = run_unified_session(session, nullptr, nullptr, 64, nullptr, 0, 0.3f, nullptr, fits_view, + tight_off); + TEST_ASSERT_MSG(again.prepared[over].unified_handback == "reset_uncached", again.prepared[over].pflash_stats.dump()); + TEST_ASSERT_MSG(again.prepared[over].error == base.prepared[over].error, again.prepared[over].error); + TEST_ASSERT(again.prepared[over].tokens == base.prepared[over].tokens); + + // The refused turn keeps the conversation's views: a retry with a short allowance continues the view. + std::vector retry(session.begin(), session.begin() + (long) over + 1); + retry.push_back(session[over]); + const auto retried = run_unified_session(retry, nullptr, nullptr, 64, nullptr, 0, 0.3f, nullptr, fits_view, tight); + TEST_ASSERT(!retried.prepared[over].error.empty()); + TEST_ASSERT_MSG(retried.prepared[over + 1].unified_handback.empty() && retried.prepared[over + 1].unified_view, + retried.prepared[over + 1].pflash_stats.dump()); +} + +TEST_CASE(ServerUnitFixture, test_unified_widened_cap_applies_once) { + // A stage-2 widened pass widens a region's cap by half, once: at paging and again at a later recall, never + // compounded. + const std::string doc = unified_doc(80, {{7, "kiwi"}, {60, "mango"}}); + const json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc + "Which line mentions KIWI?"}}, + }); + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", "Line 7."}}); + turn2.push_back({{"role", "user"}, {"content", "Which line mentions MANGO?"}}); + UnifiedOptions options; + options.widen = true; + const auto run = run_unified_session({turn1, turn2}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, + nullptr, 65536, options); + const json & paged = run.prepared[0].pflash_stats["unified"]["admission"][0]; + const int unwidened = paged.value("unwidened_cap", 0); + TEST_ASSERT_MSG(unwidened > 0 && paged.value("cap", 0) == unwidened + unwidened / 2, paged.dump()); + const json & recall = run.prepared[1].pflash_stats["unified"]["recall"]; + TEST_ASSERT_MSG(recall.is_array() && recall.size() == 1, run.prepared[1].pflash_stats.dump()); + TEST_ASSERT(recall[0].value("cap", 0) == paged.value("cap", 0)); + TEST_ASSERT(recall[0].value("wrapper", 0) > 0); + TEST_ASSERT(recall[0].value("tokens", 0) <= paged.value("cap", 0) - recall[0].value("wrapper", 0)); +} + +TEST_CASE(ServerUnitFixture, test_unified_distinct_tokenizers_keep_lines_whole) { + // Target and drafter merge differently: the target has "\n\n" and "\n\n\n" as one token (a blank line then + // starts inside a token, so it can be cut only at its separator, and the second of two blank lines not at all), + // and the escaped piece backslash + "nitem" (an escaped line "item ..." starts inside it); the drafter has its + // own words. Lines stay whole, markers count what they hide, and turn 2 reuses turn 1's served prompt token for + // token. + UnifiedOptions options; + options.distinct_drafter = true; + options.target_words = {"\n\n", "\n\n\n", "\\nitem", " filler", " mentions"}; + options.drafter_words = {" text", " line", "Which"}; + + const std::map marks{{3, "kiwi"}, {10, "mango"}}; + const std::string doc = unified_paragraph_doc(16, marks, /*gap=*/6); + const json turn1 = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", doc + "Which line mentions KIWI?"}}, + }); + json turn2 = turn1; + turn2.push_back({{"role", "assistant"}, {"content", "P3 line 3."}}); + turn2.push_back({{"role", "user"}, {"content", "Which line mentions MANGO?"}}); + const auto run = run_unified_session({turn1, turn2}, nullptr, nullptr, 64, unified_fruit_score, 0, 0.3f, + nullptr, 65536, options); + // The scenario holds: a blank line starts inside a target token, at a drafter token start. + const std::string & raw1 = run.raw_text[0]; + const size_t blank = raw1.find("text\n\nP2"); + const size_t gap = raw1.find("text\n\n\nP7"); + TEST_ASSERT(blank != std::string::npos && gap != std::string::npos); + const auto starts_at = [] (const std::vector & starts, size_t at) { + return std::binary_search(starts.begin(), starts.end(), at); + }; + TEST_ASSERT(starts_at(run.target_starts[0], blank + 4) && !starts_at(run.target_starts[0], blank + 5)); + TEST_ASSERT(starts_at(run.drafter_starts[0], blank + 5)); + TEST_ASSERT(!starts_at(run.target_starts[0], gap + 5) && !starts_at(run.target_starts[0], gap + 6)); + TEST_ASSERT(run.ids[0].size() != run.drafter_starts[0].size()); + + TEST_ASSERT_MSG(run.prepared[0].pflash_stats["unified"].value("decision", "") == "region", + run.prepared[0].pflash_stats.dump()); + const std::string broke = unified_check_paged(unified_rendered_content(run.served_text[0], "user", 0), + doc + "Which line mentions KIWI?"); + TEST_ASSERT_MSG(broke.empty(), broke + "\n" + run.served_text[0]); + TEST_ASSERT(run.served_text[0].find("P3 line 3: filler text kiwi\n") != std::string::npos); + const int cut = run.prepared[0].snapshot_cut; + TEST_ASSERT(cut > 0 && std::equal(run.prepared[0].tokens.begin(), run.prepared[0].tokens.begin() + cut, + run.prepared[1].tokens.begin())); + TEST_ASSERT_MSG(run.prepared[1].pflash_stats["unified"].value("decision", "") == "recall", + run.prepared[1].pflash_stats.dump()); + const std::string & s2 = run.served_text[1]; + const size_t block = s2.find("[Earlier in this conversation]\n[From an earlier user message]\n"); + TEST_ASSERT(block != std::string::npos); + TEST_ASSERT(s2.find("P10 line 3: filler text mango\n", block) != std::string::npos); + TEST_ASSERT(unified_occurrences(s2, "mango") == 1); + + // An escaped (JSON-string) tool output with the same two tokenizers: the result stays valid JSON and its lines + // whole, and the line the scores favour is kept (the mark is long enough to lie whole in some 7-token chunk; + // the cap leaves room for it next to the long tool preamble). + std::string body; + for (int i = 1; i <= 80; ++i) { + body += "item " + std::to_string(i) + " value" + (i == 33 ? " kiwikiwikiwikiwi" : "") + "\n"; + } + const std::string wrapped = json({{"content", body}, {"total_lines", 80}}).dump(); + const json tools = json::array({{{"type", "function"}, {"function", {{"name", "read_file"}}}}}); + const json tool_turn = json::array({ + {{"role", "system"}, {"content", "You are helpful."}}, + {{"role", "user"}, {"content", "Which item mentions KIWI?"}}, + {{"role", "assistant"}, {"content", ""}, {"tool_calls", json::array({ + unified_tool_call("call_1", "read_file", {{"path", "items.txt"}})})}}, + {{"role", "tool"}, {"tool_call_id", "call_1"}, {"content", wrapped}}, + }); + const auto tool_run = run_unified_session({tool_turn}, tools, "read_file", 64, unified_fruit_score, + /*chunk=*/7, 0.5f, nullptr, 65536, options); + const size_t escaped = tool_run.raw_text[0].find("value\\nitem 40 "); + TEST_ASSERT(escaped != std::string::npos); + TEST_ASSERT(starts_at(tool_run.target_starts[0], escaped + 5) && + !starts_at(tool_run.target_starts[0], escaped + 7)); + TEST_ASSERT(starts_at(tool_run.drafter_starts[0], escaped + 7)); + const auto & prepared = tool_run.prepared[0]; + TEST_ASSERT_MSG(prepared.compressed, prepared.pflash_stats.dump()); + TEST_ASSERT(prepared.pflash_stats["unified"]["admission"][0].value("escaped_lines", false)); + const std::string & served = tool_run.served_text[0]; + const size_t open = served.rfind("\n"); + const size_t close = served.find("\n", open); + TEST_ASSERT(open != std::string::npos && close != std::string::npos); + const json result = json::parse(served.substr(open + 16, close - open - 16), nullptr, false); + TEST_ASSERT_MSG(!result.is_discarded(), served.substr(open, 400)); + TEST_ASSERT(result.value("total_lines", 0) == 80); + const std::string content = result.value("content", ""); + const std::string content_broke = unified_check_paged(content, body); + TEST_ASSERT_MSG(content_broke.empty(), content_broke + "\n" + content); + TEST_ASSERT(content.find("item 33 value kiwikiwikiwikiwi\n") != std::string::npos); +}