diff --git a/python/freetoken/engine/sample.py b/python/freetoken/engine/sample.py
index f1984f7576..bca85c56fa 100644
--- a/python/freetoken/engine/sample.py
+++ b/python/freetoken/engine/sample.py
@@ -187,7 +187,9 @@ def _plan(self, batch: Batch, params) -> LogitsPlan | None:
hist.append(ids[prompt_len:].tolist() if penalized else [])
if p.repetition_penalty != 1.0:
want_prompt = True
- prompts.append(ids[:prompt_len].tolist())
+ # multimodal placeholders are pseudo-ids >= vocab_size (MM_PAD_SHIFT_VALUE);
+ # send them to the scratch column, not past the scatter target
+ prompts.append(ids[:prompt_len].clamp(max=pad).tolist())
else:
prompts.append([])
plan.hist_ids = _padded_rows(hist, pad, self.device)
diff --git a/python/freetoken/kvcache/host_prefix_tier.py b/python/freetoken/kvcache/host_prefix_tier.py
index 3b2d2d5dc3..39c96dfd6d 100644
--- a/python/freetoken/kvcache/host_prefix_tier.py
+++ b/python/freetoken/kvcache/host_prefix_tier.py
@@ -59,13 +59,15 @@ def __init__(self, state_pool, kv_pool=None, *, gdn_slots_host: int = 64,
tensors = {"conv": state_pool.conv_states, "rec": state_pool.recurrent_states}
tensors.update(state_pool.slot_states)
self._order = list(tensors)
- self._banks: dict[str, tuple[torch.Tensor, HostBank]] = {}
+ # banks only: the pool tensors are re-read on every copy, since
+ # LinearStatePool.rebuild replaces them (a runtime cache rebuild)
+ self._banks: dict[str, HostBank] = {}
for name, t in tensors.items():
# pool: [layers, slots, *rest] -> bank: [host_slots, layers, *rest]
bank = HostBank((gdn_slots_host, t.shape[0], *t.shape[2:]), t.dtype)
if self.device.type == "cuda" and torch.cuda.is_available():
bank.pin()
- self._banks[name] = (t, bank)
+ self._banks[name] = bank
self.num_slots = gdn_slots_host
self._free = list(range(gdn_slots_host))
self._in_use: set[int] = set()
@@ -102,40 +104,41 @@ def __init__(self, state_pool, kv_pool=None, *, gdn_slots_host: int = 64,
torch.cuda.Stream(self.device) if self.device.type == "cuda" else None)
# ---------------- GDN snapshot storage ----------------
+ def _pairs(self):
+ """(live pool tensor [layers, slots, ...], host bank) per state tensor."""
+ pool = self.pool
+ live = {"conv": pool.conv_states, "rec": pool.recurrent_states, **pool.slot_states}
+ return [(live[name], self._banks[name]) for name in self._order]
+
+ def _copy(self, copies) -> None:
+ if self.copy_stream is None:
+ for dst, src in copies:
+ dst.copy_(src)
+ return
+ # Order the copies after the caller's stream: the scheduler stream (itself made to wait
+ # on the engine stream) or the engine stream. Overlap launches the next batch before
+ # the previous one is drained, so a just-freed slot may still be written by it.
+ self.copy_stream.wait_stream(torch.cuda.current_stream(self.device))
+ with torch.cuda.stream(self.copy_stream):
+ for dst, src in copies:
+ dst.copy_(src, non_blocking=True)
+ self.copy_stream.synchronize()
+
def write_gdn(self, gpu_slot: int) -> int:
"""D2H do snapshot do gpu_slot pra um slot host livre. Retorna o host_slot.
- Deve ser enfileirado ANTES de state_pool.free(gpu_slot): a ordem de stream do
- engine garante leitura antes de qualquer reescrita do slot (ver spec)."""
+ Deve ser enfileirado ANTES de state_pool.free(gpu_slot)."""
if not self._free:
raise RuntimeError("HostPrefixTier: sem slots host GDN livres (chame evict_host)")
hs = self._free.pop()
self._in_use.add(hs)
- if self.copy_stream is not None:
- with torch.cuda.stream(self.copy_stream):
- for name in self._order:
- t, bank = self._banks[name]
- bank.tensor[hs].copy_(t[:, gpu_slot], non_blocking=True)
- self.copy_stream.synchronize()
- else:
- for name in self._order:
- t, bank = self._banks[name]
- bank.tensor[hs].copy_(t[:, gpu_slot])
+ self._copy([(bank.tensor[hs], t[:, gpu_slot]) for t, bank in self._pairs()])
return hs
def read_gdn(self, host_slot: int, dst_gpu_slot: int) -> None:
"""H2D host_slot -> dst_gpu_slot do pool (já alocado pelo caller)."""
if host_slot not in self._in_use:
raise KeyError(f"HostPrefixTier: slot host {host_slot} não está em uso")
- if self.copy_stream is not None:
- with torch.cuda.stream(self.copy_stream):
- for name in self._order:
- t, bank = self._banks[name]
- t[:, dst_gpu_slot].copy_(bank.tensor[host_slot], non_blocking=True)
- self.copy_stream.synchronize()
- else:
- for name in self._order:
- t, bank = self._banks[name]
- t[:, dst_gpu_slot].copy_(bank.tensor[host_slot])
+ self._copy([(t[:, dst_gpu_slot], bank.tensor[host_slot]) for t, bank in self._pairs()])
def free_gdn(self, host_slot: int) -> None:
self._in_use.discard(host_slot)
@@ -198,6 +201,7 @@ def _go():
del staged_s
if self.copy_stream is not None:
+ self.copy_stream.wait_stream(torch.cuda.current_stream(self.device))
with torch.cuda.stream(self.copy_stream):
_go()
self.copy_stream.synchronize()
@@ -235,6 +239,7 @@ def _go():
del staged_s
if self.copy_stream is not None:
+ self.copy_stream.wait_stream(torch.cuda.current_stream(self.device))
with torch.cuda.stream(self.copy_stream):
_go()
self.copy_stream.synchronize()
diff --git a/python/freetoken/models/qwen4_exp/ple.py b/python/freetoken/models/qwen4_exp/ple.py
index 284e023bdd..44973bc0a6 100644
--- a/python/freetoken/models/qwen4_exp/ple.py
+++ b/python/freetoken/models/qwen4_exp/ple.py
@@ -49,6 +49,10 @@
_FUSED_HASH_ENV = "FREETOKEN_PLE_FUSED_HASH"
+def _capturing(device: torch.device) -> bool:
+ return device.type == "cuda" and torch.cuda.is_current_stream_capturing()
+
+
def _fused_row_ids_enabled() -> bool:
"""The fused hash kernel, on unless ``FREETOKEN_PLE_FUSED_HASH=0`` takes it back to torch ops."""
return (os.getenv(_FUSED_HASH_ENV) or "1").strip() not in ("0", "false", "False")
@@ -482,12 +486,13 @@ def _token_index(self, meta: PLEMetadata) -> Tuple[torch.Tensor, torch.Tensor]:
The fused kernel addresses the hash window through these instead of materializing the
``[B, ctx+max_len]`` packed window. Memoized on the shape (which is all they depend on)
- so a captured replay reads a stable address instead of re-running the build; a build
- that happens DURING capture is not cached, since its buffers live in the graph pool.
+ for eager forwards only: a capture always builds its own, which the graph recomputes
+ into its pool on every replay. A cached pair (the eager warmup forward right before
+ capture leaves one) would be baked in by address, then evicted and reused.
"""
device = meta.input_ids.device
num_tokens = meta.input_ids.numel()
- capturing = device.type == "cuda" and torch.cuda.is_current_stream_capturing()
+ capturing = _capturing(device)
# is_decode is part of the key, not just the shape: a decode of B requests and a
# prefill of ONE B-token request are the same [T] and mean opposite things (one token
# per request at offset 0 vs B offsets inside one request).
@@ -496,7 +501,7 @@ def _token_index(self, meta: PLEMetadata) -> Tuple[torch.Tensor, torch.Tensor]:
(num_tokens,) if meta.is_decode else tuple(meta.seq_lens),
str(device),
)
- cached = self._token_index_cache.get(key)
+ cached = None if capturing else self._token_index_cache.get(key)
if cached is not None:
return cached
if meta.is_decode: # one token per request, each at offset 0
diff --git a/python/freetoken/scheduler/cache.py b/python/freetoken/scheduler/cache.py
index 3f833eaf18..3b9dc55746 100644
--- a/python/freetoken/scheduler/cache.py
+++ b/python/freetoken/scheduler/cache.py
@@ -619,8 +619,11 @@ def _rehydrate_snapshot(self, node) -> int | None:
prefix, copy the snapshot back into a fresh pool slot and return the slot; None =
keep walking (the match truncates as before)."""
tier = self.host_tier
- hit = tier.lookup(self.prefix_cache._collect_key(node), self.page_size)
- if hit is None or hit.gdn_slot is None:
+ key = self.prefix_cache._collect_key(node)
+ hit = tier.lookup(key, self.page_size)
+ # lookup is longest-prefix; a shorter entry's state would resume this node's full
+ # depth with the tokens in between missing from the recurrence
+ if hit is None or hit.gdn_slot is None or hit.host_len != len(key):
return None
pool = self.linear_state_pool
if pool.num_free_slots < 1:
diff --git a/python/freetoken/server/anthropic_api.py b/python/freetoken/server/anthropic_api.py
index 48c296ffdc..7f0d61c255 100644
--- a/python/freetoken/server/anthropic_api.py
+++ b/python/freetoken/server/anthropic_api.py
@@ -55,6 +55,7 @@
submit_generation,
with_keepalive,
)
+from .openai_api import _await_watching_disconnect
from .request_logger import log_request
# Emit a protocol-native `ping` event after this many seconds of stream silence,
@@ -140,9 +141,14 @@ async def handle_anthropic_messages(
return StreamingResponse(events, media_type="text/event-stream")
try:
- result = await generate_full(uid, spec, state, source="/v1/messages")
+ # an abandoned request must not keep decoding to max_tokens (#222's watcher)
+ result = await _await_watching_disconnect(
+ generate_full(uid, spec, state, source="/v1/messages"), request, state, [uid]
+ )
except GenerationError as exc:
return _anthropic_error_response(400, "invalid_request_error", str(exc))
+ if result is None:
+ return _anthropic_error_response(499, "api_error", "client disconnected before the response was ready")
response = anthropic_full_response(
result, req.model, uid, cache_report=cache_report, metrics=metrics
)
diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.py
index 43d18d5723..5c0e60f557 100644
--- a/python/freetoken/server/generation.py
+++ b/python/freetoken/server/generation.py
@@ -14,6 +14,7 @@
from __future__ import annotations
import asyncio
+import math
import json
import time
from collections.abc import AsyncIterator
@@ -229,8 +230,9 @@ def pick(value, key, framework):
for name, value in (("presence_penalty", presence_penalty), ("frequency_penalty", frequency_penalty)):
if not -2.0 <= float(value) <= 2.0:
raise ValueError(f"{name} must be in [-2, 2], got {value}")
- if resolved_rep <= 0.0:
- raise ValueError(f"repetition_penalty must be positive, got {resolved_rep}")
+ # not `<= 0`: NaN compares False and would poison every seen token's logit
+ if not (math.isfinite(resolved_rep) and resolved_rep > 0.0):
+ raise ValueError(f"repetition_penalty must be a positive finite number, got {resolved_rep}")
if min_tokens < 0:
raise ValueError(f"min_tokens must be >= 0, got {min_tokens}")
bias: list[list[float]] | None = None
@@ -478,7 +480,12 @@ def _make_reasoning_parser(spec: GenSpec, state: Any) -> ReasoningParser | None:
# The qwen3 chat template opens an implicit (thinking on) unless
# enable_thinking is explicitly false, so the model emits only the closing
# . Mirror that default here, else the chain-of-thought leaks into content.
- force_reasoning = (spec.chat_template_kwargs or {}).get("enable_thinking") is not False
+ # continue_final_message is the exception: the template renders the final assistant
+ # turn as ... + content, so the continuation starts in content.
+ ctk = spec.chat_template_kwargs or {}
+ force_reasoning = ctk.get("enable_thinking") is not False and not ctk.get(
+ "continue_final_message"
+ )
elif parser_name == "glm":
# GLM's template honors enable_thinking (default on) even with tools; the
# generic fallback would force thinking and mislabel disabled output as reasoning.
diff --git a/python/freetoken/server/openai_api.py b/python/freetoken/server/openai_api.py
index b67825bfaf..9ce63060ab 100644
--- a/python/freetoken/server/openai_api.py
+++ b/python/freetoken/server/openai_api.py
@@ -401,14 +401,17 @@ async def handle_chat_completion(
chunks = state.stream_with_cancellation(chunks, request, uids)
return StreamingResponse(chunks, media_type="text/event-stream")
+ tasks = [
+ asyncio.ensure_future(generate_full(u, spec, state, source="/v1/chat/completions"))
+ for u in uids
+ ]
try:
- results = await _await_watching_disconnect(
- asyncio.gather(*(generate_full(u, spec, state, source="/v1/chat/completions") for u in uids)),
- request,
- state,
- uids,
- )
+ results = await _await_watching_disconnect(asyncio.gather(*tasks), request, state, uids)
except GenerationError as exc:
+ # gather does not cancel the other choices; the aborts below drop the events they
+ # wait on, so without the cancel they would stay pending forever
+ for task in tasks:
+ task.cancel()
for u in uids:
await state.abort_user(u)
return create_error_response(str(exc), code=exc.code)
diff --git a/python/freetoken/server/responses_api.py b/python/freetoken/server/responses_api.py
index eea41722bb..9fd063a992 100644
--- a/python/freetoken/server/responses_api.py
+++ b/python/freetoken/server/responses_api.py
@@ -79,6 +79,7 @@
submit_generation,
with_keepalive,
)
+from .openai_api import _await_watching_disconnect
from .request_logger import log_request
# Seconds of event silence before a keep-alive frame is emitted on the stream.
@@ -177,9 +178,14 @@ async def handle_responses(
return StreamingResponse(events, media_type="text/event-stream")
try:
- result = await generate_full(uid, spec, state, source="/v1/responses")
+ # an abandoned request must not keep decoding to max_tokens (#222's watcher)
+ result = await _await_watching_disconnect(
+ generate_full(uid, spec, state, source="/v1/responses"), request, state, [uid]
+ )
except GenerationError as exc:
return _error_response(400, str(exc), exc.code)
+ if result is None:
+ return _error_response(499, "client disconnected before the response was ready")
response = build_responses_response(
result, req, response_id, created, cache_report=cache_report, metrics=metrics
)
diff --git a/python/freetoken/tokenizer/tokenize.py b/python/freetoken/tokenizer/tokenize.py
index 6a2cad2e2c..8085dd9d96 100644
--- a/python/freetoken/tokenizer/tokenize.py
+++ b/python/freetoken/tokenizer/tokenize.py
@@ -65,7 +65,15 @@ def _map_developer_role(
m.get("role") == "developer" for m in messages
):
return messages
- return [{**m, "role": "system"} if m.get("role") == "developer" else m for m in messages]
+ mapped = [{**m, "role": "system"} if m.get("role") == "developer" else m for m in messages]
+ # The frontend hoisted and merged system messages before this mapping ran (templates such
+ # as Qwen's raise on a system message that is not first), so do it again for the new ones.
+ system = [m for m in mapped if m.get("role") == "system"]
+ if len(system) == 1 and mapped[0] is system[0]:
+ return mapped
+ rest = [m for m in mapped if m.get("role") != "system"]
+ content = "\n\n".join(str(m.get("content") or "") for m in system if m.get("content"))
+ return [{"role": "system", "content": content}] + rest
class TokenizeManager:
diff --git a/python/freetoken/utils/mp.py b/python/freetoken/utils/mp.py
index f91498fbb2..ee35c057c1 100644
--- a/python/freetoken/utils/mp.py
+++ b/python/freetoken/utils/mp.py
@@ -22,7 +22,9 @@ class _CoalescedUnpacker:
"""
def __init__(self) -> None:
- self._unpacker = msgpack.Unpacker(raw=False)
+ # 0 = effectively uncapped (2**31-1), as unpackb had; the Unpacker default of 100 MiB would
+ # kill the worker on a large message (e.g. an image's pixel values)
+ self._unpacker = msgpack.Unpacker(raw=False, max_buffer_size=0)
self._pending: list[Any] = []
def feed(self, frame: bytes) -> None:
diff --git a/tests/engine/test_logits_processors.py b/tests/engine/test_logits_processors.py
index f051dec51f..a9645f4210 100644
--- a/tests/engine/test_logits_processors.py
+++ b/tests/engine/test_logits_processors.py
@@ -153,3 +153,13 @@ def test_prepare_builds_no_plan_without_processors_and_a_plan_with_them():
assert torch.isfinite(out[3]).all()
# greedy path: argmax over the processed logits
assert sampler.sample(logits, args)[1].item() == 5
+
+
+def test_repetition_penalty_ignores_multimodal_placeholder_ids_in_the_prompt():
+ # image tokens carry pseudo-ids >= MM_PAD_SHIFT_VALUE; scattering them into the
+ # [rows, V+1] prompt mask indexed out of bounds (a device-side assert on CUDA)
+ sampler = Sampler(CPU, V)
+ batch = SimpleNamespace(reqs=[_req([1, 1_000_000, 2], [3], 4, repetition_penalty=2.0)])
+ args = sampler.prepare(batch)
+ out = apply_logits_processors(torch.ones(1, V), args.plan, V)
+ assert out[0, 1] == 0.5 and out[0, 2] == 0.5 and out[0, 4] == 1.0
diff --git a/tests/kvcache/test_host_tier_live_pool.py b/tests/kvcache/test_host_tier_live_pool.py
new file mode 100644
index 0000000000..bd12836893
--- /dev/null
+++ b/tests/kvcache/test_host_tier_live_pool.py
@@ -0,0 +1,53 @@
+"""The GDN snapshot tier (upstream #525) must copy against the pool's CURRENT tensors, which
+LinearStatePool.rebuild replaces, and must order its copies after the engine stream."""
+
+import contextlib
+from types import SimpleNamespace
+
+import torch
+
+import freetoken.kvcache.host_prefix_tier as tier_module
+from freetoken.kvcache.host_prefix_tier import HostPrefixTier
+
+
+def _pool(slots: int):
+ return SimpleNamespace(
+ conv_states=torch.zeros(2, slots, 4, 3),
+ recurrent_states=torch.zeros(2, slots, 2, 4),
+ slot_states={},
+ )
+
+
+def test_read_after_a_pool_rebuild_lands_in_the_new_tensors():
+ pool = _pool(3)
+ tier = HostPrefixTier(pool, gdn_slots_host=2, device="cpu")
+ pool.conv_states[:, 1] = 5.0
+ pool.recurrent_states[:, 1] = 7.0
+ host = tier.write_gdn(1)
+ # a runtime rebuild swaps in fresh (larger) state tensors
+ pool.conv_states = torch.zeros(2, 6, 4, 3)
+ pool.recurrent_states = torch.zeros(2, 6, 2, 4)
+ tier.read_gdn(host, 5)
+ assert torch.all(pool.conv_states[:, 5] == 5.0)
+ assert torch.all(pool.recurrent_states[:, 5] == 7.0)
+
+
+def test_copies_wait_for_the_engine_stream(monkeypatch):
+ events = []
+ engine = object()
+
+ class FakeStream:
+ def wait_stream(self, other):
+ events.append(("wait", other))
+
+ def synchronize(self):
+ events.append(("sync",))
+
+ monkeypatch.setattr(tier_module.torch.cuda, "current_stream", lambda device=None: engine)
+ monkeypatch.setattr(tier_module.torch.cuda, "stream", lambda s: contextlib.nullcontext())
+ pool = _pool(3)
+ tier = HostPrefixTier(pool, gdn_slots_host=2, device="cpu")
+ tier.copy_stream = FakeStream()
+ host = tier.write_gdn(1)
+ tier.read_gdn(host, 2)
+ assert events == [("wait", engine), ("sync",), ("wait", engine), ("sync",)]
diff --git a/tests/kvcache/test_host_tier_rehydrate_exact.py b/tests/kvcache/test_host_tier_rehydrate_exact.py
new file mode 100644
index 0000000000..f67402041f
--- /dev/null
+++ b/tests/kvcache/test_host_tier_rehydrate_exact.py
@@ -0,0 +1,39 @@
+"""The GDN snapshot tier (upstream #525) may only revive a tombstone from a snapshot of
+exactly that node's prefix: the host index lookup is longest-prefix, and a shorter
+snapshot would resume the recurrence with the tokens in between silently missing."""
+
+from types import SimpleNamespace
+
+import torch
+
+from freetoken.kvcache.host_prefix_tier import HostMatch
+from freetoken.scheduler.cache import CacheManager
+
+
+def _manager(host_len: int, key_len: int = 128):
+ reads = []
+ tier = SimpleNamespace(
+ lookup=lambda key, page_size: HostMatch(host_len, None, 7),
+ read_gdn=lambda host_slot, gpu_slot: reads.append((host_slot, gpu_slot)),
+ )
+ pool = SimpleNamespace(num_free_slots=4, alloc=lambda n: [3])
+ manager = SimpleNamespace(
+ host_tier=tier,
+ prefix_cache=SimpleNamespace(_collect_key=lambda node: torch.arange(key_len)),
+ page_size=64,
+ linear_state_pool=pool,
+ ensure_mamba_slots=lambda n: None,
+ )
+ return manager, reads
+
+
+def test_a_shorter_host_snapshot_does_not_revive_a_deeper_node():
+ manager, reads = _manager(host_len=64)
+ assert CacheManager._rehydrate_snapshot(manager, node=object()) is None
+ assert reads == []
+
+
+def test_an_exact_host_snapshot_revives_the_node():
+ manager, reads = _manager(host_len=128)
+ assert CacheManager._rehydrate_snapshot(manager, node=object()) == 3
+ assert reads == [(7, 3)]
diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py
index 74d4fd614e..02abaee1f4 100644
--- a/tests/models/qwen4_exp/test_ple.py
+++ b/tests/models/qwen4_exp/test_ple.py
@@ -201,6 +201,19 @@ def test_token_index_cache_is_keyed_by_shape():
assert embedding._token_index(other)[0] is not first[0]
+def test_token_index_capture_never_reuses_an_evictable_cache_entry(monkeypatch):
+ """The eager warmup forward caches the decode pair right before capture; a capture that
+ reused it would bake an address the FIFO later evicts while the graph still reads it."""
+ embedding = _make_layer(_config()).ple_embedding
+ decode = _meta([[5]], [[EOS, EOS]], decode=True)
+ eager = embedding._token_index(decode) # the warmup forward
+ monkeypatch.setattr(ple_module, "_capturing", lambda device: True)
+ captured = embedding._token_index(decode)
+ assert captured[0] is not eager[0] and captured[1] is not eager[1]
+ assert all(v[0] is not captured[0] for v in embedding._token_index_cache.values())
+ assert captured[0].tolist() == [0] and captured[1].tolist() == [0]
+
+
def test_token_index_does_not_confuse_a_decode_with_a_one_request_prefill():
"""Same [T], opposite meaning: B decode rows at offset 0 vs B offsets in one request."""
layer = _make_layer(_config())
diff --git a/tests/server/test_openai_api.py b/tests/server/test_openai_api.py
index ce675a2508..7d5b3cd114 100644
--- a/tests/server/test_openai_api.py
+++ b/tests/server/test_openai_api.py
@@ -223,6 +223,26 @@ def test_glm_reasoning_parser_honors_disabled_thinking_with_tools():
assert parser is not None and parser.detector.force_reasoning is True
+def test_qwen3_continuation_of_a_final_assistant_message_is_content_not_reasoning():
+ # The Qwen template closes the think block before the assistant's content, so a
+ # continue_final_message continuation never emits ; starting the parser inside
+ # reasoning filed the whole continuation under reasoning_content.
+ from freetoken.server.generation import _make_reasoning_parser
+
+ state = FakeState([], reasoning_parser="qwen3")
+ req = chat_request(
+ messages=[
+ {"role": "user", "content": "Count to five."},
+ {"role": "assistant", "content": "One, two,"},
+ ],
+ continue_final_message=True,
+ )
+ parser = _make_reasoning_parser(chat_request_to_genspec(req, {}), state)
+ assert parser is not None and parser.detector.force_reasoning is False
+ plain = _make_reasoning_parser(chat_request_to_genspec(chat_request(), {}), state)
+ assert plain.detector.force_reasoning is True
+
+
def test_non_stream_chat_completion_returns_openai_tool_calls_and_sends_tools():
output = '[TOOL_CALLS] [{"name":"get_weather","arguments":{"city":"Paris"}}]'
state = FakeState(
diff --git a/tests/server/test_sampling_validation.py b/tests/server/test_sampling_validation.py
new file mode 100644
index 0000000000..3b2e1d6473
--- /dev/null
+++ b/tests/server/test_sampling_validation.py
@@ -0,0 +1,23 @@
+"""resolve_sampling rejects non-finite penalties: NaN passes a `<= 0` check and would
+turn every already-seen token's logit into NaN."""
+
+import pytest
+
+from freetoken.server.generation import resolve_sampling
+
+
+def _resolve(**extra):
+ return resolve_sampling(
+ temperature=None, top_k=None, top_p=None, max_tokens=8, ignore_eos=False,
+ model_sampling={}, **extra,
+ )
+
+
+@pytest.mark.parametrize("value", [float("nan"), float("inf"), 0.0, -1.0])
+def test_repetition_penalty_must_be_positive_and_finite(value):
+ with pytest.raises(ValueError, match="repetition_penalty"):
+ _resolve(repetition_penalty=value)
+
+
+def test_a_normal_repetition_penalty_is_kept():
+ assert _resolve(repetition_penalty=1.1).repetition_penalty == pytest.approx(1.1)
diff --git a/tests/server/test_stream_cancellation.py b/tests/server/test_stream_cancellation.py
index ba7da7c17b..71d5ac8b85 100644
--- a/tests/server/test_stream_cancellation.py
+++ b/tests/server/test_stream_cancellation.py
@@ -316,3 +316,61 @@ def new_user(self):
assert resp.status_code == 499
assert sorted(state.aborted) == [8, 9]
+
+
+def test_a_failed_choice_cancels_the_other_choices(monkeypatch):
+ # n > 1: one choice errors while another is still generating. gather passes the error up
+ # without cancelling the sibling, and the aborts drop the event it waits on.
+ monkeypatch.setattr(openai_api, "_DISCONNECT_POLL_SECONDS", 0.01)
+
+ class _State(_ApiState):
+ def __init__(self):
+ super().__init__(acks=None)
+ self._next = 7
+
+ def new_user(self):
+ self._next += 1
+ return self._next
+
+ async def wait_for_ack(self, uid):
+ if uid == 8:
+ yield SimpleNamespace(**{**vars(_ack("")), "error": "boom"})
+ return
+ await asyncio.sleep(3600)
+ yield _ack("never")
+
+ async def scenario():
+ state = _State()
+ resp = await openai_api.handle_chat_completion(
+ _chat_req(n=2), _Request(disconnected=False), state, {}
+ )
+ await asyncio.sleep(0)
+ pending = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
+ return resp, state, pending
+
+ resp, state, pending = asyncio.run(scenario())
+ assert resp.status_code == 400
+ assert sorted(state.aborted) == [8, 9]
+ assert pending == []
+
+
+def test_messages_and_responses_non_stream_disconnect_delivers_abort(monkeypatch):
+ # #222 only wrapped the OpenAI handlers; the Anthropic and Responses endpoints kept
+ # decoding an abandoned non-streaming request to max_tokens.
+ from freetoken.server import anthropic_api, responses_api
+ from freetoken.server.anthropic_models import AnthropicMessagesRequest
+ from freetoken.server.responses_api import ResponsesRequest
+
+ monkeypatch.setattr(openai_api, "_DISCONNECT_POLL_SECONDS", 0.01)
+
+ state = _ApiState(acks=None)
+ req = AnthropicMessagesRequest(model="m", max_tokens=8, messages=[{"role": "user", "content": "hi"}])
+ handler = anthropic_api.handle_anthropic_messages(req, _Request(disconnected=True), state, {})
+ resp = asyncio.run(asyncio.wait_for(handler, timeout=5)) # unwatched: hangs, then times out
+ assert resp.status_code == 499 and state.aborted == [7]
+
+ state = _ApiState(acks=None)
+ req = ResponsesRequest(model="m", input="hi", max_output_tokens=8)
+ handler = responses_api.handle_responses(req, _Request(disconnected=True), state, {})
+ resp = asyncio.run(asyncio.wait_for(handler, timeout=5))
+ assert resp.status_code == 499 and state.aborted == [7]
diff --git a/tests/tokenizer/test_tokenize.py b/tests/tokenizer/test_tokenize.py
index ae3af3a1bd..5aea33855f 100644
--- a/tests/tokenizer/test_tokenize.py
+++ b/tests/tokenizer/test_tokenize.py
@@ -296,3 +296,21 @@ def test_developer_role_maps_to_system_unless_the_template_knows_it():
assert mapped[0]["content"] == "be brief" and msgs[0]["role"] == "developer" # input untouched
assert _map_developer_role(msgs, "{% if message.role == 'developer' %}") is msgs
assert _map_developer_role(msgs[1:], None) == msgs[1:] # nothing to map
+
+
+def test_mapped_developer_messages_are_merged_into_one_leading_system_message():
+ """The frontend hoists system messages before the tokenizer maps developer -> system; a
+ template that rejects a non-first system message must still get exactly one, first."""
+ from freetoken.tokenizer.tokenize import _map_developer_role
+
+ template = "{% if message.role == 'system' %}"
+ msgs = [
+ {"role": "system", "content": "be brief"},
+ {"role": "developer", "content": "answer in German"},
+ {"role": "user", "content": "hi"},
+ ]
+ mapped = _map_developer_role(msgs, template)
+ assert [m["role"] for m in mapped] == ["system", "user"]
+ assert mapped[0]["content"] == "be brief\n\nanswer in German"
+ late = [{"role": "user", "content": "hi"}, {"role": "developer", "content": "be brief"}]
+ assert [m["role"] for m in _map_developer_role(late, template)] == ["system", "user"]
diff --git a/tests/utils/test_mp_large_frame.py b/tests/utils/test_mp_large_frame.py
new file mode 100644
index 0000000000..f91ee5604d
--- /dev/null
+++ b/tests/utils/test_mp_large_frame.py
@@ -0,0 +1,13 @@
+"""The coalesced-frame decoder (#466) must accept a message larger than msgpack's
+Unpacker default buffer (100 MiB), which unpackb never limited."""
+
+import msgpack
+
+from freetoken.utils.mp import _CoalescedUnpacker
+
+
+def test_a_message_over_100_mib_decodes():
+ payload = {"pixels": b"x" * (101 * 1024 * 1024)}
+ unpacker = _CoalescedUnpacker()
+ unpacker.feed(msgpack.packb(payload))
+ assert len(unpacker) == 1 and len(unpacker.take()["pixels"]) == len(payload["pixels"])