From 6a948d74eca5b7195826550bfbc7e88dae6c9667 Mon Sep 17 00:00:00 2001 From: autoantohaki Date: Wed, 21 Oct 2026 18:43:21 -0300 Subject: [PATCH] fix(grpc): default each unset sampling field independently --- .../impossible/inferences/v1/inference.proto | 7 +-- .../impossible-inferences-server/src/grpc.rs | 44 ++++++++++++++----- .../impossible-inferences-server/src/lib.rs | 6 +-- docs/grpc-mcp-api.md | 4 +- 4 files changed, 44 insertions(+), 17 deletions(-) diff --git a/crates/impossible-inferences-protocol/proto/impossible/inferences/v1/inference.proto b/crates/impossible-inferences-protocol/proto/impossible/inferences/v1/inference.proto index 503212a..5d16540 100644 --- a/crates/impossible-inferences-protocol/proto/impossible/inferences/v1/inference.proto +++ b/crates/impossible-inferences-protocol/proto/impossible/inferences/v1/inference.proto @@ -9,10 +9,11 @@ service InferenceService { rpc ChatStream(ChatRequest) returns (stream GenerationEvent); } +// Unset fields use the same defaults as HTTP: max_tokens 256, temperature 0.8, top_p 0.95. message Sampling { - uint32 max_tokens = 1; - float temperature = 2; - float top_p = 3; + optional uint32 max_tokens = 1; + optional float temperature = 2; + optional float top_p = 3; optional uint32 seed = 4; repeated string stop = 5; } diff --git a/crates/impossible-inferences-server/src/grpc.rs b/crates/impossible-inferences-server/src/grpc.rs index 7e6705d..07fe222 100644 --- a/crates/impossible-inferences-server/src/grpc.rs +++ b/crates/impossible-inferences-server/src/grpc.rs @@ -254,21 +254,15 @@ fn chat_request(request: ChatRequest) -> Result { } fn sampling(sampling: Option) -> GenerationRequest { - let sampling = sampling.unwrap_or(Sampling { - max_tokens: DEFAULT_MAX_TOKENS, - temperature: DEFAULT_TEMPERATURE, - top_p: DEFAULT_TOP_P, - seed: None, - stop: Vec::new(), - }); + let sampling = sampling.unwrap_or_default(); GenerationRequest { model: String::new(), input: GenerationInput::Completion { prompt: String::new(), }, - max_tokens: sampling.max_tokens, - temperature: sampling.temperature, - top_p: sampling.top_p, + max_tokens: sampling.max_tokens.unwrap_or(DEFAULT_MAX_TOKENS), + temperature: sampling.temperature.unwrap_or(DEFAULT_TEMPERATURE), + top_p: sampling.top_p.unwrap_or(DEFAULT_TOP_P), seed: sampling.seed, stop: sampling.stop, } @@ -426,3 +420,33 @@ async fn publish_health(reporter: &mut tonic_health::server::HealthReporter, rea .await; } } + +#[cfg(test)] +mod tests { + use super::{DEFAULT_MAX_TOKENS, DEFAULT_TEMPERATURE, DEFAULT_TOP_P, Sampling, sampling}; + + #[test] + fn partial_sampling_keeps_defaults_for_unset_fields() { + let partial = sampling(Some(Sampling { + max_tokens: Some(32), + seed: Some(7), + ..Sampling::default() + })); + assert_eq!(partial.max_tokens, 32); + assert!((partial.temperature - DEFAULT_TEMPERATURE).abs() < f32::EPSILON); + assert!((partial.top_p - DEFAULT_TOP_P).abs() < f32::EPSILON); + assert_eq!(partial.seed, Some(7)); + + let greedy = sampling(Some(Sampling { + temperature: Some(0.0), + ..Sampling::default() + })); + assert_eq!(greedy.max_tokens, DEFAULT_MAX_TOKENS); + assert!(greedy.temperature.abs() < f32::EPSILON); + assert!((greedy.top_p - DEFAULT_TOP_P).abs() < f32::EPSILON); + + let absent = sampling(None); + assert_eq!(absent.max_tokens, DEFAULT_MAX_TOKENS); + assert!((absent.top_p - DEFAULT_TOP_P).abs() < f32::EPSILON); + } +} diff --git a/crates/impossible-inferences-server/src/lib.rs b/crates/impossible-inferences-server/src/lib.rs index 42fb745..0210f4b 100644 --- a/crates/impossible-inferences-server/src/lib.rs +++ b/crates/impossible-inferences-server/src/lib.rs @@ -2749,9 +2749,9 @@ mod tests { model: CURATED_MODEL_ID.to_owned(), prompt: "hello".to_owned(), sampling: Some(Sampling { - max_tokens: 8, - temperature: 0.0, - top_p: 1.0, + max_tokens: Some(8), + temperature: Some(0.0), + top_p: Some(1.0), seed: Some(7), stop: Vec::new(), }), diff --git a/docs/grpc-mcp-api.md b/docs/grpc-mcp-api.md index dfcae20..b767cc9 100644 --- a/docs/grpc-mcp-api.md +++ b/docs/grpc-mcp-api.md @@ -17,7 +17,9 @@ The source contract is - `Complete` and `Chat` for unary generation; - `CompleteStream` and `ChatStream` for ordered delta, usage, and finish events. -Messages are limited to 1 MiB. Client `grpc-timeout` is honored up to the server's 30-second +Every `Sampling` field is optional. Unset fields use the HTTP defaults (`max_tokens` 256, +`temperature` 0.8, `top_p` 0.95), so a request can set only the fields it needs. Messages are +limited to 1 MiB. Client `grpc-timeout` is honored up to the server's 30-second upper bound. Dropping a response stream cancels or safely abandons generation and releases its bounded engine lane. The listener also exposes standard gRPC health and v1 reflection. Health is `SERVING` only while the local engine reports ready and transitions to `NOT_SERVING` when the