Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,11 @@ service InferenceService {
rpc ChatStream(ChatRequest) returns (stream GenerationEvent);
}

// Unset fields use the same defaults as HTTP: max_tokens 256, temperature 0.8, top_p 0.95.
message Sampling {
uint32 max_tokens = 1;
float temperature = 2;
float top_p = 3;
optional uint32 max_tokens = 1;
optional float temperature = 2;
optional float top_p = 3;
optional uint32 seed = 4;
repeated string stop = 5;
}
Expand Down
44 changes: 34 additions & 10 deletions crates/impossible-inferences-server/src/grpc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -254,21 +254,15 @@ fn chat_request(request: ChatRequest) -> Result<GenerationRequest, Status> {
}

fn sampling(sampling: Option<Sampling>) -> GenerationRequest {
let sampling = sampling.unwrap_or(Sampling {
max_tokens: DEFAULT_MAX_TOKENS,
temperature: DEFAULT_TEMPERATURE,
top_p: DEFAULT_TOP_P,
seed: None,
stop: Vec::new(),
});
let sampling = sampling.unwrap_or_default();
GenerationRequest {
model: String::new(),
input: GenerationInput::Completion {
prompt: String::new(),
},
max_tokens: sampling.max_tokens,
temperature: sampling.temperature,
top_p: sampling.top_p,
max_tokens: sampling.max_tokens.unwrap_or(DEFAULT_MAX_TOKENS),
temperature: sampling.temperature.unwrap_or(DEFAULT_TEMPERATURE),
top_p: sampling.top_p.unwrap_or(DEFAULT_TOP_P),
seed: sampling.seed,
stop: sampling.stop,
}
Expand Down Expand Up @@ -426,3 +420,33 @@ async fn publish_health(reporter: &mut tonic_health::server::HealthReporter, rea
.await;
}
}

#[cfg(test)]
mod tests {
use super::{DEFAULT_MAX_TOKENS, DEFAULT_TEMPERATURE, DEFAULT_TOP_P, Sampling, sampling};

#[test]
fn partial_sampling_keeps_defaults_for_unset_fields() {
let partial = sampling(Some(Sampling {
max_tokens: Some(32),
seed: Some(7),
..Sampling::default()
}));
assert_eq!(partial.max_tokens, 32);
assert!((partial.temperature - DEFAULT_TEMPERATURE).abs() < f32::EPSILON);
assert!((partial.top_p - DEFAULT_TOP_P).abs() < f32::EPSILON);
assert_eq!(partial.seed, Some(7));

let greedy = sampling(Some(Sampling {
temperature: Some(0.0),
..Sampling::default()
}));
assert_eq!(greedy.max_tokens, DEFAULT_MAX_TOKENS);
assert!(greedy.temperature.abs() < f32::EPSILON);
assert!((greedy.top_p - DEFAULT_TOP_P).abs() < f32::EPSILON);

let absent = sampling(None);
assert_eq!(absent.max_tokens, DEFAULT_MAX_TOKENS);
assert!((absent.top_p - DEFAULT_TOP_P).abs() < f32::EPSILON);
}
}
6 changes: 3 additions & 3 deletions crates/impossible-inferences-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2749,9 +2749,9 @@ mod tests {
model: CURATED_MODEL_ID.to_owned(),
prompt: "hello".to_owned(),
sampling: Some(Sampling {
max_tokens: 8,
temperature: 0.0,
top_p: 1.0,
max_tokens: Some(8),
temperature: Some(0.0),
top_p: Some(1.0),
seed: Some(7),
stop: Vec::new(),
}),
Expand Down
4 changes: 3 additions & 1 deletion docs/grpc-mcp-api.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ The source contract is
- `Complete` and `Chat` for unary generation;
- `CompleteStream` and `ChatStream` for ordered delta, usage, and finish events.

Messages are limited to 1 MiB. Client `grpc-timeout` is honored up to the server's 30-second
Every `Sampling` field is optional. Unset fields use the HTTP defaults (`max_tokens` 256,
`temperature` 0.8, `top_p` 0.95), so a request can set only the fields it needs. Messages are
limited to 1 MiB. Client `grpc-timeout` is honored up to the server's 30-second
upper bound. Dropping a response stream cancels or safely abandons generation and releases its
bounded engine lane. The listener also exposes standard gRPC health and v1 reflection. Health is
`SERVING` only while the local engine reports ready and transitions to `NOT_SERVING` when the
Expand Down
Loading