Skip to content

Commit fc279ca

Browse files
committed
fix: improve error handling in OpenAICompatibleTinkerServer sampling
- Wrapped the sampling call in a try-except block to handle APIStatusError exceptions gracefully. - Extracted detailed error messages from the exception body to provide clearer feedback in case of failures.
1 parent 7b341e4 commit fc279ca

1 file changed

Lines changed: 23 additions & 13 deletions

File tree

src/art/tinker/server.py

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -104,20 +104,30 @@ async def chat_completions(
104104
add_generation_prompt=True,
105105
)
106106
)
107-
sample_response = await sampler_client.sample_async(
108-
prompt=prompt,
109-
num_samples=body.get("n") or 1,
110-
sampling_params=tinker.SamplingParams(
111-
max_tokens=body.get("max_completion_tokens")
112-
or body.get("max_tokens"),
113-
seed=body.get("seed"),
114-
temperature=(
115-
t if (t := body.get("temperature")) is not None else 1.0
107+
try:
108+
sample_response = await sampler_client.sample_async(
109+
prompt=prompt,
110+
num_samples=body.get("n") or 1,
111+
sampling_params=tinker.SamplingParams(
112+
max_tokens=body.get("max_completion_tokens")
113+
or body.get("max_tokens"),
114+
seed=body.get("seed"),
115+
temperature=(
116+
t if (t := body.get("temperature")) is not None else 1.0
117+
),
118+
top_k=body.get("top_k") or -1,
119+
top_p=body.get("top_p") or 1.0,
116120
),
117-
top_k=body.get("top_k") or -1,
118-
top_p=body.get("top_p") or 1.0,
119-
),
120-
)
121+
)
122+
except tinker.APIStatusError as e:
123+
error_body = e.body
124+
if isinstance(error_body, dict) and "detail" in error_body:
125+
detail = error_body["detail"]
126+
elif error_body is not None:
127+
detail = error_body
128+
else:
129+
detail = str(e)
130+
raise HTTPException(status_code=e.status_code, detail=detail) from e
121131
choices: list[Choice] = []
122132
for i, sequence in enumerate(sample_response.sequences):
123133
assert sequence.logprobs is not None, "Logprobs are required"

0 commit comments

Comments
 (0)