diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3cac24..3c1c7b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,3 +116,20 @@ jobs: print(f" {name}") sys.exit(1 if failed else 0) PY + + tools: + runs-on: ubuntu-latest + env: + PYTHONDONTWRITEBYTECODE: 1 + steps: + - uses: actions/checkout@v4 + with: + # --check-all reloads each preimage with git show, so it needs history. + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Test the tools + run: python3 -m unittest discover -s tools -v + - name: Check the path redaction ledgers + run: python3 tools/redact_local_paths.py --check-all migrations/path-redaction --evidence-root results diff --git a/.gitignore b/.gitignore index b73592b..bb99068 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ /target/ /willitcall-result.json /site/ +__pycache__/ +*.pyc diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7a046b4..c4545a2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,6 +10,14 @@ cargo fmt --all -- --check cargo clippy --workspace --all-targets -- -D warnings ``` +If you change `tools/` or a path redaction ledger, also run the two Python +checks that CI runs: + +```sh +python3 -m unittest discover -s tools -v +python3 tools/redact_local_paths.py --check-all migrations/path-redaction --evidence-root results +``` + ## Adding a scenario > **A scenario that a fully correct model could fail is a bug in the scenario. An expectation must admit EVERY correct answer, not just the one the author had in mind.** @@ -23,6 +31,9 @@ Before submitting a scenario: - Never pin operand position for a commutative operation. - Never expect a value the model cannot know from the prompt or an earlier tool result. - Add a `rationale` that says what capability is asserted and why the expectation admits every correct answer. +- Set `response_requirement` on every turn. Use `tool_calls` when the turn expects calls, and `text_without_tool_calls` when it expects none. + +Save the scenario as `crates/wic-core/scenarios/.toml`. The file name must equal the `id` field. This example pins an opaque id and uses subset matching so an optional argument cannot cause a false failure: @@ -46,6 +57,8 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "tool_calls" + [[turns.messages]] role = "user" content = "Get record rec-17." # The expected value is literal. @@ -60,13 +73,14 @@ record_id = "rec-17" 1. Run the full scenario corpus against one loaded model at a time. 2. Add the exact model selector to `registry/models-v1.json`. Every claimed - identity field needs a provenance reference. If the provenance cannot be - recovered, an explicit unresolved entry is acceptable and will be displayed - as `unresolved`; do not infer identity from the selector or result filename. + identity field needs a provenance reference. If you cannot recover the + provenance, an explicit `unresolved` entry is acceptable. The result and the + site show that status. Do not infer identity from the selector or the result + filename. 3. Run `willitcall validate results/.json`; new runs must pass as schema v3 against `schemas/result-v3.schema.json`. The CLI continues to accept schema v1 and v2 files without upgrading them during `annotate` or `rescore`. -4. Open a pull request adding the file under `results/`, and state the hardware and server version used. +4. Open a pull request that adds the file and its `evidence/` directory under `results/`. State the hardware and the server version. Never hand-edit a result file. Each scenario record carries an evidence hash, so edited results are not comparable. @@ -78,4 +92,4 @@ Never hand-edit a result file. Each scenario record carries an evidence hash, so ### Failure classes -`failure_class` is a single mechanical observation assigned only after a scenario fails. Its precedence is error status, then `empty_response`, then `unparsed_tool_call`, then a plain failure with no class. `empty_response` means the response had neither content nor a parsed tool call. `unparsed_tool_call` means content matched a registered tool-call shape whose function was offered and whose arguments passed that tool's parameter schema, but the server produced no parsed tool call. A `cause` is a separate human attribution. +`failure_class` is a single mechanical observation assigned only after a scenario fails. A scenario with status `error` has no class. For a failure, `empty_response` takes precedence over `unparsed_tool_call`, and any other failure has no class. `empty_response` means the response had neither content nor a parsed tool call. `unparsed_tool_call` means the server parsed no tool call, but the content matches a registered tool-call shape. The function in that shape was offered, and its arguments pass the tool's parameter schema. A `cause` is a separate human attribution. diff --git a/README.md b/README.md index 149c3c9..ae3b998 100644 --- a/README.md +++ b/README.md @@ -3,19 +3,18 @@ A caniuse-style compatibility matrix for tool calling on local models. Every local inference stack claims OpenAI-compatible function calling. In -practice support varies by model, by quantization, by chat template, and by -server. `willitcall` is a small CLI that runs a fixed corpus of 50 tool-calling -scenarios against any OpenAI-compatible endpoint and emits a machine-readable -result file, so "does this model actually do parallel tool calls on llama.cpp" -becomes a fact you can look up instead of an afternoon you lose. +practice, support varies by model, quantization, chat template, and server. +`willitcall` is a small CLI. It runs a fixed corpus of 50 tool-calling +scenarios against any OpenAI-compatible endpoint. It writes a machine-readable +result file. You can then look up whether a model does parallel tool calls on +llama.cpp, instead of testing it yourself. Status: the CLI and corpus work. The public matrix is live at https://devyrpauli.github.io/willitcall/. -The full analysis of what these measurements found (the decoding mechanism -behind cross-server deltas, the quantization verdict, and the failure -taxonomy) is at -https://yashrajpandey.com/writing/same-weights-opposite-results/. +The write-up is at https://yashrajpandey.com/writing/same-weights-opposite-results/. +The replication runs behind the case studies are in +`docs/case-studies/evidence/replication/`. ## Quickstart @@ -46,9 +45,11 @@ cargo run -p willitcall -- run \ --out willitcall-result.json ``` -The run prints a per-scenario report and exits 0 if every scenario passed, 1 if -any scenario failed, and 3 if preflight failed (no result file is written in -that case). Add `--json` for pipe-clean output. +The run prints a per-scenario report. Add `--json` for pipe-clean output. The +exit code is `0` if every scenario passed and `1` if a scenario failed (a wrong +model answer). It is `2` for a usage or scenario configuration error. It is `3` +if preflight failed, and the run writes no result file. It is `4` for a harness +error, or if a scenario ended in `error` (a failed request). Check a result file against the published schema: @@ -56,164 +57,162 @@ Check a result file against the published schema: cargo run -p willitcall -- validate willitcall-result.json ``` -New runs use result schema v3 (`schemas/result-v3.schema.json`). The CLI resolves -the model selector through `registry/models-v1.json` and embeds the resolved -model and artifact metadata in the result. Legacy schema v1 and v2 files remain -valid inputs to `validate`, `site`, `annotate`, and `rescore`; editing a legacy -file preserves its original schema version. +New runs use result schema v3 (`schemas/result-v3.schema.json`). The CLI +resolves the model selector through `registry/models-v1.json` and embeds the +resolved model and artifact metadata in the result. Legacy schema v1 and v2 +files are still valid input to `validate`, `site`, `annotate`, and `rescore`. +Editing a legacy file keeps its original schema version. `--server` selects a preset (`llamacpp`, `ollama`, `mlx-lm`, `lmstudio`, -`vllm`, `custom`). The preset only supplies request defaults; the preset name is -recorded in the result file so results stay comparable. +`vllm`, `custom`). The preset only supplies request defaults. The result file +records the preset name, so results stay comparable. -The `mlx-lm` preset defaults to port 8081, not mlx-lm's own default of 8080, -because 8080 is this project's llama.cpp convention and two servers on one port -is exactly the contention the preflight exists to catch. +The `mlx-lm` preset defaults to port 8081, not mlx-lm's own default of 8080. +This project uses 8080 for llama.cpp. Two servers on one port cause the +contention that the preflight exists to catch. -Two things to know before reading or adding an mlx-lm row: +Two facts matter when you read or add an mlx-lm row: -- **MLX rows are converted weights.** MLX does not consume GGUF, so an MLX row - for a model is not the same bits as the llama.cpp row for that model. The - trick used elsewhere in this project of serving one blob through two servers - to hold the weights constant does not work across this boundary. An MLX-vs- - GGUF difference includes the conversion. +- **MLX rows use converted weights.** MLX does not read GGUF. An MLX row and the + llama.cpp row for the same model do not use the same bits. This project holds + weights constant by serving one blob through two servers. That fails across + this boundary. An MLX-versus-GGUF difference includes the conversion. - **`/v1/models` on mlx-lm lists the whole local cache, not the loaded model.** - llama.cpp reports the model it is serving; mlx-lm enumerates everything in the - HuggingFace cache, and it will load whichever model your request names. Do not - discover the model id from that endpoint - pass the repo id you intend to - measure. Getting this wrong files a row under the wrong model name, which is - worse than having no row. + llama.cpp reports the model it serves. mlx-lm lists everything in the + HuggingFace cache and loads whichever model your request names. Do not take + the model id from that endpoint. Pass the repo id you intend to measure. A + wrong id files the row under the wrong model name. That is worse than no row. ## What the scenarios test -The corpus is 50 scenarios in six categories. Every scenario is plain TOML data -in `crates/wic-core/scenarios/`; point `--scenarios ` at your own directory -to run a modified set. - -- `single` - one tool call with one argument shape: strings, integers, decimals, - booleans, enums, arrays, nested objects, empty arguments, optional arguments - present and omitted. -- `parallel` - several tool calls emitted in a single response. -- `streaming` - the same calls again over SSE, reassembled from deltas, to catch - servers that only break under streaming. -- `multi_turn` - a tool result is fed back and a follow-up call must use a value - that only exists in that result. +The corpus is 50 scenarios in six categories. Each is plain TOML data in +`crates/wic-core/scenarios/`. Use `--scenarios ` to run a modified set. + +- `single` - one tool call per scenario. Argument shapes: strings, integers, + decimals, booleans, enums, arrays, nested objects, empty arguments, and + optional arguments, present and omitted. +- `parallel` - several tool calls in a single response. +- `streaming` - the same calls over SSE, reassembled from deltas. This catches + servers that break only under streaming. +- `multi_turn` - the harness feeds a tool result back. The follow-up call must + use a value that exists only in that result. - `tool_choice` - `auto`, `none`, `required`, and a named function. -- `negative` - cases where the correct behavior is to NOT call a tool, plus - awkward argument content (a 256-character token, a non-ASCII city name). +- `negative` - cases where no tool call is correct, and awkward argument + content (a 256-character token, a non-ASCII city name). -Scoring is deterministic. There is no LLM judge, because failure reasons get -published and a published reason has to be defensible. +Scoring is deterministic. There is no LLM judge. The project publishes failure +reasons, and a published reason has to be defensible. -Read a red cell carefully: a red in `parallel` means the model did not emit -several tool calls in one response, which is the capability that column -measures. It does not mean the model is broken. +Read a red cell carefully. A red in `parallel` means the model did not emit +several tool calls in one response. It does not mean the model is broken. ## A cell is a property of the whole stack -**The servers do not decode the same way, so a green on one server and a red on -another is not by itself evidence about the model.** +**The servers do not decode the same way. A green on one server and a red on +another is not, by itself, evidence about the model.** llama.cpp compiles the tool definitions you send into a GBNF grammar and -constrains decoding with it. A tool call that names a function you did not -supply, or whose arguments do not fit the schema, is not merely unlikely there: -it cannot be sampled. Ollama and mlx-lm generate unconstrained text and parse a -tool call out of it afterwards, so the model can emit a wrong function name or a -malformed call, and the server finds out only after the fact. - -That difference is systematic and it favours llama.cpp in every row, on every -model. So: - -- A llama.cpp-versus-Ollama delta is a property of the stack. Read it as "this - combination works", not as "Ollama is defective" or "this model is worse than - that one". -- The comparison that isolates the model is same-server, not cross-server. -- A red under an unconstrained server can mean the model emitted something - nearly right that the parser then rejected. The `unparsed_tool_call` failure - class exists to mark exactly that case, and the transcript shows the bytes. - -V3 results can record which side of this line their server sits on in -`metadata.server.decode_mode`, while retaining any corresponding -`server.quirk_flags`: `grammar_constrained_decoding` for llama.cpp and +constrains decoding with it. There, a call to a function that you did not supply +cannot be sampled. Ollama 0.32.1 with its default Go template, and mlx-lm, +generate unconstrained text. They parse the tool call out of that text +afterwards. The model can emit a call to a function that you did not supply, or +a malformed call. The server finds out only after the fact. So: + +- A delta between llama.cpp and Ollama describes the stack. Read it as "this + combination works". Do not read it as "Ollama is defective" or "this model is + worse than that one". +- To isolate the model, compare results from the same server. +- A red can mean the server did not parse a valid call. The `unparsed_tool_call` + failure class marks that case, and the transcript shows the bytes. + +Ollama 0.34.0 can use the embedded Jinja template instead of the Go template. +Set `OLLAMA_GO_TEMPLATE=0` in the server environment. In one test on 0.34.0, +`qwen2.5:0.5b` returned an empty message with the Go template. It returned the +tool call with the Jinja template. See https://github.com/ollama/ollama/issues/17274. + +Two models have a row on each server with the same blob SHA-256: +`granite3.1-dense:8b` and `phi4-mini:latest`. Each scored 7 of 50 on Ollama and +7 of 50 on llama.cpp. + +Schema v3 results can record the decode mode of the server in +`metadata.server.decode_mode`. They can also keep the matching +`server.quirk_flags`: `grammar_constrained_decoding` for llama.cpp, and `unconstrained_post_hoc_parse` for Ollama and mlx-lm. When a historical run did -not record the mode, the site consults the cited preset mapping in -`registry/decode-modes-v1.json`; presets absent from that mapping remain -`unknown` rather than being guessed. +not record the mode, the site reads it from the cited preset mapping in +`registry/decode-modes-v1.json`. A preset that is not in the mapping stays +`unknown`, never guessed. -This was established the hard way. An earlier version of this project published -a claim that Ollama discarded valid tool calls. Recovering the discarded bytes -showed the model had emitted the tool's *description* where its name belonged, -and Ollama's parser was right to reject it. The claim was retracted. The real -finding is the mechanism above. +This project once claimed that Ollama discarded valid tool calls. Recovering the +discarded bytes showed that the model wrote the tool's *description* where its +name belonged. Ollama's parser was right to reject the call, and the project +retracted the claim. The real finding is the decode difference. ## The scenario-authoring rule **A scenario that a fully correct model could fail is a bug in the scenario.** A false red is worse for this project than a missing test. If the matrix says a -model fails and it does not, nobody trusts any other cell. So an expectation -must admit *every* correct answer, not just the one the author had in mind. +model fails and it does not, readers stop trusting the other cells. So an +expectation must admit *every* correct answer, not just the author's own. -Concretely, when writing or reviewing a scenario: +When you write or review a scenario: - Prefer structural checks over string equality. Assert that the call happened, that the arguments validate against the schema, that the argument set matches. - If a text value must match exactly, the prompt has to force it. Put the exact - literal in the user message, or use an opaque identifier (`doc-17`, `EXT-55`) - rather than something a model may legitimately expand or spell differently. + literal in the user message, or use an opaque identifier (`doc-17`, `EXT-55`). + A model may legitimately expand or spell other values differently. - Where the exact text does not matter, set `arguments_match = "ignore"` on that - expected call, or `"subset"` when extra arguments are acceptable. -- Arrays are compared positionally, so if order is not part of the requirement, - say "in that order" in the prompt or do not use an array. + expected call. Use `"subset"` when extra arguments are acceptable. +- Arrays are compared positionally. If the order does not matter, say "in that + order" in the prompt, or do not use an array. - Do not pin an operand position for a commutative operation. - Do not expect a value the model has no way to know. -Every scenario carries a `rationale` field stating why its expectation is -complete: what the scenario asserts, and why no correct model can fail it for -another reason. Pull requests that add a scenario without a rationale will be -asked for one. +Every scenario has a `rationale` field. It states what the scenario asserts and +why no correct model can fail it for another reason. A pull request without a +rationale gets a request for one. -Two real bugs of this class were found and fixed during development: a scenario -expected the ASCII spelling of an Icelandic city name when a correct model -writes it with a diacritic, and a scenario expected `place = "Fenway Park"` when -a correct model geocodes `"Fenway Park, Boston, MA"`. +Two real bugs of this class were fixed. In `negative-unicode-argument`, a model +wrote a city name with an umlaut in decomposed Unicode (NFD), and the scorer +compared bytes. It now compares strings in NFC. Another scenario expected +`place = "Fenway Park"`, but a correct model geocodes `"Fenway Park, Boston, MA"`. ## Submitting a result -The matrix is fed by result files in `results/`, each shipping the transcripts -that back it. Seed results so far cover `qwen3` at 0.6b/1.7b/4b/8b, -`llama3.1:8b` and `qwen2.5:7b-instruct` on Ollama, plus -`Qwen2.5-1.5B-Instruct-Q4_K_M` on llama.cpp, all from a 16GB M-series MacBook. - -1. Run the full corpus against your endpoint, one model loaded at a time. - Running two models at once produces spurious `error` outcomes from resource - contention, not real measurements. -2. Ensure the exact model selector has an entry in `registry/models-v1.json`, - with evidence references for every claimed identity field. An unresolved - entry is acceptable when provenance cannot be recovered; the result and site - show that status as `unresolved` rather than inferring identity from a name. -3. Run `willitcall validate` on the schema v3 output. Legacy schema v1 and v2 - files remain accepted, but must not be upgraded by editing them. -4. Open a pull request adding the result file **and its `evidence/` directory** - under `results/`, and say what hardware and server version produced it. - -Every scenario writes a full request/response transcript to -`evidence//.json`, referenced from the result file, so any -red cell can be inspected rather than taken on trust. Credential-bearing -headers and URL query parameters are redacted at capture time, but read a -transcript before publishing it if your endpoint is not local. +The matrix is fed by result files in `results/`. All matrix rows so far were +measured on one Apple M4 Max 64GB host. The live matrix at +https://devyrpauli.github.io/willitcall/ lists them. + +1. Run the full corpus against your endpoint, one model loaded at a time. Two + models at once cause spurious `error` outcomes from resource contention, not + real measurements. Keep the script, the log, and the output of each batch + together. Use one dated folder outside the repo, for example + `~/willitcall-runs/2026-10-05-short-name/`. +2. Make sure `registry/models-v1.json` has an entry for the exact model + selector, with an evidence reference for each claimed identity field. If you + cannot recover the provenance, an `unresolved` entry is acceptable. The + result and the site show that status. They do not infer identity from a name. +3. Run `willitcall validate` on the schema v3 output. The CLI still accepts + legacy schema v1 and v2 files. Do not upgrade them by editing. +4. Open a pull request that adds the result file **and its `evidence/` + directory** under `results/`. State the hardware and server version. + +Every scenario writes a full request and response transcript to +`evidence//.json`. The result file references it, so you +can inspect any red cell. The run redacts credential-bearing headers and URL +query parameters at capture time. If your endpoint is not local, read each +transcript before you publish it. Do not hand-edit a result file or a transcript. `evidence_hash` is the SHA-256 -of the transcript bytes, so edits are detectable and edited results are not +of the transcript bytes, so you can detect an edit. An edited result is not comparable with anything else in the matrix. ## Roadmap -- The static matrix site (GitHub Pages), fed by `results/` and rebuilt on merge. -- Seed results across popular models and quantizations on llama.cpp and Ollama. -- Template forensics: read the chat template out of GGUF metadata and lint it - against known tool-calling breakage patterns, so a red cell can say "fails - parallel calls: template drops the call id" instead of just "fails". +Next: template forensics. Read the chat template out of GGUF metadata, and lint +it against known tool-calling breakage patterns. A red cell can then say "fails +parallel calls: template drops the call id" instead of just "fails". Deferred: harness-in-the-loop testing against real agent CLIs, automatic template repair, and LLM-judged semantic scoring. diff --git a/crates/wic-core/src/client.rs b/crates/wic-core/src/client.rs index 0f84c40..d94a948 100644 --- a/crates/wic-core/src/client.rs +++ b/crates/wic-core/src/client.rs @@ -46,8 +46,8 @@ pub enum CompletionResult { } enum RecordedAttemptError { - Timeout(CapturedTurn), - Transport(reqwest::Error, CapturedTurn), + Timeout(Box), + Transport(reqwest::Error, Box), } #[derive(Clone)] @@ -185,7 +185,7 @@ impl EndpointClient { .await { Err(RecordedAttemptError::Timeout(turn)) => { - turns.push(turn); + turns.push(*turn); if attempt == 0 { continue; } @@ -200,11 +200,11 @@ impl EndpointClient { }; } Err(RecordedAttemptError::Transport(_, turn)) if attempt == 0 => { - turns.push(turn); + turns.push(*turn); continue; } Err(RecordedAttemptError::Transport(error, turn)) => { - turns.push(turn); + turns.push(*turn); return CompletionResult::Error { reason: format!("transport error after retry: {error}"), raw_bytes: retry_raw, @@ -294,20 +294,20 @@ impl EndpointClient { let mut response = match tokio::time::timeout_at(deadline, self.http.execute(request)).await { Err(_) => { - return Err(RecordedAttemptError::Timeout(CapturedTurn { + return Err(RecordedAttemptError::Timeout(Box::new(CapturedTurn { request: captured_request, response: None, retried, - })); + }))); } Ok(Err(error)) => { return Err(RecordedAttemptError::Transport( error, - CapturedTurn { + Box::new(CapturedTurn { request: captured_request, response: None, retried, - }, + }), )); } Ok(Ok(response)) => response, @@ -318,7 +318,7 @@ impl EndpointClient { loop { match tokio::time::timeout_at(deadline, response.chunk()).await { Err(_) => { - return Err(RecordedAttemptError::Timeout(CapturedTurn { + return Err(RecordedAttemptError::Timeout(Box::new(CapturedTurn { request: captured_request, response: Some(CapturedResponse { status: status.as_u16(), @@ -326,14 +326,14 @@ impl EndpointClient { body, }), retried, - })); + }))); } Ok(Ok(Some(chunk))) => body.extend_from_slice(&chunk), Ok(Ok(None)) => break, Ok(Err(error)) => { return Err(RecordedAttemptError::Transport( error, - CapturedTurn { + Box::new(CapturedTurn { request: captured_request, response: Some(CapturedResponse { status: status.as_u16(), @@ -341,7 +341,7 @@ impl EndpointClient { body, }), retried, - }, + }), )); } } diff --git a/crates/willitcall/src/site/html.rs b/crates/willitcall/src/site/html.rs index d981c5b..59aa1f0 100644 --- a/crates/willitcall/src/site/html.rs +++ b/crates/willitcall/src/site/html.rs @@ -588,7 +588,7 @@ fn render_methodology(

Each cell measures one full stack. The stack combines a model, quant, server, and server version. A cell does not describe the model alone.

A failed observation applies only to the tested combination. It does not mean the weights are bad. The same weights can pass on one server and fail on another. When the evidence proves that difference, the cell includes a cause annotation.

When the result includes a transcript path, a failing observation links to the full request and response transcript. Legacy schema v1 results do not record transcript paths. Read the case studies in docs/case-studies/ for controlled comparisons.

-

The servers use different decode methods. llama.cpp compiles the supplied tool definitions into a GBNF grammar. It uses that grammar to constrain decoding. Ollama and MLX LM generate unconstrained text. They parse the tool call after decoding. Cross-band differences therefore reflect the full stack. Compare adjacent models only when they use the same server. Each row states whether the run recorded its decode mode or the site read the mode from the cited preset mapping. An unmapped preset has an unknown mode.

+

The servers use different decode methods. llama.cpp compiles the supplied tool definitions into a GBNF grammar. It uses that grammar to constrain decoding. MLX LM, and Ollama with a Go template, generate unconstrained text. They parse the tool call after decoding. Cross-band differences therefore reflect the full stack. Compare adjacent models only when they use the same server. Each row states whether the run recorded its decode mode or the site read the mode from the cited preset mapping. An unmapped preset has an unknown mode.

The site includes {} distinct scenarios. Each published cell represents one run. Hatching marks that the cell has no verdict.

The case studies draw a verdict only after at least five runs per arm. The current case studies cover {}.

Excluded rows

diff --git a/crates/willitcall/tests/golden/site-contract-index.html b/crates/willitcall/tests/golden/site-contract-index.html index dc747f8..5504432 100644 --- a/crates/willitcall/tests/golden/site-contract-index.html +++ b/crates/willitcall/tests/golden/site-contract-index.html @@ -176,7 +176,7 @@

Method and limitations

Each cell measures one full stack. The stack combines a model, quant, server, and server version. A cell does not describe the model alone.

A failed observation applies only to the tested combination. It does not mean the weights are bad. The same weights can pass on one server and fail on another. When the evidence proves that difference, the cell includes a cause annotation.

When the result includes a transcript path, a failing observation links to the full request and response transcript. Legacy schema v1 results do not record transcript paths. Read the case studies in docs/case-studies/ for controlled comparisons.

-

The servers use different decode methods. llama.cpp compiles the supplied tool definitions into a GBNF grammar. It uses that grammar to constrain decoding. Ollama and MLX LM generate unconstrained text. They parse the tool call after decoding. Cross-band differences therefore reflect the full stack. Compare adjacent models only when they use the same server. Each row states whether the run recorded its decode mode or the site read the mode from the cited preset mapping. An unmapped preset has an unknown mode.

+

The servers use different decode methods. llama.cpp compiles the supplied tool definitions into a GBNF grammar. It uses that grammar to constrain decoding. MLX LM, and Ollama with a Go template, generate unconstrained text. They parse the tool call after decoding. Cross-band differences therefore reflect the full stack. Compare adjacent models only when they use the same server. Each row states whether the run recorded its decode mode or the site read the mode from the cited preset mapping. An unmapped preset has an unknown mode.

The site includes 9 distinct scenarios. Each published cell represents one run. Hatching marks that the cell has no verdict.

The case studies draw a verdict only after at least five runs per arm. The current case studies cover 90 runs across 18 quantization arms, and 40 runs across 8 arms for the peg-native anomaly.

Excluded rows

diff --git a/docs/case-studies/2026-07-19-ollama-drops-valid-tool-calls.md b/docs/case-studies/2026-07-19-ollama-drops-valid-tool-calls.md index 9ac185c..bdab990 100644 --- a/docs/case-studies/2026-07-19-ollama-drops-valid-tool-calls.md +++ b/docs/case-studies/2026-07-19-ollama-drops-valid-tool-calls.md @@ -258,9 +258,9 @@ The dedicated verify-dupes pass ran on 2026-07-20 and cleared all three gates: so the issue cites it directly. No merged-but-unreleased fix touches the qwen2.5 path. 3. **Independent re-verification.** Every row of the table above was re-run from - scratch on macstudio without trusting the M3 session. All confirmed, no - discrepancies; the llama.cpp side came back 12/12 across both tool - descriptions. + scratch on the measurement host without trusting the M3 session. All + confirmed, no discrepancies; the llama.cpp side came back 12/12 across both + tool descriptions. The `eval_count` contradiction was folded into the same issue rather than filed separately: it cannot occur independently of the drop. diff --git a/docs/case-studies/2026-07-20-granite-emits-tool-calls-no-server-parses.md b/docs/case-studies/2026-07-20-granite-emits-tool-calls-no-server-parses.md index 8d49895..f2303f5 100644 --- a/docs/case-studies/2026-07-20-granite-emits-tool-calls-no-server-parses.md +++ b/docs/case-studies/2026-07-20-granite-emits-tool-calls-no-server-parses.md @@ -9,10 +9,8 @@ well-formed calls, with the right function name and the right arguments, in a is left sitting in `content`, so the harness sees no tool call and the scenario fails. -The seven scenarios it passes are exactly the seven negative traps, where -emitting no parsed call is the correct answer. That is the tell: the model -scores full marks precisely where not calling a tool is right, and zero -elsewhere. +The model passes exactly the seven scenarios where making no call is correct. +It fails the other 43. That is the tell. This is not the M3 finding. M3 was read as Ollama discarding a valid call from its own engine; that reading was disproved on 2026-07-21 (the model had put @@ -120,7 +118,7 @@ That is a model that did not call a tool, not a call that went unparsed. phi4-mini's 7/50 is largely genuine failure and it should not appear in this finding. -The original claim was generalised from a single example, which is the +The original claim was generalized from a single example, which is the failure mode amendment 4 exists to prevent, and the second time this project has had to overturn an n=1 conclusion. The lesson is recorded here rather than quietly dropped, because a benchmark that hides its own corrections is @@ -151,13 +149,15 @@ willitcall run --endpoint http://127.0.0.1:8080/v1 \ --model --server llamacpp --out granite-llamacpp.json ``` -Both should report 7 passed, 43 failed, and the passing seven should be the -negative traps. +Both should report 7 passed and 43 failed. The passes should be exactly the +seven scenarios where making no call is correct. ## Evidence files - `evidence/granite-ollama-run1.json` - full result, Ollama arm, run 1 - `evidence/granite-llamacpp-run1.json` - full result, llama.cpp arm, run 1 +- `evidence/replication/m5-replication/` - full results, all 10 replication runs Each result's `evidence_path` entries point at the per-scenario transcripts -containing the raw request and response bodies quoted above. +containing the raw request and response bodies quoted above. The transcripts +are not published. diff --git a/docs/case-studies/2026-07-21-llamacpp-500s-on-llama-3.1-tool-calls.md b/docs/case-studies/2026-07-21-llamacpp-500s-on-llama-3.1-tool-calls.md index 45986eb..77abc56 100644 --- a/docs/case-studies/2026-07-21-llamacpp-500s-on-llama-3.1-tool-calls.md +++ b/docs/case-studies/2026-07-21-llamacpp-500s-on-llama-3.1-tool-calls.md @@ -85,8 +85,10 @@ The anomaly is therefore reproducible and not fixed between b10050 and b10075. - Observed on one model family. Qwen2.5-7B and Qwen2.5-1.5B on the same server, version, flags and corpus never trigger it. -- Affected scenarios skew to `multi_turn` (6 of the 8-9 errors per run), with - the remainder in negative-trap and parallel-call scenarios. +- Affected scenarios skew to `multi_turn`. In the greedy runs, 6 of the 8 or 9 + errors are `multi_turn` at Q8_0 and Q4_K_M, and 4 of the 7 at Q3_K_M. The + rest are negative-trap and parallel-call scenarios, plus one single-call + scenario at Q3_K_M. - This does not say Llama-3.1 cannot call tools. It says this combination cannot, through this server's parser. Llama-3.1 on Ollama is a separate published row and does not fail this way. @@ -123,6 +125,10 @@ captured in the evidence for these runs. - Deterministic under greedy decoding, and the errors persist in the seed-varied arm at temperature 0.7, so this is not a sampling artifact. +The b10050 run files are in `evidence/replication/m6-armA/`. The b10075 run +files are in `evidence/replication/m6-armB/`. The seed-varied arm is in +`evidence/replication/m6-armS/`. + ## Environment - Host: Apple M4 Max, 64GB, macOS 26.5.2 (single host for every run) diff --git a/docs/case-studies/2026-07-21-mlx-8bit-emits-doubled-braces.md b/docs/case-studies/2026-07-21-mlx-8bit-emits-doubled-braces.md index 790f201..7644bdc 100644 --- a/docs/case-studies/2026-07-21-mlx-8bit-emits-doubled-braces.md +++ b/docs/case-studies/2026-07-21-mlx-8bit-emits-doubled-braces.md @@ -45,6 +45,7 @@ output is wrong. - 5 runs per arm, greedy. `Qwen2.5-7B-Instruct-4bit` scored 46, 46, 46, 46, 46. `Qwen2.5-7B-Instruct-8bit` scored 7, 7, 7, 7, 7, with 30 scenarios classified `empty_response` on every run. +- The run files for these 10 runs are in `evidence/replication/m6-mlx-repl/`. - mlx-lm logged `Failed to parse tool call (JSONDecodeError: Expecting property name enclosed in double quotes: line 1 column 2 (char 1))` 295 times across the 8bit runs, and 0 times across the 4bit runs. Column 2, character 1 is @@ -58,10 +59,8 @@ output is wrong. mlx-lm swallows: it applies the same chat template with the same tools and generates greedily with no parsing. Switching `REPO` to the 4bit conversion gives the control. -- The 7 scenarios the 8bit arm passes are the negative traps, where emitting no - parsed call is the correct answer. That signature - full marks precisely where - not calling a tool is right - is the same one that identified the granite - case. +- The 8bit arm passes exactly the seven scenarios where making no call is + correct. That signature is the same one that identified the granite case. ## Why this matters for the quantization question diff --git a/docs/case-studies/2026-07-21-quantization-does-not-order-tool-calling.md b/docs/case-studies/2026-07-21-quantization-does-not-order-tool-calling.md index 5be6db4..8ba844f 100644 --- a/docs/case-studies/2026-07-21-quantization-does-not-order-tool-calling.md +++ b/docs/case-studies/2026-07-21-quantization-does-not-order-tool-calling.md @@ -122,6 +122,9 @@ This is deliberately narrow. five runs per arm, and the amendment 7 requirement that a difference-claim be supported by a seed-varied arm. +The greedy run files are in `evidence/replication/m6-armA/`. The seed-varied +run files are in `evidence/replication/m6-armS/`. + ## Environment - Host: Apple M4 Max, 64GB, macOS 26.5.2 (single host for every run) diff --git a/docs/case-studies/evidence/replication/README.md b/docs/case-studies/evidence/replication/README.md new file mode 100644 index 0000000..5c92cdc --- /dev/null +++ b/docs/case-studies/evidence/replication/README.md @@ -0,0 +1,35 @@ +# Replication runs + +These are the runs behind the replication claims in the case studies. Each +JSON file is one willitcall run of 50 scenarios, as the CLI wrote it, in +result schema version 2. They are not rows of the matrix. The rows in +`results/` are separate single runs in schema version 3. + +All runs used one Apple M4 Max 64GB host on macOS 26.5.2. + +| folder | runs | server | case study | +|---|---|---|---| +| `m5-replication/` | granite3.1-dense 8B, 5 runs per server, greedy | Ollama 0.32.1 and llama.cpp b10050 | granite | +| `m6-armA/` | 9 quant arms x 5 runs, greedy | llama.cpp b10050 | quantization, llama.cpp 500s | +| `m6-armS/` | the same 9 arms x 5 runs, temperature 0.7, seeds 1 to 5 | llama.cpp b10050 | quantization, llama.cpp 500s | +| `m6-armB/` | Llama-3.1-8B at Q3_K_M, Q4_K_M and Q8_0, and Qwen2.5-7B at Q4_K_M, 5 runs each, greedy | llama.cpp b10075 | llama.cpp 500s | +| `m6-mlx-repl/` | Qwen2.5-7B 4-bit and 8-bit, 5 runs each, greedy | mlx-lm 0.31.3 | mlx 8-bit | + +Greedy means temperature 0, top_p 1.0 and seed 42. The 9 quant arms are +Llama-3.1-8B, Qwen2.5-1.5B and Qwen2.5-7B, each at Q3_K_M, Q4_K_M and Q8_0. +The `*-version.txt` files hold the server version output for each folder. + +## What is not here + +The transcripts are not published. There is one per scenario, 6,500 in all. +Each scenario in a run file records `evidence_hash`, the SHA-256 of its +transcript file, so a transcript can be checked against its run. + +## One change from the raw output + +In the five `m5-replication/llamacpp-granite-run*.json` files, +`metadata.model_id` held the absolute path of the Ollama blob on the +measurement host. It now holds the blob name, `sha256-`, as in +`docs/migrations/2026-08-05-local-path-redaction.md`. No other byte changed. +The two run 1 files are also at `../granite-ollama-run1.json` and +`../granite-llamacpp-run1.json`. diff --git a/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run1.json b/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run1.json new file mode 100644 index 0000000..f187fa8 --- /dev/null +++ b/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run1.json @@ -0,0 +1,527 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260720T192149Z-7af7401b", + "timestamp": "2026-07-20T19:21:49Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "sha256-44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8", + "declared_quant": null, + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6326968fd1ce764dded954a65566586e2073f14872cf3a6aa47117f6241b5316", + "evidence_path": "evidence/20260720T192149Z-7af7401b/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f799a6d2f4e6494fa5ffc7f37b2e3e8ec192609a8be61d896c2c1c335b4fa00f", + "evidence_path": "evidence/20260720T192149Z-7af7401b/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:8827fddeca2c93f0f80aa197d07616385237e7cdeb54147a85fbd5ae9536c76b", + "evidence_path": "evidence/20260720T192149Z-7af7401b/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6f3e6f12f5d4fdfa3b57ae4421af15c3f2fceafd712e687f09742556e264957b", + "evidence_path": "evidence/20260720T192149Z-7af7401b/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:d0dd9c64af907811b4aaff7542e223b78abbcfdc2583873bbd201f6a5a85655e", + "evidence_path": "evidence/20260720T192149Z-7af7401b/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3ffbce3f50b7e07dac0f12e3bc32909bdbd3a8357792423fc68152813acbd34c", + "evidence_path": "evidence/20260720T192149Z-7af7401b/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:302227e465494672584debd482caf0a46bf0c588ece0a5de40c8f8327abd8eec", + "evidence_path": "evidence/20260720T192149Z-7af7401b/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:96025bffae69c34390587b7895ab0f992fefa7554a3b57ae8135978c7d42f909", + "evidence_path": "evidence/20260720T192149Z-7af7401b/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:00f45c738969ef910f642985457b0dae987808285678db111b3903136836ac45", + "evidence_path": "evidence/20260720T192149Z-7af7401b/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0a70905f925ebe10af0565fe0eb6dccfc7051434574d77c3840ab8c995b9a683", + "evidence_path": "evidence/20260720T192149Z-7af7401b/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0851a336294ce023bc38ab5af720a09594ce0fb83ffdd678eda4c12a8146649f", + "evidence_path": "evidence/20260720T192149Z-7af7401b/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3c5e2f5c6782f28fa025b06e3aa5d42413387676885861b3cb128376c8b81bcf", + "evidence_path": "evidence/20260720T192149Z-7af7401b/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:37ef433c92978eb7500f05d90eb93371f76f80b16a87cf1667914a51fa8fa5a4", + "evidence_path": "evidence/20260720T192149Z-7af7401b/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:c4fd670db1fbdb09b2cdea4ec52b8a0b7a0738fe072ba7f08767ea1e863e8cba", + "evidence_path": "evidence/20260720T192149Z-7af7401b/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:2a70d331839f14af5b1e7e3e7b7d705f5f0eeb8427c15038e58f0b1259033bfa", + "evidence_path": "evidence/20260720T192149Z-7af7401b/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:0456a6ed15188b12c09714950c0704f4c72049124b2a02705bc09b982a16ecf8", + "evidence_path": "evidence/20260720T192149Z-7af7401b/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:4da4624edca8afb2c6785bd0bc5677984fdce75b2127c7ceb594f80d54acec70", + "evidence_path": "evidence/20260720T192149Z-7af7401b/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:23d63036f34c9db45b3aa7d3158dcd28efe65407a28705543de3497425ce87c2", + "evidence_path": "evidence/20260720T192149Z-7af7401b/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5a26e0ed967564ac37b577f1a65cac0a0555e7ea24869ab22565b27d59413db6", + "evidence_path": "evidence/20260720T192149Z-7af7401b/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:a49e83c55f451487431d1432da90530084941bca53e5c065104e68f291c46693", + "evidence_path": "evidence/20260720T192149Z-7af7401b/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:805ebb69b9d418e2ef1532383e8ebfb7598fb776a27df475aa462e2c864cc839", + "evidence_path": "evidence/20260720T192149Z-7af7401b/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:390ab8acc9694d46bb71002a3229372e72a2051f8a615f336d5bf8b23ccb5d8e", + "evidence_path": "evidence/20260720T192149Z-7af7401b/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:57cba8ef9ce70aa0ad1a3ab02d1614b712b998ed8a1b5dded9abeaa1b93471b9", + "evidence_path": "evidence/20260720T192149Z-7af7401b/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5b8d6263902ad04ffe8060f4ced8fb99b57e7ccacc3ec2c7e9bc48f3821aa364", + "evidence_path": "evidence/20260720T192149Z-7af7401b/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:db8120b38898001421fbd5aa43f5101add762e100edcc0667544f989d8f15f8e", + "evidence_path": "evidence/20260720T192149Z-7af7401b/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:7cc40f0641a30938e9a0e2dfd19868bef04af48dc03e6440a5912501ef5e1b02", + "evidence_path": "evidence/20260720T192149Z-7af7401b/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:b4f368d684b89f7140f2ee744edc4f800575c536d149f25003968bd88f0c22d4", + "evidence_path": "evidence/20260720T192149Z-7af7401b/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:d20e543031be32230b16b709a3a1309da1bd4cb533535b3abb41122c61251b90", + "evidence_path": "evidence/20260720T192149Z-7af7401b/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:ad170a7453ac7087feb8418ebad168956b1caf3caf4c3368afac17583630ca9e", + "evidence_path": "evidence/20260720T192149Z-7af7401b/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:60f695bcb7f83624e5ffc33dae9cfdf2bd3aeaf7ccaa3c8442045c25306a8461", + "evidence_path": "evidence/20260720T192149Z-7af7401b/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3909deeafb7ab2c134a675b6c4c352889f74dfaab2000b611866f02951a798db", + "evidence_path": "evidence/20260720T192149Z-7af7401b/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3e12052a8b529ecb0cb52911d1d51be0f4e64126dd4c676e5e219de5c4592faa", + "evidence_path": "evidence/20260720T192149Z-7af7401b/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:4dd45198252c64a961a3a719810aa346c5435da77b2003275c11633d3a17344e", + "evidence_path": "evidence/20260720T192149Z-7af7401b/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:477065d7391189b34cc6f0f05bdac9ffef2d56a6e43e87381be0eca650b1f703", + "evidence_path": "evidence/20260720T192149Z-7af7401b/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6c763e8f04652036eefe159ffde12ee93a1d95893969e7fab7ffb69115e0147c", + "evidence_path": "evidence/20260720T192149Z-7af7401b/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:71ecf5e7d2e323f8f490410deff4d9b6d807f5e6d0fd624a14ad930b4dbd9f83", + "evidence_path": "evidence/20260720T192149Z-7af7401b/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:352c80eed5b0438cba16b90bb98afe6748319536e655eca3f8d62ea16bc20213", + "evidence_path": "evidence/20260720T192149Z-7af7401b/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:4be92e5b8273c2cfbe391d89115c8871f7a9ecb8d20335c6002bc3b31b0f71c7", + "evidence_path": "evidence/20260720T192149Z-7af7401b/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:db61b920690b78b1edcfbabfe9d4ef76b1e04f298b113c5fd47de9f650f26dec", + "evidence_path": "evidence/20260720T192149Z-7af7401b/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:865107a2d3bb8c81d10129ea28b334c3e9183f140d499ca7decab200be2fe9ef", + "evidence_path": "evidence/20260720T192149Z-7af7401b/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:eb7ff97360809356af66f38f2013f11eb4e165bbe980bfa5c251bed8227e1368", + "evidence_path": "evidence/20260720T192149Z-7af7401b/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:2b06f42c7bc7c4977eab3c3fd0cec9a82da4ff32eb20350d0d96bb0471fe15c2", + "evidence_path": "evidence/20260720T192149Z-7af7401b/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:ae94fd8f083a49544cef598d2f99cd5687d278564af35db5994d0cbc0f853027", + "evidence_path": "evidence/20260720T192149Z-7af7401b/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:cda71fbd8c356ff21d75eb9a6094aa529ed8c027cfec7e6b5dd96905c33ccddd", + "evidence_path": "evidence/20260720T192149Z-7af7401b/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ff746adcb07e7630274e2f7059fe98db71b94d2a7422e2f449b3cbfb9f229fc6", + "evidence_path": "evidence/20260720T192149Z-7af7401b/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:cb5db4f3a8dfb458984066583025b5b38cd3f1e41c7235e1413fb95c9ae3495a", + "evidence_path": "evidence/20260720T192149Z-7af7401b/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:de9afb2f0c628928da3d9767d56d122337097cb4eaaed23e85171a179c2f8b81", + "evidence_path": "evidence/20260720T192149Z-7af7401b/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:78cb57b976412df83917c4a18a6f73adac8945338505a3d0f6ee1371293f11e3", + "evidence_path": "evidence/20260720T192149Z-7af7401b/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3edbd6f74c6857355afcda687d9643abdd809359d53cc34fbe455fc0187a8a89", + "evidence_path": "evidence/20260720T192149Z-7af7401b/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e57321af458d86ff0f844701b6150850e839c6928d2c50e04a3c12b1525d010d", + "evidence_path": "evidence/20260720T192149Z-7af7401b/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run2.json b/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run2.json new file mode 100644 index 0000000..a6c37fa --- /dev/null +++ b/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run2.json @@ -0,0 +1,527 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260720T192224Z-c2878fe4", + "timestamp": "2026-07-20T19:22:24Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "sha256-44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8", + "declared_quant": null, + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:d98fdbfa2bf3d1a92dcbea4bfe9f30e2e5877181c9df917899275179d76bc3c7", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:ef4bcbf09ef0f6ee2ebd485d021a82e92aa3bee8590987401b4223ac9d102c1c", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3cc6a888449a32854681fa73837197c648a8b760c818fb7d4d98d880376376e4", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f9f07b685deadb4d2f2c8022594d9c797bc07d8c95c5130aaf543e15dfec0db5", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f30d81c9eec339047abb3906a5e51d3d3f8994e602b281dc0156e14d63644306", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:75f1bb11cc3eed68f3738c12411ab9c81e20c01a146ae9814c52414434e94e37", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:955dec6ce8a20888794df1779e2e3dd749628ad3009559186e45156872a66554", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:37dda545b49e77942d8724fc221ac4401a3cb08cb6c1911b12c0d3ca507f22ea", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e5c619f2204fdcd34877dd2b7fbffa61e77b5c410da25bab3d01a0c194484570", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a63b01931607325a14be05992cbd3356289724f7ad045c1e7ae9ffa47d574b53", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:30e11e273bef834abcb688abef11d1695a02b27e3a66d0acddf8d982dae7ac92", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e14f3231e86dbfbc61592f808b1c32361946d7dc4b1346f576436f6e2961aa0c", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bf97124d7c952da3882549e6ef5625d8eda05183db75bc560b4695a019ded05c", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:a511a19b31598bc8beab8ad453778b7f13044e6f0e1ddcd32f6f769a47a88e50", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:235169a2df30990e2663e0a953193dff79fc9960d729bad26fda246ac113c6a7", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:603e6d47572090ad9b2c5aa6bb5db64fd550fe2f799d32d2783fe6e26f852fcc", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:8cd8823eda0a8a236ecaea66594822b967443928afab1458468153a15f59417b", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:d68c68762082f3db498389d00f25030eab7845ffe5773579eb7c24b72a45876c", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:934e826f7febef3eb5797ac83aea7a337dbefde63d6f97f17fb41731cdc6a364", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:ad4bef5f571b6f03a02f1b57395d47112371cc2c6fc7bcd400dd01129ae5f08c", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e674170cbefe5ce80da3f11833d0f298ec03fdf84feb2b97803b731d09e7bf62", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:8da173da266500e6fe17a58aec7a9fc1fce3557aa3017f8296735b174434a7aa", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f7e49f11a787f8f75d3daedb7ca6962ad8065f1b974e70671543af267111cacb", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:21ded041cad35c15a4d0f15e931ba94b8d78de6f0a17e2767e3849a8a8c1ecce", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:0ea911ce5b81af3bb313f81677e5361ab82c94debe71cc78ea2c4b26b9648965", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:ba107f2d355efb7cd8a15329d36a27160e0331bb4b38dbd529f75f6ac96da1e9", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:904a6c79e200b01c5200e373fa57b79f413e9d8e68c02752293f1d33143c2de4", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:627b1e1e30341976e750fa136cf725c849ffcdb6f00134c1249f2ffe4af5aab3", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:330cc82306cc3b9ba8becdd35cb7e990b1c3b0d5c9ba6566aa6d82278782a22b", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:84c3499e6f47f6175bf3787144a9fb8745551f26068a4cb519a7919eeb7aa205", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e18931c94857d3355251fb509142ce85040af716d9a99dce91c7404e3276829f", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:b4df773adc989361093e40998a0dc5c29160e45e3aa6a29a4cde11c949b503a5", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:0b3b4996d9bb8d616ed83eae82f566757b6a9a4c64585a1c2c5b2a2512e52807", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6ab045867b4b51e49c0a4d9e7371931b474535a4d4656494db080302cefe8ae1", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:86ff88cf8c2fb2b05dd48b60868aba78ea38d15d9808bec68ee61515c4ecd255", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:0ad5748df20616d36ef64d6bb9e49dcc8f1d458908695a6faa2be6983a4a46b0", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:22d1fcfc558c8757752472054ff3bfa8c3de33a243ff8111921a27a15000864b", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5c4d20ab770284e9e851039b5a57b70cb5c180c0d1d2b4648c13c0ec3ee83d63", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6dfb2bda9bfa22049fbc0340d3961ea1bc65e69dbae7cddf7706c780a02a76c6", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:1e8e66d5ce40890a0843fa3f1490c06d0ebc9580b78037a380f2a7be8c3d5685", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:05af23ced2ecee181d624ca8847a11539ea1de4d07ed12666ed8ea45b942dace", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5c12216dac6cc303bd7009448ccd7b3e891d2143ff26e8ba0ef1ba1bc0026be7", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5786589deb5c9bafe86cecbd5b48e598e84159a885ad42cbc285a4ccb1ac5e96", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:4fd63078aeadac4247acf5d9f3adc289c5fee89e5d21dbd2ccbf462017dc90ae", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb5d9f6f1f1889abc915d69b01e760ece8b074a6286b0b477cac71b289f784b5", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:b6d7550ef02154aba63f23b7d7c8348fc5081fc75ea71009fe1a476a67adee23", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f5014c88af0fe4e365f3cf5f92b92e3098940991893dd2f6c5ae98416899704e", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:97928a1e3036a0a5a50af3b3fc76a2d871a3f73d3570ac3b9cdf073b8b7ec6f0", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3d874f68b989a52feda1baa81fc6d100ec86071fdd0c117c8112fab66aa92178", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:b54b5440845797845e81e0c3c065f0d0a63b75f9709db98c0cc3ff53a74f5763", + "evidence_path": "evidence/20260720T192224Z-c2878fe4/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run3.json b/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run3.json new file mode 100644 index 0000000..2103b38 --- /dev/null +++ b/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run3.json @@ -0,0 +1,527 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260720T192255Z-558dfb48", + "timestamp": "2026-07-20T19:22:55Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "sha256-44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8", + "declared_quant": null, + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:7d8916efd2b3d927592aea92c72241bbe54386e2dbfb23019f3357254d2c998d", + "evidence_path": "evidence/20260720T192255Z-558dfb48/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:4d3d951ebfad4ad6f26388e7279080f80905d17897ddf6521d9ac56f0e13dd93", + "evidence_path": "evidence/20260720T192255Z-558dfb48/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:55db22afa7107e9bb34bab40d2197f1f7d8ddf1e167eeaa9bc2152374ccfe8a9", + "evidence_path": "evidence/20260720T192255Z-558dfb48/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:b7b34e3da3482ca16961523ed70ce8b367f4da50ceaebf0294826a047a91d8e3", + "evidence_path": "evidence/20260720T192255Z-558dfb48/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:cddbeb53980dc420818275090867c0f74593eda84f7e31a924d1b6334bd145c8", + "evidence_path": "evidence/20260720T192255Z-558dfb48/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e08d5712835362e9f6ccded751ae5528aba2b6a909ea78c2e00dffa46fc0396d", + "evidence_path": "evidence/20260720T192255Z-558dfb48/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:68b6958b0a5a479b817aaec8085bb26108211c41c0d346967cc39ac04b35fb9b", + "evidence_path": "evidence/20260720T192255Z-558dfb48/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:98ef68188ffb680808d073623ddf52cb797389594e3377350f4c5067fbcc019b", + "evidence_path": "evidence/20260720T192255Z-558dfb48/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ff60bd53ad86ce66f04ea9b07a5920fb1cae2532010f3d8d078f820d29d60877", + "evidence_path": "evidence/20260720T192255Z-558dfb48/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fe2819572f6d5c3b739c67884143931c67e0cd7ba2d5625f02c23d0df074e1ab", + "evidence_path": "evidence/20260720T192255Z-558dfb48/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:873eed5c7588f90daa87e592af47f6b650100d4fd1a400e5434b1d30da291f31", + "evidence_path": "evidence/20260720T192255Z-558dfb48/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:1198c86eaf99cb37ac0d9cf6d4f950e103afb495c23724709b4502dd21e927a0", + "evidence_path": "evidence/20260720T192255Z-558dfb48/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:216463ffdb298d960dd35a465b0fd639ba723bd3e378d98a08125a9150a5fef0", + "evidence_path": "evidence/20260720T192255Z-558dfb48/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:0abc2433d880de79f443a78db43555b2d2e1424444e971d4546cb65729ee0ce5", + "evidence_path": "evidence/20260720T192255Z-558dfb48/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e12bd2e91d8b1bd0b4279999dd4fdb438d609249753a11eb77787ae0608b60cd", + "evidence_path": "evidence/20260720T192255Z-558dfb48/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:fa7f91e1efb425034cb7f28f42976c04388b22e938d808449fe323078ac3b848", + "evidence_path": "evidence/20260720T192255Z-558dfb48/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6c2fefd121645dd54507ba22961ce94e2887b6b9426c4d78e6f9a2b9b573a74b", + "evidence_path": "evidence/20260720T192255Z-558dfb48/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:c3a03c741241c12904678935951c6dddf6d6b7d3f3019eb6358063a1208afbb4", + "evidence_path": "evidence/20260720T192255Z-558dfb48/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:cc1d760376e7f56eb32106b959617a11828d41f4d1c233d6a5e71ddcf9c9ae0b", + "evidence_path": "evidence/20260720T192255Z-558dfb48/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:019809fdf3ae7c8737dfe6c4013fbd9fe60e7a937fa9624b90eedd95f54fcc84", + "evidence_path": "evidence/20260720T192255Z-558dfb48/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:df4b8cb1dd4706eeda04a0d0382ed9dd823d887fc778a632decf2ef0be7a4129", + "evidence_path": "evidence/20260720T192255Z-558dfb48/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:11fc8b068480cb0015acf9acdf55bd24735617bd6d92a3edf5319d2cbdc9cf4d", + "evidence_path": "evidence/20260720T192255Z-558dfb48/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:c85b53880f60f1f568dcbb9058c355734efb81f918b61b1c1ffaae63a243b0df", + "evidence_path": "evidence/20260720T192255Z-558dfb48/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:874a099afc8c08e212b1030b0648089e076a853fa54b1b51e00284514a6c48e8", + "evidence_path": "evidence/20260720T192255Z-558dfb48/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:215720ddbebfc805fd270971f7d85ca93dc208bdf3d2038833f8b4d698001089", + "evidence_path": "evidence/20260720T192255Z-558dfb48/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5214dcaac8e46b065b4d9990cbe7dccb844550ff681bae219266d8eadd856ede", + "evidence_path": "evidence/20260720T192255Z-558dfb48/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:182eae8b3534195b0020a127709900f8e7522994c72b6307531e2cc630b5dd02", + "evidence_path": "evidence/20260720T192255Z-558dfb48/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6c81c8214a4b18dc03f00f9d58f27920056bf81b30297bd9678ff5d8617f128a", + "evidence_path": "evidence/20260720T192255Z-558dfb48/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3125337fb9d0772d17bbbf131228047410e11bb17cfd726b6410a71e205d1391", + "evidence_path": "evidence/20260720T192255Z-558dfb48/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e7dd7b410690ab1f6e20c82ff2ee70ac8dc03cd8aea2d169c8bf10489c78761a", + "evidence_path": "evidence/20260720T192255Z-558dfb48/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:8a597405de7b30171c3184f6f8f278948392d7cbe526f306a8d76d13989d8d8a", + "evidence_path": "evidence/20260720T192255Z-558dfb48/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:12c4b663b7630757c5d9ac2476e503df00ce95289af7412b38ee35e1ad2a2ab2", + "evidence_path": "evidence/20260720T192255Z-558dfb48/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:eba67fc72b79e611e4cb53030d09dcae285012affc214fb9f081c09f0dca90d6", + "evidence_path": "evidence/20260720T192255Z-558dfb48/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:dd718d04b25ce27ad2a9750e75d485b0327f320317bc35e2dfea36c315972c93", + "evidence_path": "evidence/20260720T192255Z-558dfb48/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:a688b743e0f73d80639404d836c4b721299e65473005f20b90924c6b83fc7eec", + "evidence_path": "evidence/20260720T192255Z-558dfb48/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f4292c4c2b733b273f27cd984ba4a028c8715ebe659d39b0e3173cb628a02955", + "evidence_path": "evidence/20260720T192255Z-558dfb48/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:384795d9d35ee99f93c9db1d1655cf6dc5b7042eaf133b8a4767b89a78e4ee15", + "evidence_path": "evidence/20260720T192255Z-558dfb48/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5d8377301088e71098d8d58e7ac75e616aed62241fe87e2ab000414a3fa343ea", + "evidence_path": "evidence/20260720T192255Z-558dfb48/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6025671c16c9eda5c15481023efa02c0c2faf77b1ffc526cfbef5250e933ee39", + "evidence_path": "evidence/20260720T192255Z-558dfb48/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:cd2480a6aba3b84c5a03e5e3d766b0bf4fc931d6dbe7e65b94bb4e91c8554de7", + "evidence_path": "evidence/20260720T192255Z-558dfb48/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:55fb36554e297b7cb3a9febc5784ca04693999e5989b4d167c340e5e5f87880a", + "evidence_path": "evidence/20260720T192255Z-558dfb48/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:1ca67dbb23e6748abf73486c2144d20f14d4d63f1da45548cdcc5fea74d0b765", + "evidence_path": "evidence/20260720T192255Z-558dfb48/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:1575dff5acbd38116137de7465ae7fa98b95621786f2203f900e7495b1ba921a", + "evidence_path": "evidence/20260720T192255Z-558dfb48/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:360e91e0ef02de9ad00f7129669464b147265140b78025520f17e098c050da43", + "evidence_path": "evidence/20260720T192255Z-558dfb48/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d322be8639322719082706a68df1fa3b714da5e625e14a243d2176fed8ca115b", + "evidence_path": "evidence/20260720T192255Z-558dfb48/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:cd69779e728fc2c8693b6bb7018108acedd8b8e2630abfb89bb115c0edb0d0c4", + "evidence_path": "evidence/20260720T192255Z-558dfb48/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e56fe7f4796fc25649d43999c457615f2337710cdc70448b0ced505752ad267d", + "evidence_path": "evidence/20260720T192255Z-558dfb48/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:758d556a8423df13b754cc11fa6edc3e4c5dd18c88d88a58644cc67749df3c54", + "evidence_path": "evidence/20260720T192255Z-558dfb48/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f588d276ef3740f7cf3d0a29528b2d06fdb97338d98b6c7d15fff9bc260bbd03", + "evidence_path": "evidence/20260720T192255Z-558dfb48/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:b3b30e3ab5f1d9dc4068e04ddff006f1e030cabbe991acbe2f82f6e889bfb012", + "evidence_path": "evidence/20260720T192255Z-558dfb48/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run4.json b/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run4.json new file mode 100644 index 0000000..eb6830b --- /dev/null +++ b/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run4.json @@ -0,0 +1,527 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260720T192325Z-a5ac4fa7", + "timestamp": "2026-07-20T19:23:25Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "sha256-44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8", + "declared_quant": null, + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:25d6db8fb70cd5debc44468e654c52d70997aa74c20c85534bc48deb49be528c", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:0528001430d1c8ed5dad8c8ec63beae917771d1ffb6eba545f264908d6d3bcf5", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3e112f6c6347db052add1273f5057e26e27edb00a4921ef5aaeeb68fd2d0e1c7", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:cf4978c495ab1f9372b061aaadf6409adcebd1caecbe3721e48fc7f9053dbf77", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:17ae16c0a646f9f3628166cc93ec9a48984912b58796df6d0231f05c86c6c8a2", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:351cff0d65a9fee4690c47626089b9ef554db0e029bc03e346740cf241614f5c", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:144184905cdc005cd997dba84fba87a5ba9d95eaf73f83ff574dd4f588e28444", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7f42b8c5fab24c3d81d1c957ddac5cfea691a6c5e9795fe2f85b59f58810a49f", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a24bd6e7c57a2a361c0fc3a2d7df93a96e57b34c9da08534cef0adbfffda1e4e", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fab00c1f603ad9bf906980c281f20bbf70068f44a583e98ffee4c246347706ce", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:96248cca8bfbf2d04ec2803a2687c6477cf28493e598becffe279bed42043d04", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:cea10e133b34c8604e2a8ca64af0dc1105843d368828109925e5e625195734b7", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:37c335d45f370b9aac2922f3e21a05aaa625eafb6d518b77438a2858cab7f9ea", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:cbc049727dcf30050f55d4ecddcc23c023d46279f6572fcc77ec70419e84019f", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:72428d15d0837e178629e89db1a4ff72d1d10e784cad8d8e593c229e92228285", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:8cbce8572fabf20902f1bc36e5009016e407a82c350cfcefaef05576b208932b", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:0867a19aa9b3382c733ffc6b8930f9f965904c18f3b7f777445b19c747462551", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:70c2af8ba5961f1ce69a29d75c418aca9da9a80ece5301800f048225581f7b8e", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e0aba641fadf030a51402be999255a81c5451c1f21bafcbc8d03f35a95462a84", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:1ef3e8120f03fefca8bc0305be7c20a36b7e1106d0416f40d8cf1a7263d2a838", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:756e4b6eaee0003b4b27a3767cdd8015ff67ef45d9ded2a851776544d4e24c14", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:edc5dd72e91703fa3c3c53baf9e48bd1400912c5ac8c85eef40bfb2a0ab3a02c", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5f6dff0ece5345532f216ab3e40d2fde94794c7f54a548291eb7b24bdd9c05eb", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:bbe9f06082cd3637dbe6c8d94fee4c2bb9c9c61962e049369b3d76c06ed83947", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f70d7f1af2841044960f8f38f1703908b2b3b2dafe725567a8a3fcf1e1022a2e", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3f6d7e2b6895302f935439c7c6624c235eb08af1f83360e7422da0272542f736", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:d6f9778e99e8f012eb3ba4ea723f9b10ed3fd407c07ef0c413e9d38e0753bebd", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5775062ddfea27db5a3fc623e0dad8af15dbd7c02a25f6aa8f9749344ad287fe", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:fd4e4d2fdde4d0e9e4bfcda981d8c21658f1c74b26321ead4776bac160b8dfc2", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:dbc99e05057151b372075b3ebdceeee5779a80f9eeae3513674c91b8be1f92d4", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:9e6fa0fe3cc8fd14c47cdefea4095c402ef01e67d6e148d0929a4003c9140b3c", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:380c83d82dafa7276f6d84a6af56c7549185d026136205b37b76db024740912e", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:1ab23c8a90709d868975225fa75495b4c63ae7313b6160df6c9817d8521edbc7", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:cde6184154e8b9269970bbb9a09dcc2599415e0e18482a43f4c3af9fccc41c88", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5b222630ed9a881ca3b38242d50e993a97f2a0dd156bb9b97df807753c0ed1cb", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:b2b7d822ac0cd4979365b21045457684a1c6ef7c6c6058337b06f978c37662ed", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:4678fd8b29b396fb82acbc9156363386799e7399ad518c79b66bccbc5d78f238", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:077a3c286762f642ae3a7aa9a91aeb3cf6dfeb6a79e4b78d164e8024877c1e4b", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:319df0f40304354417637f828a1e0e28a4f21f49cb7b8c091bbb2f868ee2368b", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6a13f5d22bb74f3391615368a0aad16eb6645687532e3db5af32fb80a361192f", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:65e68b808b3a964e3ae75c2a0f810a36a2cffacfa57b746267c4d0f886ac7ea3", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:145758d88ba49a8fccb79d834d26fa9b4cf1eae1c4688a2cbc45ba5cdf578d86", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:be57cec1882e1b6cbac2e4f5b407f5c51c0903f4d5b0a68bedcfeb1019594999", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:be49f94c6cfb470f2c5df1280125bdf8f18f64ba9192cc41d7ee935abcf2b016", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e4555010eee845636642d893a614d5e7330c522c6159bb8f9fe2cd6f21da8086", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f76a8940433d673d83fccdf79a9037b208320ba9eb479a30304c41426409f874", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3d4ef97482e5b740eb005f98cbbc3d60c649e29245ac55f3f27ccae83d604d8f", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ac44e184ee7326685b01b487d1e9ca15b9cd288796fd36414cbd665d5a273eaf", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:dfc0094a179e87f4439a06fcc93f3d42d6911f0c114a98b03a17da2c9b56f587", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:57f6e1163677c26f2df20cd659cfe8fd0c47d3f27d485dd12396cc00f7411b81", + "evidence_path": "evidence/20260720T192325Z-a5ac4fa7/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run5.json b/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run5.json new file mode 100644 index 0000000..477597f --- /dev/null +++ b/docs/case-studies/evidence/replication/m5-replication/llamacpp-granite-run5.json @@ -0,0 +1,527 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260720T192356Z-2568c21f", + "timestamp": "2026-07-20T19:23:56Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "sha256-44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8", + "declared_quant": null, + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6a9cd1981d18f1e2b59ca513a91a2e2e64dac4472520fe52c092be4a6c5b1b18", + "evidence_path": "evidence/20260720T192356Z-2568c21f/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6c1187be3f009765ac9e18f073ff5486bdd8fe0359c3d71177d4de1b8cc4bf38", + "evidence_path": "evidence/20260720T192356Z-2568c21f/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:c7604fb8a4048f2d86cd0afa137b38c46bd62d1c7a86a422a5ce17b4697a1b03", + "evidence_path": "evidence/20260720T192356Z-2568c21f/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e3194dfb9e6a233ab341fa4001afd21a314173d0458571de9daf67b525b8ca16", + "evidence_path": "evidence/20260720T192356Z-2568c21f/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:7a8af47699c3b231a777804780dce4352b7e42053353d72d695de938a879489e", + "evidence_path": "evidence/20260720T192356Z-2568c21f/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:af306c02eb3127fd5fc066e6aa716cb696cbb9de164c4a2ce5ac270e86f8876a", + "evidence_path": "evidence/20260720T192356Z-2568c21f/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:8d2b821f8ded9a92b52b76cda88d62b39ceee9785b4f49bdce1e706f5061bf35", + "evidence_path": "evidence/20260720T192356Z-2568c21f/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:06a86e066b054fab151611ba895c929054d9af20ce0ca994c4e99afeb274786e", + "evidence_path": "evidence/20260720T192356Z-2568c21f/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7b1db9142e8278d9e610d0d1dae1d060ab0ca2d4c8967f9af66f55af167cc852", + "evidence_path": "evidence/20260720T192356Z-2568c21f/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a1fa7d6e81fdf28f74440bc0cbfb96e135eadb956c9810b72b833e5d60cb2710", + "evidence_path": "evidence/20260720T192356Z-2568c21f/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:555cd5729ed39d44dc02888b846cdf2bedbf64b800a181b5b1dc85d60e05de2e", + "evidence_path": "evidence/20260720T192356Z-2568c21f/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6af7baa13a6c55d1c2b61a779b64255c01a914f35ddeb288d3b5bb61d4ed8ab2", + "evidence_path": "evidence/20260720T192356Z-2568c21f/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:05ae932f38c3c1c9b23ab97919a740faf4c86187968bf3a216198d03d7cd7e06", + "evidence_path": "evidence/20260720T192356Z-2568c21f/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:86db0776ddcaf4424603d222c4d3e4e5b3b4a974b3a7909464cb3635f86060fb", + "evidence_path": "evidence/20260720T192356Z-2568c21f/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:9e0edba5008023fdd08495d4ce82a76344b6fffc8e6eb391b4235e29ce1d7a89", + "evidence_path": "evidence/20260720T192356Z-2568c21f/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:124d8ce9a78edc0246dec0d650a04e53d60e9e8644b65835c4be0fecb49edd7a", + "evidence_path": "evidence/20260720T192356Z-2568c21f/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:172adf3bbdaad6de0e4d9dc3a4793458d9091a17f7de7e78b95f186eb83a459c", + "evidence_path": "evidence/20260720T192356Z-2568c21f/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:68c56546221c5acdb1bc34be07579c3caac9657ef387816b25cd6b822c79729f", + "evidence_path": "evidence/20260720T192356Z-2568c21f/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:51102b6d38f31f503949244ead520b369093a3e16717a762e4cf66e40f85a6e3", + "evidence_path": "evidence/20260720T192356Z-2568c21f/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:116ea6412784ec231761dd8ba59ca64101ba438466ea299a8b4aab7c9e2902a2", + "evidence_path": "evidence/20260720T192356Z-2568c21f/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:7220649afbede982a87d9d218cf8a04ae17e5716de05afa81b54e126cda32c39", + "evidence_path": "evidence/20260720T192356Z-2568c21f/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:2174f9b2e025fcddef33e0f488ec6952f66668aa0fb7a8c080f2185662d071fa", + "evidence_path": "evidence/20260720T192356Z-2568c21f/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:895dfd381a7966f693da47cd750c82e16ad1c2da598de99773e554e7600a749c", + "evidence_path": "evidence/20260720T192356Z-2568c21f/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:039c38d051dbe10fcbc4e364a91c1324ac973a4383dfe9e55850afe2615cf150", + "evidence_path": "evidence/20260720T192356Z-2568c21f/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:417f4370a9050e024defcabd4762b28fe1b9ff3fdb7955c326dbc969faa07994", + "evidence_path": "evidence/20260720T192356Z-2568c21f/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:18b93f2a1fffea3f613295204022c45cd3cce3cf64263f68987b89dfd3c88db3", + "evidence_path": "evidence/20260720T192356Z-2568c21f/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:31a22767625fcab17bf0e6f28552767d9663094310f6d10d3f9118fa25817f76", + "evidence_path": "evidence/20260720T192356Z-2568c21f/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:50350d9bb7c8a9346e08027e919c92b5999217400e79b78353913abf1cc69c4e", + "evidence_path": "evidence/20260720T192356Z-2568c21f/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:075650097f3faa9c4d65b20d9bd3efd90f42cbc02b03dc6b9c29f791e5d707f8", + "evidence_path": "evidence/20260720T192356Z-2568c21f/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:09dfbddb347c95ec5326fbb44ef15039795fafc9b095dbb65d48f7a971cd0319", + "evidence_path": "evidence/20260720T192356Z-2568c21f/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3136715fa394afca807a58b16dd88eb163b2bdc1a24daefe9a50a74c30bbf776", + "evidence_path": "evidence/20260720T192356Z-2568c21f/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:1dbc9c926fb4610ec67fe0e292522ef80f0ed9dce1e0edd326fa2ffae5f50b44", + "evidence_path": "evidence/20260720T192356Z-2568c21f/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:944a73f0f655a7fe16e802ab0d22a23e4a39bc0a3b9d30962f395144d0a5f3ea", + "evidence_path": "evidence/20260720T192356Z-2568c21f/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:83160bf37f31e012c850de95d5508871ff5aa43b36b727ddb94a50c0d7b61bdf", + "evidence_path": "evidence/20260720T192356Z-2568c21f/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e44c0106853139c981854d86460e8c2c48a80e4371c749093499cf9add63d8af", + "evidence_path": "evidence/20260720T192356Z-2568c21f/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:75b6cb0f213ba4371bb62cbdfe460e7a0bc7910c9bbb8bb13340b2407ad4de02", + "evidence_path": "evidence/20260720T192356Z-2568c21f/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e3e2e370501410e9325b63385a834311891aebd90160c5665340057ca944e16a", + "evidence_path": "evidence/20260720T192356Z-2568c21f/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:ee6373193a1717d05961a0acf558fd5111b7a373c266db0caf046094990b0bfa", + "evidence_path": "evidence/20260720T192356Z-2568c21f/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:bc5ed8f8b8311065ab48ac1fccefa233cf36903540c9c80d20090dae4f8bbc41", + "evidence_path": "evidence/20260720T192356Z-2568c21f/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:19b350845d0ad795a1c7e4d3ce0b162ce2cc20271cea985448566d86d0afd415", + "evidence_path": "evidence/20260720T192356Z-2568c21f/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:bf667e4b093dc1aec86d7928dc3cc4061dac6238de3d121a486d37fba9ba0736", + "evidence_path": "evidence/20260720T192356Z-2568c21f/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:ceb2aa74ce5bcacc6264c1a1dedc4ed49712f60aa5cb329078ba1e70f8550647", + "evidence_path": "evidence/20260720T192356Z-2568c21f/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:063c0e30651f960cc76c207b9967f87e091feab630d4f45223f677db6107b26d", + "evidence_path": "evidence/20260720T192356Z-2568c21f/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:1ed4ff1699b491441ee4cec3cd95f0bf9fce38dffae5308518c549db9ff676d9", + "evidence_path": "evidence/20260720T192356Z-2568c21f/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e7f291049297bd46931b9e0fe7755de5b7c608d170bacdadc81de2f564e3573d", + "evidence_path": "evidence/20260720T192356Z-2568c21f/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:33c17337bc57a94bb5b0a598558974ddb93559543550ecc5002e4f0d6025ef3e", + "evidence_path": "evidence/20260720T192356Z-2568c21f/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5db66b83f43935f1836149d814126489f9cd50daeafac4b72e8bac0f8ca58d25", + "evidence_path": "evidence/20260720T192356Z-2568c21f/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:04f3d71a488d212d6e9990205967c99caccc6bfa4ea0e3f1daf15190106cfaa7", + "evidence_path": "evidence/20260720T192356Z-2568c21f/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:aa6376dcf1c4bb240d4911f04ea4fc8caa5e84a963b16a00ca5085c36ed3bc99", + "evidence_path": "evidence/20260720T192356Z-2568c21f/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:c518e5eb1d552207c5688d82e21df1a455eef3746f06f098aa048b56d3a65dc4", + "evidence_path": "evidence/20260720T192356Z-2568c21f/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run1.json b/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run1.json new file mode 100644 index 0000000..db07a67 --- /dev/null +++ b/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run1.json @@ -0,0 +1,520 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260720T180605Z-7d77cb86", + "timestamp": "2026-07-20T18:06:05Z", + "willitcall_version": "0.1.0", + "endpoint": "http://localhost:11434/v1", + "model_id": "granite3.1-dense:8b", + "declared_quant": null, + "server": { + "preset_name": "ollama", + "reported_version": "0.32.1", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:431726d1a811af716897d1cabb183ceeef0d8ca6344467ec7dc33b3844c56055", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:a510f5649be2894725595b08373cf6744337eb04e0a761fb6671a8eb368e70df", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:455ba4f97892927d443a0fbf8f3429541c9df98d7b5830238de6b87857278815", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5c34541f8ce996bd36b74c5bc44943329fb880adce193ca140ebc7c88bf3d0c2", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:348203d8cfa77c80c07f3d5a37ecdb4cd03efdd531143411f1dd8ca69079b53a", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:c38fff6df7f135bb67a041770aa2f3a145b3004f5430686d48674caebc43c38f", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:50c94f509122745da24192865765f75e37b3b9d07cb9db762984989a4c6dfd6b", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0bc77147d147fc2dda25a3795ec36861e1d2d35115006b453cb79bf4b8dd5e25", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7d0904cd4c0c525c7713fdb1c3a0d095bbc7f340b7b0123e2ff57f638791a23e", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8701250087c3de9a148b673faeea7fae33237879bf49e88d85bc57bc96305d3b", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:958ba827eb110154f19d5e3fbd0dd0610cc10eed3d50d311d9347ba2b6770ddf", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:c1bc0cef07da4e507ccae98aacc9c4e52b991bbeb404cebd98f3cf12e53f5bda", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:42d5f2a22e3a559f4afb85f34dcc156ee553966cdf6f7004b2a0ba896b58872b", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:d488b84a279c1adb94317131a35df26ca9c11118d7cfe3c39245c998c29d4d22", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:0f5e8e1a3f5fdf60dcc8c79a88ffc71da9d55ad7a085bf7729eb008c28d2586c", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:ad5a4f954d9e36a9746dd965be6d59a0c26da0219ed8e1193f8a24d7ba21e86f", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:20b30bf7fc2148c925c9936b1a20b320cc0998f9c9aaf92bc2115d09a6c9247e", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:897c6a56253abfa5ff18d91745786131dffb0e72d89ccde7e87114b123b43828", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3fe9837f712a7bfd449250d2e7bdd3de8670f502392f80a5b0418c5a7c17e1fd", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:9e9dc6b015f6ec706eb6bfad5d74c5354623bf038275381d4c70c3d721219453", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:9dd64e6a370febf2b16f4c7eece2e853cec52fc5fe6ea02be907984a15fafc38", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f678457d0f515d43d950844b37151e63dba9bb1498baf1940fd7268abf2cd05d", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:2a5690ff5a667303b1e4d95c9d54f53bd942d7e5d0d5efde68b29fa1c2646df5", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:a45378a42839843c62c7d024663fc1f5548b87cd341d41fccb8b78793fc5991b", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6b3f84a639292a57621009d7f3927257c80c54f149f617d4754868e538ae2ea6", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:997ec1561bc35adb31c8cff9b594b97f5154acb7766b1221dd383876c128c749", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:754388d8543b1a83472a612620070bfa690b730669b256409e5f707b6a71e26e", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f4ec4befe806d6ce2ad9b3a19f2ea0ab796d7c720ba159c726f7dbc147a72aa7", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:d98f4e6beb7f73a175ad8e285be40bed394c37910742b3722ddfc490842acaa8", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:c0eebf669ec32052891a81da47e2c788756390d27e086b8edb325a65fad83792", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:67137d55334f3788c233778762ff4fc97b4430975a79b915b88fc9b29b7a9527", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:14fbd6af84a6059f040ec19c90b0a976f837e1dac036a63eb1bb395389731cfb", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:0204141f752a444ab4aefcbd8f9c2a44781a875f6ebf722bc204d17f9f976cbf", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3cf133c97185806697b8285e3b971a5ed727425ca8505bd9f604143e015d90b8", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f57584c2010208f50436907cd6426634a56e19c9021f2e2d9dd21b3e4aa09082", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:9f961bf6e5be8399d76acd82fbd516f90032797f0557327b3cd209d3f91ba13c", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f438899f44346f013f1463f540d1996bb108f764391a121827765990bd319da1", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f10a2b8bb1cfb3f66154515e9ccee14fc06d2525e596d98d9b9e9764125c39fc", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:797bc228b2dfa3ad951b793a6f98ab8fc2d0c17573a21b488683493c5525dfd1", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:7053812304306781cc8940d5002532e7e263b6887fed4e841f6c6be48befd197", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:7b2764a57d9029bad489034cc4e3e37b765780492a49d4ac7291844c4fbef6f7", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:1a9405bc973469860292a2f030ecfddd9ca93514e2c011caf8e0397d07dfbe28", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:03257993f2fa3e1f2235cf740c8eb6444db754739848e4ec2f068cf08730a889", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:95fd4335860785019003183573224614b9c29ffee17c9eb953939272c8c36541", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:74cc33b163c3f5fd32c3190ae16441a7fed912d35cf7151dd261e3c4de2db0a1", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:c919376fc8e14d9e9c61a5a7f2d01759fca74997bfb2175ac459d95abebee4a3", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:d086acf2352f3838c14b049c9262456d8d12a21a938bdd92be9f8a8caa2ff993", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7666dd5af761980b7a03f7389db7ea7036e3aab9e336b2023d2c0e0be303f01c", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:8a4339657920306583926f6d4499ba95a775897522388eb3c8a180d91915bfba", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:8e8bd30f2d30d4f34e552f2a6a5b98d63c08efc91ecd42215abfbc61a8297daa", + "evidence_path": "evidence/20260720T180605Z-7d77cb86/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run2.json b/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run2.json new file mode 100644 index 0000000..db5de4a --- /dev/null +++ b/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run2.json @@ -0,0 +1,520 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260720T180641Z-19132017", + "timestamp": "2026-07-20T18:06:41Z", + "willitcall_version": "0.1.0", + "endpoint": "http://localhost:11434/v1", + "model_id": "granite3.1-dense:8b", + "declared_quant": null, + "server": { + "preset_name": "ollama", + "reported_version": "0.32.1", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5966bd0db5ee260429e04308dc23e3a07a9465f82bf963b38e222bf87bc75cc2", + "evidence_path": "evidence/20260720T180641Z-19132017/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:238d1d717214961f59c66360d1e5879fd8e4531b3fdfc434fef7ef74b04ab498", + "evidence_path": "evidence/20260720T180641Z-19132017/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:570aaf0692418d2921d82430ed1ee50c1601ab5591f9787d7f09275c74d1fbc0", + "evidence_path": "evidence/20260720T180641Z-19132017/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:2a70c69c2c880cedfa78cdf7a2680475c0f3481f1c8905cfac2340b88bb913ff", + "evidence_path": "evidence/20260720T180641Z-19132017/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:b79883d83dad22bdc9d4ec0b2ccf78bedb4b1bb3cb59811425dd249ba444589e", + "evidence_path": "evidence/20260720T180641Z-19132017/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:c67adc44a96ce7edd48c5e0fcad33a44b90619b2a6433af36c63111248b6b6df", + "evidence_path": "evidence/20260720T180641Z-19132017/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:893a66eab6010f3502e81fe7fe3c990446295d76a9e255c3ea03ce2e92ca0b5b", + "evidence_path": "evidence/20260720T180641Z-19132017/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dbd8b7ca36b1ab867ae2550f2bd1e1f93c3d2561dd779a75a25c54f11ceb2141", + "evidence_path": "evidence/20260720T180641Z-19132017/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8ceda42a0563307630435fd76db4a7c9e5ced9e97d8be09f0d532dc0843ccec8", + "evidence_path": "evidence/20260720T180641Z-19132017/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ff94095cafa8396bb51934cafd4751bf70df448e1ed24c04fd0e22e6da53aad0", + "evidence_path": "evidence/20260720T180641Z-19132017/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:91eb91609b3eb7b2f652d3f0ffb8ec0db09bbeeb924eda8069c85f892fa802b9", + "evidence_path": "evidence/20260720T180641Z-19132017/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:7234832b3b64c186d955c23f69cafb08110075cba1b9bbb82a1592fec1fcb91b", + "evidence_path": "evidence/20260720T180641Z-19132017/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:234a6e0f5a0f284a1a738d3ac242820b22f46470289a75bfc5b3358b3711ec3a", + "evidence_path": "evidence/20260720T180641Z-19132017/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:e4c71c700b352e27db7e088526c4b4418b2a03047cfc226d115ba4cd2a5a2fea", + "evidence_path": "evidence/20260720T180641Z-19132017/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:55b11d7e2c60cf0b086b17e372dfa1f00b770c1f219072194466d1797b9a08a1", + "evidence_path": "evidence/20260720T180641Z-19132017/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:9fbaba674082b0d22308c7917d13920c93f88d341f77c3f7c28d2681fd8c24da", + "evidence_path": "evidence/20260720T180641Z-19132017/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:faa430f3c8af2fa146d773926a7f03906e5c2f280b36e7f628178120f4041b4e", + "evidence_path": "evidence/20260720T180641Z-19132017/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:42ce369bb56263c2e04b8b475a581ca172063d2f8147f1d20cfd00dc8eb84fba", + "evidence_path": "evidence/20260720T180641Z-19132017/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:ad0e66a6275a227c13dad490d199672d1b9a8a838e3d2bc88a6ea8e28c7551c4", + "evidence_path": "evidence/20260720T180641Z-19132017/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:39574054f3e9a04f8578a79840da4a2e8027ec26f07b0c0a6cf0567d824e7d5f", + "evidence_path": "evidence/20260720T180641Z-19132017/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5a5c2f917bca52bdabc6834005b8069ab446016df533cd25cf66afe5667ef201", + "evidence_path": "evidence/20260720T180641Z-19132017/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:23d57a6a39fe0db096e2423b9a1dd33b75259cff6b0c81f6604d2a6e802b9661", + "evidence_path": "evidence/20260720T180641Z-19132017/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:543b05963638cfa8fffb15da03bf4670a7cb07318a307458e95edef344b08e5b", + "evidence_path": "evidence/20260720T180641Z-19132017/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:1992ecd00acd8b3c192b54dea82a032bc577138348fb0a6aee1283310f57f4eb", + "evidence_path": "evidence/20260720T180641Z-19132017/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:9274fb979ec7bc529cfde14ec049ffa13ddc8e54b981a99a10b17e46f803ca1e", + "evidence_path": "evidence/20260720T180641Z-19132017/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:30da9da32cb8862fc9a54418aa801844b049972ba6b51509954cf2f547ba1e3a", + "evidence_path": "evidence/20260720T180641Z-19132017/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:65605bf99e524a3b61d4b8eaea8732f542bf1fce827a82faf2257413d5dc12a8", + "evidence_path": "evidence/20260720T180641Z-19132017/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:11facf72b9fa360fbf8526e74b605533a542a330dd419b0e43c6674b37bc0c39", + "evidence_path": "evidence/20260720T180641Z-19132017/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:cc6bf707ae9b03fb1c9ae4f13743cbf745e71fad5c87c9fd75fbe1dc929457f5", + "evidence_path": "evidence/20260720T180641Z-19132017/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:405734163fc0bd009d65f7623601a23512fb955dd4d6aead725eeeb0460f0df0", + "evidence_path": "evidence/20260720T180641Z-19132017/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:d2fe61816667c443789aa0399f3f291b17257459760b20b2702b6b85319cc19d", + "evidence_path": "evidence/20260720T180641Z-19132017/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:779112e14459dbee185b135f7bdaed4abe3ffee5ab87352f215b7cd74929e6d0", + "evidence_path": "evidence/20260720T180641Z-19132017/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:19fb7a6f8eb47a5116ac134d80eaee6b63d0ac03db6adb6f6f5fd7d4be7d5ce2", + "evidence_path": "evidence/20260720T180641Z-19132017/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:d23e610b438aaeb6d02e2575aff224df9983a55a11b4fc935b84a1182676459a", + "evidence_path": "evidence/20260720T180641Z-19132017/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:29096b5e40eec00bfcda25d307afbab437798c8244d300f76b493d1a439a30f7", + "evidence_path": "evidence/20260720T180641Z-19132017/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:85b33427d5b0ce06e97c3f0e770cc63da5ceb84eb26d681ac7b8a5ad10e98f9b", + "evidence_path": "evidence/20260720T180641Z-19132017/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:2794ae193962f205a65b939dfee58ee871e54f375de13319d0367f6ee68f4891", + "evidence_path": "evidence/20260720T180641Z-19132017/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:89738c71fc7b8c1cab578fd48088b00ff86168dbbff1f2b5f83050b5e2cc8b71", + "evidence_path": "evidence/20260720T180641Z-19132017/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:dcafac8cd25cb8f073f17d78d94cac9cee307dff405ecb006cdcca9e55339ef1", + "evidence_path": "evidence/20260720T180641Z-19132017/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6eb6b85aa85da096b6a075a7f3825d530d08ad58b5aaf910a4a02add84fe2838", + "evidence_path": "evidence/20260720T180641Z-19132017/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e1b53dc8871df7821bf24ceb7e3cc2f1eee1146f46e2ec06901e1f6e634660f4", + "evidence_path": "evidence/20260720T180641Z-19132017/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:fb87c1d26a90d6aaf8ef7cfe35fed4dc96cd62eeb58d99d4d19ecb7410102e4d", + "evidence_path": "evidence/20260720T180641Z-19132017/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:981632abd62f2d2cb217b9b5abb7cccf69819847b19e06b8925112d79581c916", + "evidence_path": "evidence/20260720T180641Z-19132017/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:1065553998705e494eb333fcc652417b2ca0edb1777d3e90b267283754138a73", + "evidence_path": "evidence/20260720T180641Z-19132017/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b67cdd19e58db320977be6f684efc63e5777339874fe56459c27f3af6e204e64", + "evidence_path": "evidence/20260720T180641Z-19132017/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6091becb5856651ac07ac54b56bad8d24fdd17caa712235d003f6b4e647e32e8", + "evidence_path": "evidence/20260720T180641Z-19132017/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:1c80899a912d833eb0f6e2db4d5e3eeeef765bf86bfdc64ea327eecb85d25b64", + "evidence_path": "evidence/20260720T180641Z-19132017/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0408cf2c5205df5fce9216805a800e99d42870cf71d2965a935cde3278f66ab3", + "evidence_path": "evidence/20260720T180641Z-19132017/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e47240594a59d1c423064207f1e506cbba3bf31db141c020177ebb91024752e3", + "evidence_path": "evidence/20260720T180641Z-19132017/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:044f54f3c4547d3d755995d288acf5703045e3fa3c733a8acc7915230fd218a1", + "evidence_path": "evidence/20260720T180641Z-19132017/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run3.json b/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run3.json new file mode 100644 index 0000000..23018e7 --- /dev/null +++ b/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run3.json @@ -0,0 +1,520 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260720T180711Z-8a54f53b", + "timestamp": "2026-07-20T18:07:11Z", + "willitcall_version": "0.1.0", + "endpoint": "http://localhost:11434/v1", + "model_id": "granite3.1-dense:8b", + "declared_quant": null, + "server": { + "preset_name": "ollama", + "reported_version": "0.32.1", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:639cba6fb1250ad46718b7f5b17ce89f8c2e112c1c2668d8d7880f91c68db82b", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:589305a1f65e7fa37a7bf77cb79a40abc076c3e142faf433ce67baca86696ed0", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:d23e1572f98a0bc3ef0119302bb3bba4df24e48b2fbf83ccbb30c326e2a95e96", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:35ab6e92823886a2a6775e3390eae576f435b7e3d57e15e8cf1133f007713cdb", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:1a46460bf1a8a30575a4fb9320c32df38f60997ce26866f24816cf322cd74baa", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:0746210fecd26c48a7ee3fa2882d05d2157a4bf6fc78ac01ca858a8609d3de67", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:acbd978bd2162b827fa750838e688689cd7da010f983a13c9153bc296c530ce8", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8f81b96285da2870841c945acbb6fd95ee9e806506dbdfacfebb49dedd91ac62", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e11a1437cc8cb04bcbec95ac56bb119df6b32b29cda2f0beb3faa9b426979f76", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fc976fe9de432125061e655e494a2308070f4c0f85ead2c34706b5e1b3517516", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8bc0931dfa0de4582721f93b31c00f49784c1e27ff1ee88711e99d5ba5cb37b2", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:def6d38e9789bf8bd9dab2923ec50a65fabce46cbfbbaa36205c55906663b329", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e319ae314402394372dd3db09182e883e718400038218ba3d230485c3ee93c87", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:31afa10ddaefdcc7f3c9a40940a645cbc198266909c8798276eebcc98d2fc458", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:11c0f0cd93d68103f554914eb4f653b76529b73ffbcf53e76f6e7d8a780b932d", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:c4ae6fdae5ee84323c5b718069a4437adea1c10e394b25a4b9cb876845c891a3", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:8cb4fd8078437628ad78593fdf57c1aafc0fe0697d674fad7a2a4fef460d06af", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:2e9068d2f10d77b09777cde33e06de6f0fa8b3a66d4099448aa02354305bb5c1", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f2ff28f2a5c86baf9b595c32d3ddfdc7c64ab3de06f6337f19275a19f15f746d", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:9ab38fe219fec37bccfdbd349b9146c514bb6aac112474cc5ac27a5950ab1744", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f9853ec331ba0a7d35cffa72797e2699a63795984cdf971e7e8c0f674c1d3b0c", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:2a829cc6fcafd676251fbff5da506cea56e3d5e026a0e62f225161c3c106ae8f", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:4aa5d9056d8d8ab2cca97af0e5222ccac87beee1b289c406acc4ce12e29a8fab", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:3009cab7fbd9595a13c7d7bd78dcb801251992ca4bc9b8bbfec88946a918e70a", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:d3dabce63da2244892fc010bc424f1a051b7b6851f6610dafb6227492bd51024", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:bacb7215fb32e7f19a03bb3aeb014ba699be265dc7f4f930abfe417a6bc97509", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:a9bb201ef2873b1e05fe78c34babc85425d1f45c5e536a42c862ce23c3de7e8d", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:bfab305ab8ff37620c274f51b8167d194373b037972ed41ede0778828879b959", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:b960592245c9945f192b59152de530fce224d9b5a7ae112707cf1945a250cbb3", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:ddca4cd527d4c1acba55ddcf482df3a14791282c8cb018a2bc895c33c836e53c", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:21995dd44decb7f0c9417131e801847e0eab468fb7770f642624e8083fad4277", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:e3addfca1da2f5bb671ec84a73ca99dfccd7017c0bdf8273c03b37da4bb36f03", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:4f32e6258ad8775bb2983a359c8381a4c1391ee0286551e546b95260ef1fbee6", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:af43aa89dd5e2f0deea18850ec6e06074a5321f59b19f15765b136f675649e5d", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f1eb12ba1d232583d1519712bb8632cc01839cf2e0fdf140f873c14bb81962e1", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:559122230c7b97f40187eb8455a5c614b35eed95adfe42a4c723ebba3900cb35", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5b4ab3542a590409c7971a5b39a0c11cff5953ca00d5c263600dbc9389ee0402", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:9aac9d87db29b53fa1e657fbc017379978183606c174c26b6c7a1efe330ec47d", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:2edeaf300f96e73ebe6972d9f21c57dd7b8ff7ef61a0af626db8696996a6a205", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:9f58738f649a08258ff50bcc40e0bcae4871b9e423ecd5aeca6807c0fde3ed8c", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:60b088760c3be0912f981f2a1d17d7f4d901bf073175fafe9f2bffcf7bbe8b71", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:eb3b8a713c3f73d78d3009575b008dcffdf0cc0369472570c23f3f52bf3ae8eb", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:66668e805f36d034f7de9b97c279c53573c4e4895e091b9a11f0cb607cdc0e5f", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:b1d26c9d9ebe4571c96dc68f3e3b3f57f60bbdd837621414586e944c3c1a73fa", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4a2e02822805b603098f028624f37c9189113d33ecba39de8ced48627a64768e", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:419797693135cda0fc65480b99484c542bf7d1bb31952d1ee776b4eb3e82be90", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:2484088a59401a203f919b39544a3d467e28b6b6a1d057abe3ebf0018ea895b7", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b3c599f611f00e80118e47d2d1fe96466fb365e6f0fa61e79633916498f19599", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:95cab040ba588abf34e5f8d86292c099ecf5cc03eaf4b9b3969824ab8c63ab63", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:d43c616ef013e857bab9d5a719fbbba6c64ebe094ae8e231f4b49c787d0273e6", + "evidence_path": "evidence/20260720T180711Z-8a54f53b/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run4.json b/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run4.json new file mode 100644 index 0000000..ffd1565 --- /dev/null +++ b/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run4.json @@ -0,0 +1,520 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260720T180740Z-7f165148", + "timestamp": "2026-07-20T18:07:40Z", + "willitcall_version": "0.1.0", + "endpoint": "http://localhost:11434/v1", + "model_id": "granite3.1-dense:8b", + "declared_quant": null, + "server": { + "preset_name": "ollama", + "reported_version": "0.32.1", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:107054b5f7aa2a0e43be9638022869a400277caa44b48d48e6f86ba640ad37c6", + "evidence_path": "evidence/20260720T180740Z-7f165148/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:21d66bac9814506454a8768c8ff6643bba22722d5c3b15404026dd461f145e9f", + "evidence_path": "evidence/20260720T180740Z-7f165148/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:dbe4a1a00883b6a19bd2a26f21053b935a01a63f465ff4b5b2a85669d18ad593", + "evidence_path": "evidence/20260720T180740Z-7f165148/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:279162a251497e4da8bcec95456ab46a14319b9eb6fc96a2b335caaf3246e65b", + "evidence_path": "evidence/20260720T180740Z-7f165148/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:056776b6506dadeace0e16138b3c4011496f33efaa3474f88c98aca8ccb998df", + "evidence_path": "evidence/20260720T180740Z-7f165148/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:6bc8039b41db9f78430b12d9f1d36f16984c77c88d8eeaccb5dc303a8c848c40", + "evidence_path": "evidence/20260720T180740Z-7f165148/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:663c0147a63405a843b665f6d84b28754abcc672bf7e1bafa2829858d7dcc738", + "evidence_path": "evidence/20260720T180740Z-7f165148/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f5d4a1411560a9b871a3566b7973fbc6d74a1e6aa4083239ff84bafd4da7588", + "evidence_path": "evidence/20260720T180740Z-7f165148/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4663c61296766405b01f0c117817861c8cd5aa1e2243f000a759df6906a8e5c2", + "evidence_path": "evidence/20260720T180740Z-7f165148/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f62998ab6a6729932d4593bd51a6f53efa06100c975c21fc40a0e9bc23d6e6d8", + "evidence_path": "evidence/20260720T180740Z-7f165148/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1d6dc7c38695c28dcc9524f4b56c8dcfbc9c574a4ba693f757298bf250d1df33", + "evidence_path": "evidence/20260720T180740Z-7f165148/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:dff11248d18a402a63207b7a7d747b8348498dc18774ff311b49be144fd73288", + "evidence_path": "evidence/20260720T180740Z-7f165148/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ed99ba24b305fd07eecfd7e5eaa22c7d6cd9d690e7c159bf5aaf173fd30e78d0", + "evidence_path": "evidence/20260720T180740Z-7f165148/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:90624c01399a00e972e1690582eadf4ac545f02154026f79190bb5b91477d56a", + "evidence_path": "evidence/20260720T180740Z-7f165148/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:1bb9e8bb86d992e9cdc8601de7bc17f6a99c8e07003fb0bb805e9ee2c292a2bd", + "evidence_path": "evidence/20260720T180740Z-7f165148/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:a261003d8c2d493c70961f2aecf8492bc4a59e9c8310f1c9cfaa3f3b2dfd5fa9", + "evidence_path": "evidence/20260720T180740Z-7f165148/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:95871b8fb5853d304b561dd662fc08ee05c8ea5eb145958d5cdc7c435b31d38a", + "evidence_path": "evidence/20260720T180740Z-7f165148/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:b60df9da5ac6d9447070dd85356dba000b9702ca50fcc5cebcdf9b9874f5259c", + "evidence_path": "evidence/20260720T180740Z-7f165148/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:2bdfb0722c2a0dc631b8c0cd00dc51f64edd63b2106a751316d79b348bd718d0", + "evidence_path": "evidence/20260720T180740Z-7f165148/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:7358cce35fe662689ca8a7d0f2d352d901fda05930ef44afd7e48017caba6393", + "evidence_path": "evidence/20260720T180740Z-7f165148/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:fa20f00059f417e00befab06a3bb5c61a43d652d9b25ec2e627877bfb53199fd", + "evidence_path": "evidence/20260720T180740Z-7f165148/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:52165c5377ee0d7f7ea95a6939d63f9f86f91a4dc0bf021d098d8971a5c98fc2", + "evidence_path": "evidence/20260720T180740Z-7f165148/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:503ca8e45e58f534fae6c997d3ddcd88c3ecc76e2a84f14ecab8ae2d70784b57", + "evidence_path": "evidence/20260720T180740Z-7f165148/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:738cfa6593fab108fa092e45c64916329930928a54aa3cb912689d76d6585249", + "evidence_path": "evidence/20260720T180740Z-7f165148/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:2d3f34cfdb296072fe68a10c3738081686898b61e77fa247f19f47069d6750f6", + "evidence_path": "evidence/20260720T180740Z-7f165148/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:03dddffe26d46a0484d9b55c0aca9b05c41b99342357761db351872975042f9c", + "evidence_path": "evidence/20260720T180740Z-7f165148/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:798a987bd976c45d0efdccc851d1823d9c5d5ab719064a0b0bd27f73602795cd", + "evidence_path": "evidence/20260720T180740Z-7f165148/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e6f0a6146bc5f5a1181713def6f0d57e62c2e9bd6cab7d306e2186b2db620cbf", + "evidence_path": "evidence/20260720T180740Z-7f165148/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:86b0bd43b137bab4f42db2472c3080c8133ce0eb898c7e64bc23010858e89ebe", + "evidence_path": "evidence/20260720T180740Z-7f165148/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e546105a9155e1d6d1d39617101af805afd929dc8cafaf62db3aeba5a9d2c108", + "evidence_path": "evidence/20260720T180740Z-7f165148/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:0c41e145ec1a5fd397efb1a733eaebf506d42b78157bf67a336e627f671e58f8", + "evidence_path": "evidence/20260720T180740Z-7f165148/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:f642992bbebed367be482c5f1bb8d2217050e7af9e84ad5663f67fcaf5eb187d", + "evidence_path": "evidence/20260720T180740Z-7f165148/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:95cb698fd4874fa22030583130ef1d0d8488bc9a2569429ce93c1f55de74579c", + "evidence_path": "evidence/20260720T180740Z-7f165148/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:27d824d4659051cd9ab47a447739cbbda5fecb2c1a8ddfcf85e0761e03975944", + "evidence_path": "evidence/20260720T180740Z-7f165148/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:33860a6773223a7daf9d66954d5c9d4c0d3193cb8e150502220d0f341c6f5012", + "evidence_path": "evidence/20260720T180740Z-7f165148/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:3c3b624955d47a05009d5098bf21308db021606876c33b10afd42b14f688752f", + "evidence_path": "evidence/20260720T180740Z-7f165148/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:0bb23217de731c547f99716fb892e48e8d63ee6f0190ea3b772e70316816f58b", + "evidence_path": "evidence/20260720T180740Z-7f165148/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:b7ac2a273772479d88ccb50b1e9c4749785145fd40c376834cc8b9e2775054aa", + "evidence_path": "evidence/20260720T180740Z-7f165148/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:174f9f808dd3c5675fc56e4ada4f3145015298b1713537d3b6e3f665bb6b4367", + "evidence_path": "evidence/20260720T180740Z-7f165148/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:87bb39a1b1f8ff0beebbba21aee38faa57d84115a44e9b0bbc86fd49d11fcd35", + "evidence_path": "evidence/20260720T180740Z-7f165148/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:7d93fa29830ff75ddba86c46dd5abe58ede84ff6e61d4549997f074da5041a43", + "evidence_path": "evidence/20260720T180740Z-7f165148/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:0c42fad31fe6af556bc0f6d58745fed2eb0fdf5ac07c08867511c7fae597ab6d", + "evidence_path": "evidence/20260720T180740Z-7f165148/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:d1218f207c20c2753011021c6ce991bd70132374ff96f34ff6af6da5ca3b98ae", + "evidence_path": "evidence/20260720T180740Z-7f165148/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:50341aba33b0b47c716623bf8033bccd0f47ca47a3cd42bbff99cfc96cdfc429", + "evidence_path": "evidence/20260720T180740Z-7f165148/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5106c850932bbc751685f28a5f3eed3c9c07b5309f8243b175f47e1fb63f849c", + "evidence_path": "evidence/20260720T180740Z-7f165148/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:717b85afb94198663eeebe7e1fb8b517d7e53e20a7a3c3a0da7daca545190131", + "evidence_path": "evidence/20260720T180740Z-7f165148/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:766ce789550fbe663dfa4f3c779d7b0f120696d2b05cdbe507dcdf2698caa9ae", + "evidence_path": "evidence/20260720T180740Z-7f165148/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:049be423cee89f9e56614cd7f30dcf68585174f31170494f18774be4b6fb7332", + "evidence_path": "evidence/20260720T180740Z-7f165148/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:97164b82773b2e6a1b9010defda1b260458abe6adffeb34408e2280039235d53", + "evidence_path": "evidence/20260720T180740Z-7f165148/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:512d8e25ae2c983251e2cdf898440a08a5a1996460c365092c9e153e60141ca7", + "evidence_path": "evidence/20260720T180740Z-7f165148/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run5.json b/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run5.json new file mode 100644 index 0000000..d6453a1 --- /dev/null +++ b/docs/case-studies/evidence/replication/m5-replication/ollama-granite-run5.json @@ -0,0 +1,520 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260720T180810Z-814178a4", + "timestamp": "2026-07-20T18:08:10Z", + "willitcall_version": "0.1.0", + "endpoint": "http://localhost:11434/v1", + "model_id": "granite3.1-dense:8b", + "declared_quant": null, + "server": { + "preset_name": "ollama", + "reported_version": "0.32.1", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:f09cae87b1cf01f41ae5199731f5a3fc64a2ed11d53fc1467284d45851af600c", + "evidence_path": "evidence/20260720T180810Z-814178a4/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:d30272c156455a4eb6f39cd011455dc3480535c2fb8dfd2e4f605a4bcc2ef018", + "evidence_path": "evidence/20260720T180810Z-814178a4/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:cee9370df44be12023bf30095c20cebcf44ca34f70c12f1db0c002867c8c7757", + "evidence_path": "evidence/20260720T180810Z-814178a4/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:8a9dfc53eedcefc69101b9a06e6e8df2db67ea8904a17a0544b9ce8b625c968b", + "evidence_path": "evidence/20260720T180810Z-814178a4/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e3dbafe4cff1268eca1ca4403d5164cd1606127714dbe917827168a58d0e4175", + "evidence_path": "evidence/20260720T180810Z-814178a4/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:71f8f07f91fa4b801d99ca20ba7faf8e69f3c36dc2e0ceba3ac33def9746667b", + "evidence_path": "evidence/20260720T180810Z-814178a4/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:830928feb4f87ff109e253a75de59ca1fa24738016c62096b92e20973452b095", + "evidence_path": "evidence/20260720T180810Z-814178a4/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f46f35802b97114eaa7296d89a216895a06795df07d47a9b1f87abbf87fcb0c9", + "evidence_path": "evidence/20260720T180810Z-814178a4/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6b0d7bff488de8d675dae8ec943a98c41770e2d228b6cffaf7c31a9d42ece85a", + "evidence_path": "evidence/20260720T180810Z-814178a4/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b6d48f368fa06ce4a0ed0d9ff8d16067b0e905e06de4c9004964b989dc47a148", + "evidence_path": "evidence/20260720T180810Z-814178a4/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2127734fa3dc2f26b32171021d0b29707cc906655b7415f89a283342ccbd035a", + "evidence_path": "evidence/20260720T180810Z-814178a4/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:02fe153729fa63f0487b4632833490ed85366eb6108af32b8cc1e024e969cb23", + "evidence_path": "evidence/20260720T180810Z-814178a4/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:897ee1bcf0c316643c604d089243a0164998a6b414a08f9e394d3a7b0b0c37c8", + "evidence_path": "evidence/20260720T180810Z-814178a4/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:cbfbadfc207750831d1512591d37a9a77f3b910f77956470bd1f8ee3337ed702", + "evidence_path": "evidence/20260720T180810Z-814178a4/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:1669aaed8ba2bf7732151466786d411476ddabcdd1084f10f0be4f4066d931c5", + "evidence_path": "evidence/20260720T180810Z-814178a4/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:c2cfa58468a8d0b8f3418074cfcee8fb2bb8e8b4cdc7c386c11721d774bf9c77", + "evidence_path": "evidence/20260720T180810Z-814178a4/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:747fe301bbf1a76921ec2593e75029556634d007bd764a439d36cf4450426dfc", + "evidence_path": "evidence/20260720T180810Z-814178a4/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:7ef44d4b2c7bfd0b8b82bb53fde1462a7174cf0f226cab36523f89bd57c03afc", + "evidence_path": "evidence/20260720T180810Z-814178a4/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:c847352ed54bac6948f6666588aa0b838a99e00fc6bdf85a1c48d97988378961", + "evidence_path": "evidence/20260720T180810Z-814178a4/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:66cc225eaf9982ebbb8caa05dc8b3dcabffa51864cae6cd8f4810998675152d6", + "evidence_path": "evidence/20260720T180810Z-814178a4/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:913c56cfae0e509854a67f5ff326fabc195435bbbdfdb0be8308bb8737681cdd", + "evidence_path": "evidence/20260720T180810Z-814178a4/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:c411cf4bc09086867e37703a7188f7f5d46c72a13da04cfba0bcc44c6c5b66da", + "evidence_path": "evidence/20260720T180810Z-814178a4/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:d0b222cbe0310a25d0042a71821910af3b4c620235e6f9c09b2096d2fb8337ff", + "evidence_path": "evidence/20260720T180810Z-814178a4/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:556548c4c2c0b71000bd287b90598e9038265bb798cd4c04a36430dcdbb997b2", + "evidence_path": "evidence/20260720T180810Z-814178a4/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:22d9b17eea33c57233876b2408829322433a15bdf0837ef2e68be2e05188dec2", + "evidence_path": "evidence/20260720T180810Z-814178a4/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:94ec1ab7aeca483630e415b31d4339d592caef86870d6105c944ae935a1968c5", + "evidence_path": "evidence/20260720T180810Z-814178a4/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:457a63d766572a32d815ebfd69e7213b7c0a30184945a67c43b5a4ac2610e40b", + "evidence_path": "evidence/20260720T180810Z-814178a4/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:94d1dac7df8e95c124694f570dfbdb13799375ee29dcc9b73fd575ee707afca1", + "evidence_path": "evidence/20260720T180810Z-814178a4/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:02a4d418ec46b0c5d3356c55417eac8b69888f484adadeef9a34d8935a35bb39", + "evidence_path": "evidence/20260720T180810Z-814178a4/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:d3d95696a2a0a0726c5a373223b0dc067a3482f14f36dceebc37c74ce47987bc", + "evidence_path": "evidence/20260720T180810Z-814178a4/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:0fa53fe7686ccdf2fc2860786d4297dd97bd58395c9d8edd9ef644fe818a6ffb", + "evidence_path": "evidence/20260720T180810Z-814178a4/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:1c5bda000cf3e5ec21bccc68c9c203a3e2fae73606c73f1659afbe6e6bed7864", + "evidence_path": "evidence/20260720T180810Z-814178a4/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:5e98f2d0cf8df836631bb86f55fe31c62468868d5ab1337876f36f2c85c2e1c2", + "evidence_path": "evidence/20260720T180810Z-814178a4/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:814e4b5f607c0290f0d492b98ce16a776924bc7118942c08115f3bae538142f5", + "evidence_path": "evidence/20260720T180810Z-814178a4/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:d204544abe2cc97ae85c6bc33cc52975eeb2a51f256e2748a84945b23b3c2276", + "evidence_path": "evidence/20260720T180810Z-814178a4/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:1f4fef643fc8a5c5a03ef8a8b87440630c4907eaae341e79e568d3856358cdac", + "evidence_path": "evidence/20260720T180810Z-814178a4/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:7df0cd3ed036f2dab7011547f531abe5e6e99dfe076f4226e4246fc5bebf7395", + "evidence_path": "evidence/20260720T180810Z-814178a4/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:adc383bdf9f3f310b7d63b48f6994f8608a268a668e1f937bbf1bcac6c79e181", + "evidence_path": "evidence/20260720T180810Z-814178a4/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:8375c0e158b78f231677fa6b41994e5800f4cf7bf110894385de976f3b655f2c", + "evidence_path": "evidence/20260720T180810Z-814178a4/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:5d34408e17adca28413d9e718693e0433fd5dd23b0525286cf17a14a29135b29", + "evidence_path": "evidence/20260720T180810Z-814178a4/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:79a9838d38df2701821043b234be18880704f0ffdf00b1768efc61ed0000b971", + "evidence_path": "evidence/20260720T180810Z-814178a4/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:2d2ed20c18b4ef0a0dffefd2b0ace44007ccec7324ba16c6186698b88cf442b1", + "evidence_path": "evidence/20260720T180810Z-814178a4/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:7f6076db837719eff735aa6c3ac409468b7f24c75850425d82d96d0c8b830bb0", + "evidence_path": "evidence/20260720T180810Z-814178a4/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:e2873c4c24d4533b1f3c8de4019abaf800e332bb31e3eb738664609a86e97cb3", + "evidence_path": "evidence/20260720T180810Z-814178a4/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:61b3cd75f32480b81d6d667390874d17633b1a5043291eafa343e0dc16165b29", + "evidence_path": "evidence/20260720T180810Z-814178a4/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:cbf9e5d220141de10b053faf2824b2eeb9ac12a8d4bf63117054aaea3db42af7", + "evidence_path": "evidence/20260720T180810Z-814178a4/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:8b2235b005a4f81147c9965ae49facf837207ffba97a06c1aaea97a369bde9e1", + "evidence_path": "evidence/20260720T180810Z-814178a4/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:be6457b4860056bef09a270f5c9737f82b242f3ed212dd34a4a0492a288a55d4", + "evidence_path": "evidence/20260720T180810Z-814178a4/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:66efc6f8a91efebc7525a3f92769274efd939d78082b4eb4a5ed9fa205b31b11", + "evidence_path": "evidence/20260720T180810Z-814178a4/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "no tool call emitted", + "failure_class": "unparsed_tool_call", + "evidence_hash": "sha256:907a326122b6292a36784362487f1e3d6011cb82287cb7a124756874ec79a27f", + "evidence_path": "evidence/20260720T180810Z-814178a4/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run1.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run1.json new file mode 100644 index 0000000..7054fc4 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run1.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T145256Z-5d3daa41", + "timestamp": "2026-07-21T14:52:56Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:dcbefa626f8a4e18b0e5ad5eb7a222e2a99743caf7fd1518ae066c06f73b3086", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:d511a847af884b7cb4f4f35005ca077755da0ee64f23c0c9f850e0ced2184837", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f931c4313053acf13114027e6f6c9850a28c4b7c2905bedec8639b7de464b364", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'first_value', got string", + "evidence_hash": "sha256:18871b2f3e5ef517b53a5df719b54cc2c3a8b1a752aece7ccf0c415295d95ab4", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:7acf55fad86a570ffd63bde946ea58a7a4a99a86834fc4d2e69664317b4da620", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:37c6056747ead26bbeb7e3335014d1404b04ab9a976281e8509c1ccb441e36a2", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:aa34daed9852544ca6dbd328dd35e956173d02d8a8ea8cc4c4ef00000d6fd981", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:745d71b2d87e59c72b578e15c31ca910882bf0330ff9c70ff22f02ba728693d6", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:1226efe74dfd370e44ae39d2cb6435f351b5663a85fb3391d476d4dc0aa60675", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:da955eaf93f62159a016a9cf859449cc2f06bbfbd24adee789aeeebe8c808b2a", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5b3a7c30612ac5ee64545cd9a0c7a3f74b3e369ac2462e14bd8d43ad22e709e0", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/negative-invalid-schema.json", + "retried": true + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:54ac0711ffe4e8c4bd48b4e67a183cebce04fdd0b13744653809a309c1813bbe", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:4721e768084eec8adb1f964fac8f1ccaa443c4be04135087ed2e94f1efd94bb8", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e4f7d70380ae1b7662fb53911d336917485da57aea04f3a40c67c295a3419054", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:300e20618935544997d1eb56bc7e4af51c441654ec8cfa834d2ecd38bf012e27", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:2d4eb94e68e4437a5abfd4b72d562f40ed4bc6f6c6ad0c55bb26ec1c3e111ab5", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:30248c6f0d7b26c1d8d78e753f531e0ce4d5e3c30833ba13ab024487f63d4533", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:35b51e32d231be9671d37a5e00312adcac0bd39f8efc35b5a178144ee8532df4", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:444ab08f005bd2f3b605330cb12c8e1cad28d1ff4e781a48bb64da2841db113a", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:bc59b1014a40b8919a81c432dde1d07dd3171ec03b139c035b57ddbf737e906f", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:caa358ac1eb0e9782cd09059b3aa87a501df34d7a7494e30e185e1cec5b9bdec", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:baf0ab6a164ad39fbcd1b394fae7662c7dd9178d3fb73f45326384dd9edbc704", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:77dc84face4c7a78477a9161f0f218f0d7ccc408521bc50f6bf0aaa605b9a485", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:b79552c31caf782fb32ff7127ef7613ab0a4b9f10a3917317bb46e3cc1a0e858", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/single-boolean-flag.json", + "retried": true + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:12f0ad330ca34a81df9372614c44028d7a53e1a662354f0d986cad3bc23072cf", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:26e54124ca4b0bb70208b411710ee760cdfd151bc3d1abefac56484125f1c7cb", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8ed5c13943b9d0e997f6c5700bab863e1246299747f9c225873e67e4357a1e37", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6dbb9ce2b02785b4bca11144c7c92b918393f5e6cda9ae1662eb64aaa9cd2d49", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3414cf286a650b34de5f609f40878eede5d92ba5a2eb26811fcc2ee8e7bc1b6b", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c54671f60a08dadb4aba9f9746ed723e4df1381d7f629f92c98decbc4b9d66e5", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got null", + "evidence_hash": "sha256:1e79ae699145ca3879d6927d3f9d1b9a3c741cc38a1445ee0b1557570013ee9f", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:065eef15084d524ba42298a68695904881e5663445c436cc0c2fc5f9bb78554f", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d180a0085dc22a0201a8b857b32067762de67d0bcf43c17e9c89f0e2f942e7cd", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6ab3a412a6992635fa1b25c519547e4eabac29ada2055b840ebaa84c1a7cfa6", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fa398ab020a41e5a7de0ab3264c9b66d3edbb80df45ce100fe666633c8c0521e", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b54f9b7811f36fdf79a8fb874049f34a60fb84938f2d588e322fa21c73b03fdc", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a05b35d7b8caa04be2b4ccf727667b4c644cf37ea1fccbf859316cc27cd9dafc", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0e034b06279e0082b76418b56b8c158af9cfec545221bbbf37bbcd18e9035712", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2f7b9ee15478fb36a5ac6d6093186a12b6077f0ee52301e18ce4ec4e0474ec37", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:19484deaffadf9d91e96dfaf209cb62048e5fd69af11017d94a5354b1756be17", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:e9727d08f17ecb8f51ff90d8377ae73253651ea6e0c1c4f7f43ab927e0bd7ecf", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:747b50681e34ff208b7de9593160abf20ca61c81a444e45fc5e58013d449018f", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:091dcdb4ae33e29d6c6be4b30b668828f097054f3bf9994711ffa21d7786d329", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a964b15083e34ee52c4546ca1fe64b35337a0061fc3b44db47ea4c26fc309959", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:8e654d598239aa55cfa0c321e8bd0afbd8f942bbca33cfc9ce368b8772602045", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d3db89343d605bd4c0d979bea8e71927a1423ce22997dd9759b20223abff00bb", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5efce9bb9c5d5cbffd813d955f7a0e4eedf9c5fd2e8fee9143e7a40b90eba3ab", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:61b5b743c445b36a9b0bceebaceb1d86e6d79aa46a2ad1f76acddbf95c126f53", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:404e8945b167bbbd9667f96774ee87bbba6e489975e2fb4d8ecb4aeee501a94d", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1e3af43d34d7a35db6f082ad634a27b62009827b9b91dd652fe83bb2a8a921ce", + "evidence_path": "evidence/20260721T145256Z-5d3daa41/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 25, + "failed": 18, + "errors": 7, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run2.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run2.json new file mode 100644 index 0000000..0e4cc5a --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run2.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T145336Z-3a1b7421", + "timestamp": "2026-07-21T14:53:36Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:45d6ad456fb8cc954a716ea69314a89c4d95356316cfdb861d1977aaebd4b48d", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:93d56b4179586e00cf297c0000f6274c6928048b32ec5f403e23a043a9c58dd9", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:d4617288959baded1972cfe2655fb12379b04b99570e6635fe255699fa605cb1", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'first_value', got string", + "evidence_hash": "sha256:2c2d10cc4ead990e5e03de0399986be8b2944692954757a1610c55b14a20c4d5", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:77c3bed18714a5484dcee9b7d5de3e52d26401dc2b1086ee8ed26d8b0c42720c", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:a884c63c9e055094be8b754ce3c714a4b8f6ac862be118c2fd61fd05765ffd06", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:04ad608a8491641aec7c09dd49d84959a1b8cb0fb944f52c897f632c6c5af486", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:ad737437313138a81983d9db7424751366cb9b2be0cbd42c22cda6f1b9f2b402", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:cdd39f501b640b479301cb52b084fbbc4d780940d24622f5f4f3f301c5a5fac0", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1f2c4aa9139e9e0ac427a564fa0f48d1e07a7f4645d7376101cef454b8c42afc", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:1eb4fc69cf7aa0010ec1727a9e3b58912256d939f8b7e1ca7e609ab6d0a16938", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/negative-invalid-schema.json", + "retried": true + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb4396ff93847035acd399cdd4e150c3c984d99f97e6ff4f6e2e53065c9dcf4b", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:6455fcf6cd1dd65152a5120a163f10cfafa599578bf1dc438b3d7d2c8fd2cab1", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b64690add225c624afc5368066ac70cca60e485812ceb950354c64ed569dfed4", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:59b0c7094ba5b58e5f63fae294dfdf430f761543e5478df598431db54def7ab7", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:3b0399d266eddc91fe75dad46c9dc292cfcde02cf0ee1151136884e8b3d12569", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:23e777dc9e1c7b70de01e8df3ca09c0d46f4bc614d077a11b47f7976d0674034", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:5c41b119d057ca0b6f62984046f00499bf52361c6377aaf70ee42811574a5bf2", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:c2fc63de0c50ed3aac794d2fce7034976bad8616f8aa391938db18f7224f2785", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:cb65a1676f1614a5ca0f613c64f65186ffdb65811a9f9098b08bf07155fef1ea", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:0958278a4931536efe3cc308e0fd2727cc50077e4b3d1e969ffc553443ea8f6e", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:7fe0a157fab557cc6940e1bbd81e0c2372adf0b8dbd9bfa04faf2b63d0a64791", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:542514828b3aaf39ca0f51a24db39c906e3ad754c30e506d8e465c45d6592282", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:6b73bf24338dc49c7c22be29231509982324474853399130664c6dd472bf76c7", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/single-boolean-flag.json", + "retried": true + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:61e106e1fef7187a110ed6e644ff2f9c9e8d046967b277d3572a0617949196ce", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a4d906b8fb733af5bbe5bb831e99d275f82c18d0549162e5877e715e8bbdc982", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dda18801d08d392443300e966639f0a061490c6fe8f63edcbebc92b362613bd5", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:684ee063dfad74ff90a15b9d917adcef16335c43fd033ff242799101721043a8", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:402bd2c5b6a64178316c5ba6f1ac3c0ff0313244ba4cc4cb095dddb44d323212", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3898934b4c700ae71f7a2cca9efb818238883e13cd9e03f9c4586f9715945fa1", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got null", + "evidence_hash": "sha256:2f8c5c65e11ade5bb103f9131b7b0ace68702f1ad0dc71a142f865f94bdc8ca9", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:64eb67e78e852af07b1eaa6e51b8c5244207c446a6d908852e7c12bd84cebded", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cdbe488cb55aa097cd247a41b9f33c14446d4c05f1e8b7cfc792ea13aae6874e", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ebb2f54f75b6aba4ec0c86e9e0545214cfeff74b272255269b37f19e7aeb7e5c", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:23009490b8f3e2240c4a94020d057418f9b87e4aa94577898b32e9c028e2286d", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9637d14ebf480ab21af29bc369df6c052f3db59dc73f84b9b555f1acf77c42be", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:be6719f2ad26e0a4fd5d802402b8ef86ab664a5964765d847a55a20678453b15", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b46e84494033e843cda69581cc393457b3cb2362fe1922c21dc7b421b66fec3d", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:314f2352bd36ff4e78f49f7948ca2a5501aa77cf687e12f69dc24782a34f490e", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b12dece82f9dc35839e0b0a15526baf8f04cf78dd70330dc58c99d65629560b5", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:2521ec80da96bf60855c0e74e039f4bdcd9d469c8d2945640b0dd0b71ad48c93", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:55918b787de26194504bbebfed0c46918c621d983399de0b0a16a5e61ff69264", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:55038866bb266943cbb315f32cbd51b3f1a7a31d0447baf21e446a7df9273662", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f9b3c7540c07021112651b4ecb007fdf798025ba0cccaf697e92c6adbde242c8", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:774e025a2d40c95be0d0035c4e68009164cef4b9ba505d7863101ef65df2bdd8", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ef9e254029be860e6bae7374ec429b2c3cae3a6fbe6d071088837afb9f76abf3", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:835a043c6909f9434963de2ee29b8814321927c1511eff30358261384156c875", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f4c6b9d394d530c539928c06e8efe66e2b12acb88dc73f8aa2a6e76ac8b2a3f", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8569115a36b48030d9e7e9f87f925442ea0f2c08a8198238978af75e480c828f", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8e78afd67d2fc0a6ad7257a69adbc18e5180355cde4e0f753484714c8291917a", + "evidence_path": "evidence/20260721T145336Z-3a1b7421/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 25, + "failed": 18, + "errors": 7, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run3.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run3.json new file mode 100644 index 0000000..644f644 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run3.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T145413Z-507161ce", + "timestamp": "2026-07-21T14:54:13Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f50a3edfd75d0b247dea26d76b0cc505ca5ccc3845fd05f88c24f1853060c008", + "evidence_path": "evidence/20260721T145413Z-507161ce/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:c41a10445e167fa0cbc256145b7f98af9a691c4c10764858e1be5d40e4f709b7", + "evidence_path": "evidence/20260721T145413Z-507161ce/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:1b6580f5d4d67dbb9edd45d30436142a616b2ac100af69dc8c32e6e9e2064566", + "evidence_path": "evidence/20260721T145413Z-507161ce/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'first_value', got string", + "evidence_hash": "sha256:bc92a68d80ce823a2996000803cf5b65429ba48bb9d7b182e71f0f486fe58780", + "evidence_path": "evidence/20260721T145413Z-507161ce/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:a418949d8914a5813c2ebc0b63195de78f842811c77b8e3b2a0b1b573f0c0d5c", + "evidence_path": "evidence/20260721T145413Z-507161ce/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:cac39b9c5f86b99f8bc5a4a52f0369761b6cd130ca0945bd48700bf002eab955", + "evidence_path": "evidence/20260721T145413Z-507161ce/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:55a4e2bd585e24046baa80f2f8bb26f63e861db5e41607cba573afc060bcfec3", + "evidence_path": "evidence/20260721T145413Z-507161ce/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:983f654b3c6c791e6576746abd3fa8c3671dae209926724c9da1561efad02c37", + "evidence_path": "evidence/20260721T145413Z-507161ce/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:e6a27f0c08958f1a4757ab33a2ddf0a60b8415a696b7af408f2267b6819feb0c", + "evidence_path": "evidence/20260721T145413Z-507161ce/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9499cad1c4fb1bc3ef4e5e26a386b4a78a69ad7902b32bcadf87f9368db7936c", + "evidence_path": "evidence/20260721T145413Z-507161ce/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ab3c5fcca9b96b4b819d015e06f299ca53079ebefbe4ba544cdd304e8fee32e8", + "evidence_path": "evidence/20260721T145413Z-507161ce/negative-invalid-schema.json", + "retried": true + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:474194a4d63876399187510860abb24b1d40e439ac13b13a546b326fe848f180", + "evidence_path": "evidence/20260721T145413Z-507161ce/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:a59aca6c26de71f4e8d6b4b6015d22fd0dab612846904cc7a8bc4af5b2da2a23", + "evidence_path": "evidence/20260721T145413Z-507161ce/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:54ed15c1c6a0f2576918b358e6b408879021641398bbc1a48226e021eb034b72", + "evidence_path": "evidence/20260721T145413Z-507161ce/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:9e3ed4919013468c21b179bafd624b0557881bbfd2068f22ccde0e449b96c20b", + "evidence_path": "evidence/20260721T145413Z-507161ce/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:f654df20c479894729d3884c2b06471fbd9e22b50f2ff030cdc3484cd13857ba", + "evidence_path": "evidence/20260721T145413Z-507161ce/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:5aa2963362ca83b19fdb385d72d8187fc28ea24ff8398e7c1004bdde068f04a1", + "evidence_path": "evidence/20260721T145413Z-507161ce/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:488b1617c39bd65dcf90ba758fda40c4727033db715195da204bf27167cac6e7", + "evidence_path": "evidence/20260721T145413Z-507161ce/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:551d1c6576a5b1c2e69464e05111a281e6574a98788a4cd94d146d609d04f393", + "evidence_path": "evidence/20260721T145413Z-507161ce/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:90c93c4d177dfb333d4a858c13ab750c1ce2ed60ba55e00b8cb7f0f8dd749b63", + "evidence_path": "evidence/20260721T145413Z-507161ce/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:ca7768b23936072d036249723394ad4b42af46b6745a75cff2b8afc4fab82aaa", + "evidence_path": "evidence/20260721T145413Z-507161ce/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:fe09089cabcd7f14eea4538991f02a69d419d30871a8ed9f783c86821095e79b", + "evidence_path": "evidence/20260721T145413Z-507161ce/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:8b84a4681cee851e3aa39bd7212fde0d19d24e4838fc92bf702c5064166a9914", + "evidence_path": "evidence/20260721T145413Z-507161ce/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ed4e40c9c88dd088a9018a077e70ed7910126c10bb32d3c3bc22fa018c21b6d7", + "evidence_path": "evidence/20260721T145413Z-507161ce/single-boolean-flag.json", + "retried": true + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4c1575e855ea78bedc4943d873ae60f3ec447375a2a18ec503800de618853c44", + "evidence_path": "evidence/20260721T145413Z-507161ce/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:965b8dffe5cbed79c84ebba397f962b59e25d4f76fb95bf8ffcf99c0457f5c01", + "evidence_path": "evidence/20260721T145413Z-507161ce/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:edaeaf5531bcb19673ab3106956fb05370943ffe37002b98c9e970b1bcf63aff", + "evidence_path": "evidence/20260721T145413Z-507161ce/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bdc1b6df8062278f50e89267696ceb3a058ccccdcb849a0ecd5ee7869c40f27d", + "evidence_path": "evidence/20260721T145413Z-507161ce/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:754fb03bfc30a53d5142be6eb680fb3d87a8c13987d94715b59f7e6268ba5975", + "evidence_path": "evidence/20260721T145413Z-507161ce/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a4261d38fffa2fb002aa9f73e70ffe52f6a7ee1fbb43cf1eaa10037b504e4ce9", + "evidence_path": "evidence/20260721T145413Z-507161ce/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got null", + "evidence_hash": "sha256:1fabf08fee0b34e63d533c3c496bac94074b224fe902cab337b3edba94ece6a8", + "evidence_path": "evidence/20260721T145413Z-507161ce/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:461943fdfd61b0124b69076522e9d2759e2b4b9ae1046e4ac6f0e117295bfdf0", + "evidence_path": "evidence/20260721T145413Z-507161ce/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ba845eaa59fa3a6242fa3dabd63fbe2d4ca94ee3ccff7ebe7f46b97f06396158", + "evidence_path": "evidence/20260721T145413Z-507161ce/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d409a25700e9441cec0f6fbd9289d66c38952aa6b4196e1a74aad001bec99899", + "evidence_path": "evidence/20260721T145413Z-507161ce/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cd0db0536d471f2f8cffc7dfa061f8bab56f7816d157caf6a435d3f055c0e2f7", + "evidence_path": "evidence/20260721T145413Z-507161ce/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:14ae9b1b36a22c418c394e3dee89e33fe74d70a1d593a8aca5ddd241a12158eb", + "evidence_path": "evidence/20260721T145413Z-507161ce/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4d890827b813b3d5d3c3c6dcea7bf2f1d91ca2b606cad71f1f40fe081b4230ad", + "evidence_path": "evidence/20260721T145413Z-507161ce/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0028f0c20c58d9fb1bdf9c7f0fa6516b287d2c342bc7f606fbd975e1f67a0aab", + "evidence_path": "evidence/20260721T145413Z-507161ce/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3eeb29dd4abf8aa0dcfa2e4636b96e8723062d2607e3838554957b1223513cda", + "evidence_path": "evidence/20260721T145413Z-507161ce/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bc0445dc75c579ecb4e50cf88e3d99d499fc1baa08291ade27522c23ea59ee3d", + "evidence_path": "evidence/20260721T145413Z-507161ce/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:02717e149c00b714e5f3c29b90beebe2c0bcc6139ed83352c60559dab463272a", + "evidence_path": "evidence/20260721T145413Z-507161ce/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:db107bf7152edf7bbd5ab5e4b65c92f4499860e04f1a5f1815f4fc8c476890f7", + "evidence_path": "evidence/20260721T145413Z-507161ce/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bcc95e3afddbc3c539a31a581f428cf2b14e57c4b71ff688e3c3f9d448b93e61", + "evidence_path": "evidence/20260721T145413Z-507161ce/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1bd3fd278f9794183e466bca292d52d363a03ff36397dd76ab5bb9c72cdc97e2", + "evidence_path": "evidence/20260721T145413Z-507161ce/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:262d23f9696c31d4c91efc534c8cc9a0a15e788dd5dcb6b0295099298efb55ad", + "evidence_path": "evidence/20260721T145413Z-507161ce/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1301ab9fd7d5b627cca9cfb1204b068ebf0063a7eda2dd9679863eae222d0a63", + "evidence_path": "evidence/20260721T145413Z-507161ce/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9373794a7fa963118624c1d1f35419e886f30d13f35149d66cc415b58222707f", + "evidence_path": "evidence/20260721T145413Z-507161ce/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1b9ccabeafa6634471bf7c60feea71c1c36e461996d98587adaba262225e0735", + "evidence_path": "evidence/20260721T145413Z-507161ce/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:319384c82c88df3b4537e7531ff955aef076816d5a272c1e904bf71b7f8f57fb", + "evidence_path": "evidence/20260721T145413Z-507161ce/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f4663c6201266ed8333ce6ac9af97336b9f9dbea952ca3d133f6830f5174c6d6", + "evidence_path": "evidence/20260721T145413Z-507161ce/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 25, + "failed": 18, + "errors": 7, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run4.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run4.json new file mode 100644 index 0000000..fb2d8fb --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run4.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T145449Z-0a8482c1", + "timestamp": "2026-07-21T14:54:49Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:022792747b1bc144a6269bafeb0dd69dfa07a3412f88934c0d40b2d3d02682ed", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:b1a8bf6d324f007c0f394b586992ab7dc0f2b491f4ef24f78508fa9650454852", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:80472bf752068d0c08fe21e19e817d10ffe241911530419b1c2fbdf35f502628", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'first_value', got string", + "evidence_hash": "sha256:5cf2b6f6827ef88cfc4a2e16835058cd2baf68178c2a6bab8f5b61dbf28df50d", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:4a354df40b4182e10e67ce53af0b62260550f4194e68665bd5cd12a66430bf5d", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:94372706e04a49761e837b378a121009a23f55d921358dcbbddd1a28945fdd60", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:95e5edfbd4f9fa1766abbb21364c01b6e14ea4b3dfa919085648ecd1ccbfa3cb", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:bbeee37e0e5db0e595dbe7e86c120c40d267b69b19142bed5dbafea5c9e9d07a", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:5b2f08fa7dde550d68693e4764ace4506b2b61f6ed1214a0a6a72ada4603295a", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:049ebe555574c5a5e580c70ca962bd0a824884cb8c74c7e187c6661f3c898439", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:83d166b03275d648a4c771fe3357a57c5edab97e9cbb15d2b00d6f3658c0e234", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/negative-invalid-schema.json", + "retried": true + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ca5e7d843eddee3e335a62dc460eaea1e41ade37e2b35661573308cfae8f98c7", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:c91b3862eb2d72adb66275609ffbf3dc517927c823b1c97e0401a4d3701df464", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b5f7822954f577a6a4af870db065bfe9ddfcac3af91b15acda80f06a29f28789", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:a04cbe5e73d5733492489cd24bd7db788e30ff744db8d151f76f758d9947bb9d", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:5383faa7b5a7fde5d930e9e2c215fcbee1e1811617da1dd6df248c7447b87245", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:0d5473599266066d88333504574b63d8863e3a0db933c45d4136cd96f2027fd9", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:ce3bc86378b9d35cf5627fc7f17a25355bcfd343b4a46784691a43c2980e3db5", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:2d60195dcf50963c4ea4c2c5d1be952e33e43a2d7ab802cd86d0b72b9056b576", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:777f45286b6fa58294fa2c9239abdf6e526edae753fc6289991cc7dd24820ff8", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:9b19ac309e66dddd80b4c82c6388607219a5e6a38de894f867f448a2fd0d809c", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:000ba7bcfa14aa1c31b394b0781a33aec2ca4e493b6797855855c28147d4c43a", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:b78abbc04d58764ec53f3028c2ddc46fce39525214e0484f2e8acdc41bd5fd05", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:b5a664d8261ecd9fe20f03557784de2f597943176e47964170e80de24935cf54", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/single-boolean-flag.json", + "retried": true + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a8f0882499d47b41dde96dd4b0dba9175ab160d298a8769666c1f627b2dd4c12", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5bf287b66f3906a2cfac35d63421c2e0e735191c2c4bdbbe33cdef9484250450", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4ca84e637a05484694a315f1daa4e8bf9f47067e4ab9f9a9d2a382e985c1f6f6", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:724cc645d03bc65e8f96b71ec0aeae5a6d9e7d5009fd375185acc79733c64c3d", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1cb2e7d7796460301fc56fc479d63ad40ce929bad14e855fa876c0e9d624a510", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c988b7b3cc31f312c7eb7c7839b211d889d585bef39e1588e48c759f8466796e", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got null", + "evidence_hash": "sha256:7066f124f80674509c5d2d0602721c7c370af86f81e98880f7a62b1a0a3a57f7", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:385780530d05a13f5580925d6d19c0ce962e256c8c4d2b87854640672a09718b", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8b1999a776ff35168958443162b2a8dedba3910fe21e35114724ecb6e35c9c76", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4179a6a53721b67248c706c5bc299d3b9b53546cbee1815195a62874340e5d16", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:173cf959318e853c22f8847daeebad43dcdfa251d1c7c7f3d4b071adfc19503a", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:be73603c870f401c9ef48cf85f32ae3b5879b4134ee3e15451f2fbe08c6e4d49", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5e1d2ec8c27c94d5bb3059cfd8b6ba3c48e3871783e89f34edae090b9fb601d2", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:686c44d79731058b7cd4714d1f9dd268f20dea222cceed5299cf68aa7e3b93e5", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8dcfd1f5f33638951e50bdc3e0022c84c75c459ac723382da4c7f8e0901b0782", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4c8c0063df80d5a021837afdafc23ac8394a95e472b5e9f5165d5ef448a6755b", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:8d90e703546cc616ef6d59b773d5a6731c35ca07f8a1dd9e4b966767a64ff6fe", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:0e9b891d9e25486d885ab8feb38ce6dc670ea86f3a32817f1c6df4731343ad58", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:88d7d02358d46c4a1dbb43b565621d4ebb2dfd98f0e60b46996a07e273144952", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a59bd455b09a26e979765be0237e5fe706e755e3bafb59729522e7002be2cd38", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:77e9bd8d227627f931cab233ddf41abf7a88bd4a04d62af4761719a1b67e4341", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:30356b0c4450f8773ac33562fb9687d0771161c933d82c2228e67499c58212d7", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:45552d215f8e00ff9286a562dcf1d7352dc07ff0b52b0ef0824f72d9c796adf6", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7b71a05e5b3fd74d6880f1e9f42cfe006db46f38a0ae4f795bccf7c649b337be", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:24300e54538cc8d7db71d8f90cfcdbf41e55b391951e9e093759601b295af5ba", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:770bea3fc1ec26cda5daf69eb0c7e4f45e5c77170771a535160ff8ff3e1657a2", + "evidence_path": "evidence/20260721T145449Z-0a8482c1/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 25, + "failed": 18, + "errors": 7, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run5.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run5.json new file mode 100644 index 0000000..7ad0d15 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run5.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T145525Z-468f30b2", + "timestamp": "2026-07-21T14:55:25Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a4a8d57303714d7e9f0d3cf5abf17e43a69ea9818837a2064fe761de5d7b28dd", + "evidence_path": "evidence/20260721T145525Z-468f30b2/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:1dde1a602b8bef73818bacdb99c29316a14601e1400da07d7adb826544ba47f6", + "evidence_path": "evidence/20260721T145525Z-468f30b2/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:706cfcb22b82786d8e0373568c5d57a1d334bdd72720930aac20495ba31b9987", + "evidence_path": "evidence/20260721T145525Z-468f30b2/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'first_value', got string", + "evidence_hash": "sha256:8c92698c5670e80d6f9f5b7eab4e7fb7a0974c35897ca8f62343056b29591811", + "evidence_path": "evidence/20260721T145525Z-468f30b2/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:e487ffb4ae4d002e9cfbec3d3cae6d5d941692b4421d37f44e170c319eede6bc", + "evidence_path": "evidence/20260721T145525Z-468f30b2/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:9864d1398bde8663f31ef6d887e85e03a4ce98f8fd905c16f70381920e7a2f99", + "evidence_path": "evidence/20260721T145525Z-468f30b2/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:b75d40c4cfb8e63f6075a31515cfc81905c8614e8cd60511fb81a8cfb9a5bcbd", + "evidence_path": "evidence/20260721T145525Z-468f30b2/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:26e853649ab056cb502b0bbf5e9830db34da136542a1ee3a00b328e3e24d558e", + "evidence_path": "evidence/20260721T145525Z-468f30b2/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:3de8632c33d581e896daab8a7710f18867e8a1ea464bb3f1818d3c91335c929b", + "evidence_path": "evidence/20260721T145525Z-468f30b2/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:879f2bcfb8c0dee78e98e7749ae15c53955aed05837f8e96603d2db15a17d113", + "evidence_path": "evidence/20260721T145525Z-468f30b2/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f5bc3f6cc8756aeca77038571ba519e99cfa1261e7ea443ed9ab6ca5c05a3a88", + "evidence_path": "evidence/20260721T145525Z-468f30b2/negative-invalid-schema.json", + "retried": true + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dda9320b9ff7aef2a10623af2bdea8653d86a41eb7a4c69374bca4efdce4b2a9", + "evidence_path": "evidence/20260721T145525Z-468f30b2/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:d4dac379a98ad672b7218f49eacb0a1af7b106a81a07aba4e3af2f80a88ae7c7", + "evidence_path": "evidence/20260721T145525Z-468f30b2/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:789e630be4d4b74a8b7cad59f263059576ebf27cbe25c25eb302dfba1296a0cd", + "evidence_path": "evidence/20260721T145525Z-468f30b2/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:98cf308b266a61900676bfc94295897813b2a8dfe7f546b68f8a18241f670587", + "evidence_path": "evidence/20260721T145525Z-468f30b2/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:d257700b46b8d86dcbb0b5eab358878105b7378b3891fd7f4004da61f98a5538", + "evidence_path": "evidence/20260721T145525Z-468f30b2/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:431ad965e4c900ca8ca5fae39056b8a1c4d4956ed9f5cb46576e9cfa788d58b9", + "evidence_path": "evidence/20260721T145525Z-468f30b2/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:4993e0a147800bca455bb87665e8765bd96baa174bde4fdb7386b8f6aa290ac5", + "evidence_path": "evidence/20260721T145525Z-468f30b2/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:f6a9089402d9d5310c102ecbb7c3578c973b92ddf714c19904caabec94216f1f", + "evidence_path": "evidence/20260721T145525Z-468f30b2/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:474257e22b30b7820c0fe815d86f18b09f0131e75c6dfd564096a2057f6e3f42", + "evidence_path": "evidence/20260721T145525Z-468f30b2/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:103c221a2617c509efcf477d2e4585fe53087d7ce029692a43385ef25b3aadec", + "evidence_path": "evidence/20260721T145525Z-468f30b2/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ffeb83cdba249feb2c7975e4b9164f9f1c766886e01eac2a66ab0beb1b75d746", + "evidence_path": "evidence/20260721T145525Z-468f30b2/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:f1ca4f45aa234d3f310b089819749be5db9a2f62c830dfa1d05a8b61050b3474", + "evidence_path": "evidence/20260721T145525Z-468f30b2/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:162ae80d0f6fbe2b6d024ba129c385d9787af484fe84953f308be67296883c00", + "evidence_path": "evidence/20260721T145525Z-468f30b2/single-boolean-flag.json", + "retried": true + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb7da3b776ae653d00eba13a72e598eb3b937d8f2df5777b5f1ed1d03709847b", + "evidence_path": "evidence/20260721T145525Z-468f30b2/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c77396046fdc18115a3d1967ce3c78a122514c59cd205d0f814c5548a9bf6ab0", + "evidence_path": "evidence/20260721T145525Z-468f30b2/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7313ccce3769364385f50f99de675d8261255403a7a455eebd0cae3bdf94f1e7", + "evidence_path": "evidence/20260721T145525Z-468f30b2/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a1e62db13a9171234c9def8b8bdf2fab90ecc695df96080ba6c6052a8c03b471", + "evidence_path": "evidence/20260721T145525Z-468f30b2/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f0d146b6175c0144b2f726c1832a3a0665ce0be916e85453b1e2bb0b0d04f261", + "evidence_path": "evidence/20260721T145525Z-468f30b2/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0619513ece6793f58654fb50ba02e1c00665de494a09e71462c52a1cb724addb", + "evidence_path": "evidence/20260721T145525Z-468f30b2/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got null", + "evidence_hash": "sha256:4f627dc0a839c2023c224e6ada3bf4c67c6206d46df2f1778e1021eea92ca461", + "evidence_path": "evidence/20260721T145525Z-468f30b2/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:78b10fe42e6dc2bc7e809c710bafa914afe92c851dbb9bb9635033eacd666f5d", + "evidence_path": "evidence/20260721T145525Z-468f30b2/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c8a3ed795082b3161fe4bebdcdc7010cca1ce47b452b0005cee28919e6f43834", + "evidence_path": "evidence/20260721T145525Z-468f30b2/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6efb70068ca8957f7915bd40eaca6234596a9d6448fc0450ed03b826b9a64564", + "evidence_path": "evidence/20260721T145525Z-468f30b2/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0a2c91ae4deed037728bf5f0b30b5db8590024011d7a2bf6cdae535afc456589", + "evidence_path": "evidence/20260721T145525Z-468f30b2/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8544803adf63fe786f0b0dfe06f806402a846dc756059d3513ae9da53ba62158", + "evidence_path": "evidence/20260721T145525Z-468f30b2/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d058e335cdc9b9548de2ec6907151014a2ed22fa3a39f8b7065b941f5715a8da", + "evidence_path": "evidence/20260721T145525Z-468f30b2/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:06b3126eac5e09c018601b5ace3bf823dbbd336057a1445dfcbca569c52c3453", + "evidence_path": "evidence/20260721T145525Z-468f30b2/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bcae8a8e8a6cbc19b5a40e9cdf6e43e23918b48cb279c5a0413866035e5bb190", + "evidence_path": "evidence/20260721T145525Z-468f30b2/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f6029a98aea4deff53679a6844c3664308a69121ecfc06f07750564cfab38153", + "evidence_path": "evidence/20260721T145525Z-468f30b2/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:d07c02e96489b4fc5be1ef8de143874aa6b7fa0c70fe08c39216c0a40694d62f", + "evidence_path": "evidence/20260721T145525Z-468f30b2/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:321ca21bc53a0c5123ee47d3fe284449a7e951fcaacd18935bf080899ba2189c", + "evidence_path": "evidence/20260721T145525Z-468f30b2/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:51d98faa5732de30cf2dc14089bc2a48ca7ce03e9d76267c05e822c31226ffa4", + "evidence_path": "evidence/20260721T145525Z-468f30b2/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8768dc66552ed72afeb0ddbf08f50bcdcd51780fc39ac464aacf5cf73482b138", + "evidence_path": "evidence/20260721T145525Z-468f30b2/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:2652136e46b18c6d1f3c81f74414e5651814692bf984db51f83afd7309102c9b", + "evidence_path": "evidence/20260721T145525Z-468f30b2/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:06f7fa91e72aed468709c0053886121862a1a1b2866b9eb79c50771f440438d5", + "evidence_path": "evidence/20260721T145525Z-468f30b2/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:30562379500e42df7bd35bcc0e54953dbd6ee079311264645ab9ab683c23bf6a", + "evidence_path": "evidence/20260721T145525Z-468f30b2/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:47c17bf63f7ca3ab734361058a85039b2e4e447cade4d82f3d6dee3a304ea60b", + "evidence_path": "evidence/20260721T145525Z-468f30b2/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b14f8f84f0e74d090485942b673f6c877855f3e32a474a265c053222740a9db7", + "evidence_path": "evidence/20260721T145525Z-468f30b2/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e5b983a1008f380621b5d859cafd53e3ac97308242094fb96dccc13d26a6e646", + "evidence_path": "evidence/20260721T145525Z-468f30b2/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 25, + "failed": 18, + "errors": 7, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run1.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run1.json new file mode 100644 index 0000000..e4ac340 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run1.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144950Z-62e8b294", + "timestamp": "2026-07-21T14:49:50Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:1e98d0a20ef7530a736363c60c43c786cd6e96fbaafcb38c8f389f279d2fa986", + "evidence_path": "evidence/20260721T144950Z-62e8b294/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:d49ccc8b4c36849ff90f3c87dc16a8d2b7b4848d8d436273f9d8d4fe1d508c44", + "evidence_path": "evidence/20260721T144950Z-62e8b294/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:3b7feba8c64ce735dc089e262d7ba117ebcf40b516b815d07866ed6d67be899f", + "evidence_path": "evidence/20260721T144950Z-62e8b294/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:fad70839abd8aa88291ae311c9177e6986733f692db444ad689f8cd4e062645a", + "evidence_path": "evidence/20260721T144950Z-62e8b294/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:2b19c5d906f277ec4442e26d2455c1f701f90ff7f6fc1af73bfcc28965986d78", + "evidence_path": "evidence/20260721T144950Z-62e8b294/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:c9426563ccc0c711c9413d4de74e288f3d418a555891704661ec7ec1d9537f16", + "evidence_path": "evidence/20260721T144950Z-62e8b294/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:c77e4fe93b999e67b751d45a9d3a2200f832403da1479fb761edee70cc967c52", + "evidence_path": "evidence/20260721T144950Z-62e8b294/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:d20d456f930acee2b6223900916e44307b713f4dfef94ea25f80075ccac5f84d", + "evidence_path": "evidence/20260721T144950Z-62e8b294/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:e1c74de80a4062c1a4c92ad2d2a6d9e4e16e800555f21ecd7f4105329722b7c2", + "evidence_path": "evidence/20260721T144950Z-62e8b294/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de85fbcfebd344b8ec16cd0aed5e8552083b19e417d106f5bb36db8466350cf7", + "evidence_path": "evidence/20260721T144950Z-62e8b294/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:ea5b6b88b9c274f10c4016204c6f85b7f8924cd3a77242290116558f69dfd281", + "evidence_path": "evidence/20260721T144950Z-62e8b294/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0ae8cb38b288101f5936a0872259a15feff967e501490a7e628a56545c8b9648", + "evidence_path": "evidence/20260721T144950Z-62e8b294/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4d8e9cb2808a279a6fe9bb7568a0dc952b027c3a83a524603c4bfe971e62d8f1", + "evidence_path": "evidence/20260721T144950Z-62e8b294/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:14c0dcfb6ab98ed0ccf63bc844ba2377c6a41ce9b6d44ca88c7c096ab32f815c", + "evidence_path": "evidence/20260721T144950Z-62e8b294/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:a9b9db88509b5e6a83184be03bf9bf80ac798582de1f02188522528abd38e21f", + "evidence_path": "evidence/20260721T144950Z-62e8b294/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:f9153d2b421c7fde1667fe562de24c4997d9ef47c91b9e0373d88db8ae6f13b0", + "evidence_path": "evidence/20260721T144950Z-62e8b294/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:0876b7143f3f7adb45dc1e7f7be08bd53fc7a50fc2d2892fdb3f160877a197c2", + "evidence_path": "evidence/20260721T144950Z-62e8b294/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:362f8f30996be914c1eabc12c22566d0adaa0fbed784c1be272c805bab6b191b", + "evidence_path": "evidence/20260721T144950Z-62e8b294/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:c0bd606dcfdd1a29e24e8c6414bf7493481afd41eb04dc38ba177dc9b9f99cb0", + "evidence_path": "evidence/20260721T144950Z-62e8b294/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:d6cd552dd9461391896feeec72894476203945d78fd31b521cb0a44e155e4d8b", + "evidence_path": "evidence/20260721T144950Z-62e8b294/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:713acec193fc6050ddee7558aae6eb7c2110b70a9691ca96a68ef5099f1e1a3b", + "evidence_path": "evidence/20260721T144950Z-62e8b294/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5bd46eb912bf2485a2365af7ec5e8663b9128d728295bbca590b2a94deaaffcd", + "evidence_path": "evidence/20260721T144950Z-62e8b294/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:ae47ed11df681f323757ef2d425fe737c873b529ffc85f9a18a2b10f144750c6", + "evidence_path": "evidence/20260721T144950Z-62e8b294/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:bf6ebfb091e50a05b6c93ec0289fb6c89b3cc5bcd8fbb3fb55aba801250cc43b", + "evidence_path": "evidence/20260721T144950Z-62e8b294/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4d7aa201b3883cec18b9d64177ce39d8866c7fcd8ee61532d91637a9ed5173a4", + "evidence_path": "evidence/20260721T144950Z-62e8b294/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bf5f3de0ba4c572b0db44329ec4315572c555486773d74a2382f439df6bad77d", + "evidence_path": "evidence/20260721T144950Z-62e8b294/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:97b3e64c359516dca8b50eb7d691902cc937d8e778ebad72c9cbb6cbc9a4e5ed", + "evidence_path": "evidence/20260721T144950Z-62e8b294/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e8c2799ddf7b30fc84cdd60c0a9d37309d51eaa2f9774da86b40156b9567fe40", + "evidence_path": "evidence/20260721T144950Z-62e8b294/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:8d182e033a5fcfa5bd9b277592154c9c269532ff96d7fe6634460964ee0f1bb9", + "evidence_path": "evidence/20260721T144950Z-62e8b294/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:80ba0d93d5f344de2c2ab33b4bbefb8ce730fbc3cd265f787ef442f490601568", + "evidence_path": "evidence/20260721T144950Z-62e8b294/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:1700487c1ac1dcf54a1e5f48d607340b4f8d826511826b0d53c39cf92f2bc78c", + "evidence_path": "evidence/20260721T144950Z-62e8b294/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:648d17eca2cfc5e623e5f6b67a82ad19d11ef2636a1b0bc5a817e347360e5040", + "evidence_path": "evidence/20260721T144950Z-62e8b294/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:4c92305a95bf305659227f38814c7be401a985ed6f8d728e4d16c124a0f34dba", + "evidence_path": "evidence/20260721T144950Z-62e8b294/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:15560e90d51a17e2e28a313e4829eae5fa59798e126fb1cb9a35308a646d0885", + "evidence_path": "evidence/20260721T144950Z-62e8b294/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0610d7a7ddbef66d74c0098c2ad9fb99983e19380c41b88a0118f8d8895e1316", + "evidence_path": "evidence/20260721T144950Z-62e8b294/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:43d3249e63d57d0de68a54d8a7fb12aab3159e6a9166cdf3e31a8eb5744fd8e6", + "evidence_path": "evidence/20260721T144950Z-62e8b294/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:c8a6092ded15d7090203fdb4859d508e49e1ba661107c9a48c6c10d8e94b7b8d", + "evidence_path": "evidence/20260721T144950Z-62e8b294/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e997a6ca4ac667ec41c7d01245d9f7b68a094aeb11143b8471e0f5c86c8781dd", + "evidence_path": "evidence/20260721T144950Z-62e8b294/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7bff2d22f91f89468fa798ab0331c7988c53a29fa3bb90f12ad209684698427b", + "evidence_path": "evidence/20260721T144950Z-62e8b294/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:71df7cfc1a6325b379e17b11714ae5c84b56a7c07fa59db19d8721238ce12eec", + "evidence_path": "evidence/20260721T144950Z-62e8b294/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:901624206b48593b0f46e413cd8b8f4c14885554e109c1de9328a3e702b84131", + "evidence_path": "evidence/20260721T144950Z-62e8b294/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:03d357dafb6c3976537864ab38f79b3268bbde8c9786d54d9bdca511c9308012", + "evidence_path": "evidence/20260721T144950Z-62e8b294/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0b0ec976ef01c3ac4b5d3e4e6c85d1139f7ccc5ec209b9fa48dd82283982cc11", + "evidence_path": "evidence/20260721T144950Z-62e8b294/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:502b63da582addc9575a08cd2b11c785ffd2de992ae3069f21f26a15e35a808f", + "evidence_path": "evidence/20260721T144950Z-62e8b294/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:ba6de302aef712f09f750711a05f4f7f90ebf9a8ffed73b16cc23d2c11c1c714", + "evidence_path": "evidence/20260721T144950Z-62e8b294/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7e31caca842175aa5f83d90ffabba0cd88d6689199dbc054ec7bd852dee3c7c2", + "evidence_path": "evidence/20260721T144950Z-62e8b294/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4b853122ae6a0f6c869151e864fd4733bdd3373d7cdbe23df3651529d6ceb899", + "evidence_path": "evidence/20260721T144950Z-62e8b294/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1cb0ac1f17937d6b26a11f4d7358ad9d28e66fc48bfd905f7bce7b9b5b51ae5f", + "evidence_path": "evidence/20260721T144950Z-62e8b294/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5e8761c3f84e51906ffff8ba8918f81fac10e1d0f8fbdabce479944c661fe620", + "evidence_path": "evidence/20260721T144950Z-62e8b294/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:247264f3f15b2723584796004f27fe67bbc1dad5c48401b34d9afbb77f6d9045", + "evidence_path": "evidence/20260721T144950Z-62e8b294/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 21, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run2.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run2.json new file mode 100644 index 0000000..de4da49 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run2.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T145025Z-22902606", + "timestamp": "2026-07-21T14:50:25Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:23954a3b47e60727a3a1aa9da9aa1b5ffc9f664a421b16f966d906c9a9db26cd", + "evidence_path": "evidence/20260721T145025Z-22902606/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:cd4252e8af2ab5554448e6a490c32a26c75ad317a5930050aefe5551820ce64f", + "evidence_path": "evidence/20260721T145025Z-22902606/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:66a5c052cbda57217201bb25b2869a2e87e140c4b1405e505ec10a900e9bc18e", + "evidence_path": "evidence/20260721T145025Z-22902606/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:fe64f61caa520867bd3f8db49f3ee73cdd484b95671a4f494367b4c00ed26255", + "evidence_path": "evidence/20260721T145025Z-22902606/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:86a332d68d46853ba170381c3e0057fa09b408264a2435f0875b93aa1efcccf8", + "evidence_path": "evidence/20260721T145025Z-22902606/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:33e25e118cd31e62dc94bb79bf2d874c122960aa04cc03a4ee690b21ca88b18b", + "evidence_path": "evidence/20260721T145025Z-22902606/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:d181d8a893696b54bebb61bf4832dab3070e8af5265b716fdb5e78dc7846e09c", + "evidence_path": "evidence/20260721T145025Z-22902606/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:83ddfcb66843c77360f4f2edf2bdd9b9c8ea57ed084f898355ccc939bd84f7d9", + "evidence_path": "evidence/20260721T145025Z-22902606/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:bb955d517ff43c289d73b6382536e6dba7da768471805f834d81935f65d2f08b", + "evidence_path": "evidence/20260721T145025Z-22902606/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dc1d3503ea9a4fed12127d61b1bad0ded98c6ecd6fc40fcfdb378c8ff18e4df8", + "evidence_path": "evidence/20260721T145025Z-22902606/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:f2430f1ba59a70887b3430aa57a9f06d48b3fced00032f4f0ffe4d70e5e42668", + "evidence_path": "evidence/20260721T145025Z-22902606/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4b87c73db2d57312b6cb31d8ed53abba9ba0fbb9a5ec1c59f3fbd56a71c5978d", + "evidence_path": "evidence/20260721T145025Z-22902606/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:0c59234d6b5b51990bb5bfd113ee59658aa323fc7ad92a26721f729c737ed99e", + "evidence_path": "evidence/20260721T145025Z-22902606/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:44756282242b134f4155fe91d7c26e10c385e73befd15dc1df722d6c7da9d528", + "evidence_path": "evidence/20260721T145025Z-22902606/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:7003600f940a7e12f4ceade7beb8cca6a4c39aedc939634b66e4897af43ab8d2", + "evidence_path": "evidence/20260721T145025Z-22902606/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:0bd6c6d36a748d430ef48c979a6932cfd87deb769164e2ffc55e38a299ee5247", + "evidence_path": "evidence/20260721T145025Z-22902606/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:2d4aa8b351c57ed88bcc78502c4871523c80d8c252830303f5788dac48b45fb4", + "evidence_path": "evidence/20260721T145025Z-22902606/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:f80e4abf186287bde3dbb6b3b59a03089ca339ad05818a02fc052c5a45cff1f9", + "evidence_path": "evidence/20260721T145025Z-22902606/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:408c24c9ca0913e30fbad282f78a02a80da19ba5c3a8a74144c5bdd5c38271c4", + "evidence_path": "evidence/20260721T145025Z-22902606/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:d0c815b9d86003b61e0427b772297e06fb25f24bbb639137ba4c649c5fd2e998", + "evidence_path": "evidence/20260721T145025Z-22902606/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:c3b0cc29e232110da8643f3582d8750c55b29e2d545a24742151157ce1379a40", + "evidence_path": "evidence/20260721T145025Z-22902606/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:bd2e1a7b162912a5439a6629d55466ba7b7c053295f8545419121a1d1a42840c", + "evidence_path": "evidence/20260721T145025Z-22902606/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:0ff55c49c1cd3db8a46c8a0232da752c4da4d531e150ddfbd69ef534d331bfe0", + "evidence_path": "evidence/20260721T145025Z-22902606/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:6227cc27a0e9229309fc9e9f2ccae73efcb040af512b8b103b85c5189fe1c1ad", + "evidence_path": "evidence/20260721T145025Z-22902606/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:16c53dd9316dd4ab03b484b9a1e3c1cc6d270ff43323f7b85f6ee062a4adbbeb", + "evidence_path": "evidence/20260721T145025Z-22902606/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fc353dec54742c87d42210b65e400fbfb367d183e09dd2e5b25e11932a928fb0", + "evidence_path": "evidence/20260721T145025Z-22902606/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f82422983720fd2a61d38ccdb5055541860158fd1c4976a6542aea9fbef42c1a", + "evidence_path": "evidence/20260721T145025Z-22902606/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:300d745f3462ed517823b729a4a5f693978f9e17e070b3605c3cb28be292eada", + "evidence_path": "evidence/20260721T145025Z-22902606/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:ccb002291ef71f130cf1b4926eac4c4a1158b89078d071279801113406a167cd", + "evidence_path": "evidence/20260721T145025Z-22902606/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f0ffeb0cc86b94a2b467def5439840fcf5d7100ec135dc93d92bc0fe6fbd651a", + "evidence_path": "evidence/20260721T145025Z-22902606/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:8c658c0b6d05603daf4f15dce2cc6c3f0af44e76da239ff1aa8063ceb2ee6154", + "evidence_path": "evidence/20260721T145025Z-22902606/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:d36047b0e84698a32ce052234fdeb96b4c937d2269e01111eac28af9ee390137", + "evidence_path": "evidence/20260721T145025Z-22902606/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:1909acf87a9a72e9306244846d7b8f5f3b9a6694806229ca18767008ea51866b", + "evidence_path": "evidence/20260721T145025Z-22902606/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3fccae0605a33e57e6055fea800026a8fab0aba9e3b0fdc41d590e734cda47bf", + "evidence_path": "evidence/20260721T145025Z-22902606/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8fd1f49a3ee2d57627fa86a2ca0f99842d67bfb817474e499ef1d51a810f1e62", + "evidence_path": "evidence/20260721T145025Z-22902606/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:3a214f35d0ab79d08d494797babc7ac473a1141076a663eb4143a5c87830eb9b", + "evidence_path": "evidence/20260721T145025Z-22902606/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:30ad51c232371c92f7ad8337e62f8c29cac2e2d759be068893a017f8b7e290d3", + "evidence_path": "evidence/20260721T145025Z-22902606/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:91a46d0c0ba912129b694ffa6b2507ef67aa7207ac9f3ff5961d3c2988f3f2de", + "evidence_path": "evidence/20260721T145025Z-22902606/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cd1e558d24d816c06e07604ff5f87c9b5fe6416e16cb9e02a74981aa3d540cec", + "evidence_path": "evidence/20260721T145025Z-22902606/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6e3b4f96f3c72c1da5bae2cc62e459ca563237d19f59fcf4195c86d38572bd8b", + "evidence_path": "evidence/20260721T145025Z-22902606/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:08d6b9e30d0e31eee673311cba2dbe0b5b93508f6f5c1eb075117480c6949a41", + "evidence_path": "evidence/20260721T145025Z-22902606/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:21982008e9dc8c7196f00b4badd86ab4e350116dcac3d40eccb8fdde5628f170", + "evidence_path": "evidence/20260721T145025Z-22902606/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6fa5f1ee8e7c5c44a53a02e3218bc3ddb6e0c9b501a57c81a2339bbfaecf5bed", + "evidence_path": "evidence/20260721T145025Z-22902606/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7e0dac48682c22cea87d50828166c3af4b916207ee5242500b7ec8672ab1d745", + "evidence_path": "evidence/20260721T145025Z-22902606/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:21a5a5e5e8076622dadc99e6e71ce32537e915f27e189108ae05c68e2cc012e6", + "evidence_path": "evidence/20260721T145025Z-22902606/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0ee4244027378ebd4c6ba7b0ec08bba10ecc812ed74d7d67568f41975ea12139", + "evidence_path": "evidence/20260721T145025Z-22902606/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e739adb17ef442a4dab1fd6c3e0bea9dfc93ab5e7d61af11a64a96ea7d2f8678", + "evidence_path": "evidence/20260721T145025Z-22902606/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3fa851f34256976a2b9f985da276bc18f14ea9afa703b837fd372ea70b30ce27", + "evidence_path": "evidence/20260721T145025Z-22902606/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:51ed3d5fab10d7abfe38372c234effbb04b7a1b1123ed52e4314ffdbb07cf8d0", + "evidence_path": "evidence/20260721T145025Z-22902606/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:68ff9e28be056aa33a42513d2232251265ed3c49e52333f51bcb10aa58642db1", + "evidence_path": "evidence/20260721T145025Z-22902606/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 21, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run3.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run3.json new file mode 100644 index 0000000..7d0932e --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run3.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T145056Z-b76556cc", + "timestamp": "2026-07-21T14:50:56Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:6bfa25bcddaa8b6346f3d0e1afce04abec9786e42c66ef5ab4e4d6d4dd56b2e3", + "evidence_path": "evidence/20260721T145056Z-b76556cc/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:45a7cb55ce8719f0cf50d2120c5b27e7dfc85ddefacabd52a67088346331918f", + "evidence_path": "evidence/20260721T145056Z-b76556cc/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:56bbfd1ad96f547db23528ccd458f28be3c440fa69313671001d92dfdc397b30", + "evidence_path": "evidence/20260721T145056Z-b76556cc/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:1be0d65a7979556ae1480370dcc0273294c048d1cf026c3c8db179341b9fa928", + "evidence_path": "evidence/20260721T145056Z-b76556cc/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:917ee6a8a5dc2ba2ec7682dbc6aa4e158647be0b15708ca5f5625edd37a4f9e8", + "evidence_path": "evidence/20260721T145056Z-b76556cc/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:30346519cd33f0ba93b125f56284d6ab1e304f9e4a6372f0e244d978ed55fe03", + "evidence_path": "evidence/20260721T145056Z-b76556cc/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:dadbaab07318cde0c897addb9fca042af1a7ecab53322715e1c29d6570f6416b", + "evidence_path": "evidence/20260721T145056Z-b76556cc/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:0e49c5d31dadb2920030ac8342bfaca9832562937b2cd37ee2004a46f3e1138e", + "evidence_path": "evidence/20260721T145056Z-b76556cc/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:8a3d4cff14d39229d53d787b924fc58c2248c2d968cd9e7c935c27ebc425252b", + "evidence_path": "evidence/20260721T145056Z-b76556cc/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:27f837052de0da5996a609f1fcdd87eea350443afa4493eb9027429ee9f79e15", + "evidence_path": "evidence/20260721T145056Z-b76556cc/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:9213d89a6615a243465afb44bf202d55bc3711b2a7d1d5db09ae2e2861245183", + "evidence_path": "evidence/20260721T145056Z-b76556cc/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:909b7a9ecee1ec7ff20969b5c1ab03316c50c40bfd755159e53242e7b1e4eb2f", + "evidence_path": "evidence/20260721T145056Z-b76556cc/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:501a16446cbc455f0d76d1974c873617fee79b788dc65b8bdc18dd23c2ef59f3", + "evidence_path": "evidence/20260721T145056Z-b76556cc/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bcb636334ce7ddf8269dd53915e995ce64d715ac9c77b2eefc5c9a3ca789259c", + "evidence_path": "evidence/20260721T145056Z-b76556cc/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:a8186ee1bb1cb524f3ffd4e3731f2f0679a77fb23a0b3d48440a0853444742ec", + "evidence_path": "evidence/20260721T145056Z-b76556cc/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:05dc7d5dcbe4a1f6ab9a84f2e08081130a7ade7c4e418c0ceac968d60266abb3", + "evidence_path": "evidence/20260721T145056Z-b76556cc/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:46bfb22e7147d673ae8f746ad7ebf28095c80cf7ff2664cac8320fe69314d4c0", + "evidence_path": "evidence/20260721T145056Z-b76556cc/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:fd5209a44dc8e3842cf979b4f7b15f98ac2ae8d31a4e21fddaab54a03796ce84", + "evidence_path": "evidence/20260721T145056Z-b76556cc/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:d4b97e7d434a786ba0d675459ae962fdf12b3d1d10f2e778feff6064da508714", + "evidence_path": "evidence/20260721T145056Z-b76556cc/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:d11a146f5a4c60d42f32a80d3b932cd951cff7ab2a410c32e482f6afe0007e60", + "evidence_path": "evidence/20260721T145056Z-b76556cc/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:7195c3cd71395616ebb1eda5d7a017b920206779c8ec6bef1f1fd82df6c5ea26", + "evidence_path": "evidence/20260721T145056Z-b76556cc/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:faf94f0a42f3bf4349971fa8ceb408748c446df37a09bdb0381cab4eb357828e", + "evidence_path": "evidence/20260721T145056Z-b76556cc/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:388fdf5a9bec50d5ce5098ffcc4037d6b6bc9b913963596d0ea68f4d8286d402", + "evidence_path": "evidence/20260721T145056Z-b76556cc/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:1b0584fcfcdc490ba06c3e913e41af6754c24c19c6b7007d51dc4570313ddc47", + "evidence_path": "evidence/20260721T145056Z-b76556cc/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:60d95d261f6e21474214ee8eb5ef0b5c57d47afd8d99711ae0d3044bd9579c5b", + "evidence_path": "evidence/20260721T145056Z-b76556cc/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:634b1792c5195b0715b4fa49e3ffc94f398fdd8fb22efce31b1e3c8e0624c545", + "evidence_path": "evidence/20260721T145056Z-b76556cc/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:29e756cf8bc8cf70027dc9d43bd6525ee12b57a711e3db580e5fbe716a97e0e4", + "evidence_path": "evidence/20260721T145056Z-b76556cc/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d91a149221f0e200c249c55a025aef593cbffe9504fe17e9ad53df3a1f4a3cf9", + "evidence_path": "evidence/20260721T145056Z-b76556cc/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:72bc9d7bbdd9285132d7c9b15f1735b7f1b7c99503631fdc28606b261a655284", + "evidence_path": "evidence/20260721T145056Z-b76556cc/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:30cdb61375a358ac889982a32dcda37ed26d8d757c02c5bdc4f0c8d3c03b6285", + "evidence_path": "evidence/20260721T145056Z-b76556cc/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:233e9886a0e1fabc8cb1e732bca35b5f16333850af25a4ed104145560bbf12b5", + "evidence_path": "evidence/20260721T145056Z-b76556cc/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:e6cb9fbbe72a2e9d55b55c0213eb27efc5165fb85c405e00ba03885febf7bae8", + "evidence_path": "evidence/20260721T145056Z-b76556cc/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:f2c147d26e6e4fd8fced0ac94255de1d630f5b3917af9b50c9f093bcc867f46f", + "evidence_path": "evidence/20260721T145056Z-b76556cc/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e475b5703a686aeb7179ba550608da4b18cdf3b8886a732f2b1863b731b4fee9", + "evidence_path": "evidence/20260721T145056Z-b76556cc/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6d3486ef671951da95c622d516b21a4e44650678a5c51dc68004bd134de36647", + "evidence_path": "evidence/20260721T145056Z-b76556cc/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:2de4c4aeb628a39b3802028855ce06cb0df0bee60ecd5ce23dafa6c2a41737d2", + "evidence_path": "evidence/20260721T145056Z-b76556cc/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:951e968150f095cd909ddf2a4bd0a9881257385114fb12947b3d3c2421c0440b", + "evidence_path": "evidence/20260721T145056Z-b76556cc/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:02ab2fb0416e99af58e0ed1cf0611260eda0c2445fad8b131a6431e3f9caa60d", + "evidence_path": "evidence/20260721T145056Z-b76556cc/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3c30a4533d4b7824d5155ecfe8e27b883493cdc396eae94c62b4e515e31b7555", + "evidence_path": "evidence/20260721T145056Z-b76556cc/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9ff81a04f9a15844a11443617225c46185b194678af429642d1d34d9893518ce", + "evidence_path": "evidence/20260721T145056Z-b76556cc/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:ab24f90fd580b101b967212021bd51f7d0d30566c0943fbcdd4e1a017e2ba126", + "evidence_path": "evidence/20260721T145056Z-b76556cc/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:8d1eb34717fe80038b4133b4d75d5a4b8739e872590a0bf3a8991a623aba1134", + "evidence_path": "evidence/20260721T145056Z-b76556cc/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e8bc5e36f4a0ce1d8dea007eaa09929ab431dd017c71d8ec67ed9df8464f4bf6", + "evidence_path": "evidence/20260721T145056Z-b76556cc/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6efebc78251f067edfa1e7a4a41f4f2b2c750bff7544a99b4b20faf87c44d03d", + "evidence_path": "evidence/20260721T145056Z-b76556cc/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:eec36256e0954824451de72b241f23a33641d4d78fa0d8039358b5619feaeafe", + "evidence_path": "evidence/20260721T145056Z-b76556cc/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c93ff798c5958259613353529d8ac89cd5f8c5983edc544ce6e4720f7a1045fa", + "evidence_path": "evidence/20260721T145056Z-b76556cc/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c54788590aee9b58513df6e36f6825a44ee954dc97c8bd03231b85e4614969cc", + "evidence_path": "evidence/20260721T145056Z-b76556cc/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3b0ede1e78d5399f328be4194ce82efaf884c77ee5c088689e15b41cdb1a0815", + "evidence_path": "evidence/20260721T145056Z-b76556cc/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:caefe3bc5e99203a4cff7133e593ad7645aaebf6e6a8a7e673c0d586c8f18bc1", + "evidence_path": "evidence/20260721T145056Z-b76556cc/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1f6a7b4b6e738edfaec755c407c3839013b52dcfb0afdf652e320fafe4f1ca78", + "evidence_path": "evidence/20260721T145056Z-b76556cc/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 21, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run4.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run4.json new file mode 100644 index 0000000..78fd7bb --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run4.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T145128Z-92f6444e", + "timestamp": "2026-07-21T14:51:28Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:dd406f08633dd49b3bd4a5eec5e38286b8ea93506cea4f0574953b20bf991c79", + "evidence_path": "evidence/20260721T145128Z-92f6444e/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ade7700efd6b934a4edc6c4f1ed7a2cf4e66365ca2b1c729cea5b8b82b4f7c51", + "evidence_path": "evidence/20260721T145128Z-92f6444e/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:cd95ddaea8c1c44d11ef04491a1c384027bb6fb043d8b5150107c9d4ce788ad1", + "evidence_path": "evidence/20260721T145128Z-92f6444e/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a161b21bfee0b1980658c90bec0c173428ff649a4c82a5573c7aea57990cdc8a", + "evidence_path": "evidence/20260721T145128Z-92f6444e/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:f17decf532d17c8b05b92b2b1174f9e8995c63001f41796e9d734f28cdc93bec", + "evidence_path": "evidence/20260721T145128Z-92f6444e/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:e15bcc550abfcc9837a2e487a2b7c7582bbd3fb8c80cc59206cdfd488ae3a560", + "evidence_path": "evidence/20260721T145128Z-92f6444e/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:52da4bb9ad5d3e765556b0d8b822dfe6b3ec709b607755eb8fe318c3e86c60e4", + "evidence_path": "evidence/20260721T145128Z-92f6444e/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a785865fd8c615c7bd1a40e28a3c51c34421b53c8b786378679bb8c9c7a448f1", + "evidence_path": "evidence/20260721T145128Z-92f6444e/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:72fa60dc1712af654be6f7e033dd4d85f79918564a466502093d7956fe22d4a3", + "evidence_path": "evidence/20260721T145128Z-92f6444e/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eaabd69f4ad1aa64b10e2a9a86391f4f0a5c51a25e3e89b593795e03942422e5", + "evidence_path": "evidence/20260721T145128Z-92f6444e/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:b9797e7f9529a5740e0ded8a0c2d45918f42e58057443af62e9ef4d0b7d4dbdc", + "evidence_path": "evidence/20260721T145128Z-92f6444e/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:35c8daafc7831c64e34eaeb86fea49ad491246890366c396bc8e0f6efb9466e6", + "evidence_path": "evidence/20260721T145128Z-92f6444e/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:37c461c1507b5faa1fcb0b571b0476e5f1f74c7a64936ec69b221e4fd6c979de", + "evidence_path": "evidence/20260721T145128Z-92f6444e/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b94ee30959928b2f4d73442e2786adc6765e8b80ee92ab55b4695a3ff56fedd7", + "evidence_path": "evidence/20260721T145128Z-92f6444e/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:c35eecd580a4be763b4cfa288e5bc56a2894c91710b38d7eba39a2208f742c60", + "evidence_path": "evidence/20260721T145128Z-92f6444e/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:60ed71c5b3922811ae2a1ee12661943fdca7eef385a05db7c53a1f98fb276c0c", + "evidence_path": "evidence/20260721T145128Z-92f6444e/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:2b2a2763ca3535348b8854f1e3ad5d6b9ffd14a7b1da690f48c33ea41e5492b9", + "evidence_path": "evidence/20260721T145128Z-92f6444e/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:b090422c113b7d0d0a4ba0d9653fca41e09c8f6f99bb1cdb3591a0e432a21ff8", + "evidence_path": "evidence/20260721T145128Z-92f6444e/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:00054427fca1551cc9fc3c59cee0ce2272453694841671cbbc6fe5dcc91a9ccc", + "evidence_path": "evidence/20260721T145128Z-92f6444e/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:a7a2d49262ea50dcb0312d231a32ab9cf40f976a7c80879f95800d1dd67df734", + "evidence_path": "evidence/20260721T145128Z-92f6444e/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:7630afc3d053027d3c08b87387133d34bfd4580248c76dedf5c19a752935982d", + "evidence_path": "evidence/20260721T145128Z-92f6444e/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:c0ecbc03816c5a7797f484097312a46e651129dbfb2cdc4ecd643e7424f6368a", + "evidence_path": "evidence/20260721T145128Z-92f6444e/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:dab87921c59368b9c5b2a697c2b9bb43174769f399f26dad317b47ad924e9daa", + "evidence_path": "evidence/20260721T145128Z-92f6444e/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:5f029630741155da8d7add6a841e81a7467db100a6ff5869ee321cd2450e9519", + "evidence_path": "evidence/20260721T145128Z-92f6444e/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e9162001ec3917474e9d14d4701f73c6d5a1d328237e04526b4bd3f90b0b3082", + "evidence_path": "evidence/20260721T145128Z-92f6444e/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:76a414286cf51b1adf60f4d8c7a7da05e3dabfd76b9b227c389b399912ffca64", + "evidence_path": "evidence/20260721T145128Z-92f6444e/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8658c5fe50ff0a5336a24919e05cbc0c393c545cec4fb446f8aa23cc20969ef3", + "evidence_path": "evidence/20260721T145128Z-92f6444e/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8f7c1b4a2d09ba9d483dea2e8cb91b1d6d539c1f192a2a599e503724c7f9265b", + "evidence_path": "evidence/20260721T145128Z-92f6444e/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:06ff99514e1baec3970b15448a6af7f6b78b48d437f556a54e3737bdc324ace8", + "evidence_path": "evidence/20260721T145128Z-92f6444e/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bd1781e664293cf8235e94ccc299b1b545dbdd1b180e30bcf8981888a675bec4", + "evidence_path": "evidence/20260721T145128Z-92f6444e/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:5db5532004d3500dc497c3db5a5c08ff68e77e65ce09f5a53bede935488085c9", + "evidence_path": "evidence/20260721T145128Z-92f6444e/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:867a6796b637b42e3c3cc1a57bdb8279f8d80afd451006b2309ec0b7785f3ca9", + "evidence_path": "evidence/20260721T145128Z-92f6444e/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:c86176a84164123c605c4ba577ce919c7227a29b95a41529317a10d1dca51dd6", + "evidence_path": "evidence/20260721T145128Z-92f6444e/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9a1b49242707a1df58877a6bd6c6803da74b6fbdfebda3f573a9852990a794de", + "evidence_path": "evidence/20260721T145128Z-92f6444e/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:130fbc2d2c7e0bc6d31b52054c5909f3113705199c42ce794940038e1c132ca9", + "evidence_path": "evidence/20260721T145128Z-92f6444e/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:5ac40c5eea7b39c16f34858e5bbfc38263b8260e5a4c4803b014d0812157799d", + "evidence_path": "evidence/20260721T145128Z-92f6444e/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:38b049db11fd0788ab22f5cdcf743acdd538f1471c781a63c5af56601984ab81", + "evidence_path": "evidence/20260721T145128Z-92f6444e/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:41d56a5c7f22841021736c2eafd57e4b848b70ff9187c8483f924e32cb408dd1", + "evidence_path": "evidence/20260721T145128Z-92f6444e/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:39c3f014b14d9f248e5468ad06e0be543f10aa1131b4ce04c5f9cf37aaf42a79", + "evidence_path": "evidence/20260721T145128Z-92f6444e/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3fe0ff6b20eab5fde9acc74efb254bd844148e04f38530e3c340ecba3cfe15bf", + "evidence_path": "evidence/20260721T145128Z-92f6444e/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:c219199fb1163377ab4c67111009e88b0af3df8d4089402f9dc439796d2a8cad", + "evidence_path": "evidence/20260721T145128Z-92f6444e/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:903b8804e45a870905475e7ac167f6d6dd65ee015d07fb95735bedc9650205cb", + "evidence_path": "evidence/20260721T145128Z-92f6444e/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5bb1241c23e434ae324e0222c1ebd68b524ec02dfdfb3666c5c73ffd5171c038", + "evidence_path": "evidence/20260721T145128Z-92f6444e/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:717758b4fdf486715ddbc3535e78ecf9deeadc7f487a4a0a8be5f026b18fc7ac", + "evidence_path": "evidence/20260721T145128Z-92f6444e/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:9dcd9ca8c65c5f57fa67275dd23a44d1c6ff7efdc3662da576875fd6acd8bdab", + "evidence_path": "evidence/20260721T145128Z-92f6444e/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e5829b74cb58f611642d925bcfa25651cc42f3841967897180f4a13824c04d2", + "evidence_path": "evidence/20260721T145128Z-92f6444e/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e090c7d4a8565169e3e108050de989c75c6f37b4441fe3666d34812015a379e3", + "evidence_path": "evidence/20260721T145128Z-92f6444e/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5a455d3f58ffe4b924bc33abccff90160bfa3e457d60d1fb3e2a3786b9a588b3", + "evidence_path": "evidence/20260721T145128Z-92f6444e/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ae6cc14953dab50da6c9ba40bdd6e1c3037664adeb92d46470e63163f1036a85", + "evidence_path": "evidence/20260721T145128Z-92f6444e/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d0d901b68c6f6f525210809c622ca275dbbc260567cf648682925f0857a9e0fe", + "evidence_path": "evidence/20260721T145128Z-92f6444e/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 21, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run5.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run5.json new file mode 100644 index 0000000..cf6a834 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run5.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T145159Z-19ad63b1", + "timestamp": "2026-07-21T14:51:59Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:44e4369fda1b293ac214a4b05d892dc93c0e01de43a179a351d93c9bed8136a0", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:9bf5a6655a680c90cad9a438452997dde5c2b9a9bb111e5e69e6d9a2b2e89bf3", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f9d2a2c0fac4de21a2e4d727de4a0e2bc7b94ca104ebddfb623823dd91c11ccb", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:7bc96b28a6519cedbccf191c68ff3426d43fa11406014ec2d70ac89035739e82", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:dcd4153f179601a4fe2351b21e420c5bdad3fe4e071e4c95381283252cbbe1f1", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:bdc9a1aa33147eed19b19044c89e6b2b81868170d7b5b8f7b7525c14dd6fa6b1", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:e7c66c9f56a1b44772b1a911ff4532787625bcc996f489b766c42e4b0c7c86d0", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:1ab492143702c8c32cd1837e73510cc4d7a74bcf2d2500044c3a0c46e3acd5f4", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:729b744840656b3fe9659f98cc44cceed4a5d39e9a8ed4a15ee79780397be341", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:78b09e7fa2d279fe637cbaa2de7701aa7bc4383465b4f61be38371e92a64cc69", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:4f0a87a543ef3fa58cd0948361042e60610da4cf15da3160592796edbf71f4bb", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7e556dece290ba395248edb0d8385b6d67c251e45cf658811a6f5ef047425595", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:2f47d25567ebcfa999cec818551aa40b93bb5be6cb1c6aeff2b52229be17a8f1", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce1958c787e7a1bcc41c46d682a5f62d9f751dfd752f6b7f8ae181cac017906a", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:29ac9e33cf6aa3d9d72679a6a23f45d57857aaad5dd4377ad3b1d8fa8a63f811", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:e70fc6f78230bd80fa9db1aeb911db1514253f65b35f8d50b08bf782ecedc3ee", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:a7cba152023732d15deac971bae63e2768104e86104b7495f0e718ad4b340e73", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:0b0a12caded131fdd846f6c1275c5aa676af572f6ab81867853c1ac842d0c110", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:d7ceb4905aa9f0e1b5da402093aa693f9011130962819136ae6b11e307c9befb", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:a008b73087940077bf1ce08fe0b2839671ff0f7d8e9b36d62ce06e7fc0dc2c06", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:6d87f7bd4560ff000ca6ad164137d425e64458851dc6333b392157b8e8dcd990", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4e33056a970d8a18ed0a69d4ad7dbbb8a4589a2e7fad59d9d5dfc11656ac375c", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:c3381a64e7f3df0f6fcd5c63f2f1be12b5c4223294169c917014aa3da5466095", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:a19c3fac508c6779acded4bd5dd91846e0e08519c576dfc46952874c1d8c40ce", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ec3933c8eb901aaca4315181780f62d4e67fdc395a1d7687b68bf3dcbbc6a914", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4861e158ae2c75b3048f381917530f0780b6d7355ddc3f76e973ee35609b8aa8", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:147d3c5dd1fb9ca93a52731a20b6c222b4a040229e6a24c0e3a9bcb8929f0e74", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d7f97609f5971a1969a859434da7a3bbbeed1f66a32b4d0f635f0586a6435145", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:971a20bc534e7f9b055e0431a8319789b7dbc25355ae97f9dcd4d63aba1a0660", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:54d066d54cffee90ef24fe58777d1b9ea0b522d8f7eb7daadab52b1fd9229c5b", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:a5a85e1ae5ffb0f4ec322a9f2aa39bef149bce492b2ced758321ebccff99ad0f", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:d93e2ce326ff3e3ae387d9b4f44689ac3299873236f45e600f3e2283b6726366", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:039b882928829a25d2d44410858be82d6a0cb3ac40e54300f9f6e31563202cbb", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:289c248f71edcce72dfb2f76ec47da3e68f5f01eaa4671900f47a59cfa5914c1", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f6792d365cf729ef66b5a3602df3be0f0d23dde2be3e764f465e5188ab3370e2", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:d445ace56236aae52d4d218d6354a4eca40faf0a32fffaa4afa7e376de07a879", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:8e3198b443c86ab295dd48c8df7ffd8104b6fc026186f967a94eaa3f52e37ca0", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d53a61d2a245946c7e77985e85a36fd1e362ac231b98104bf6b9440f82c4aad7", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:846927a6cb7d4f6dad3712a4b94f4090497494d1caad9dceb01e33fbf686ba69", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:90e1b682f806325138755f0b1df996684282362fbb0461d5485de73bd52bf72a", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:bae758f6f916135ee8512bed96f66e9ea7e4daa3b86ab3cea2ed36e1ffd1c49f", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:59f897abac8b7d7a750f3e0a4b4a1c62deceb56142948035cbc47f13bdaec5ff", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e3a332f571c9ab30af9fd763f0481b0f8379f588aeb0ca001425b20d380e1514", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:73676f26b380dc9063477c41a8e0d8acbc76fd8c709699ac29605f5859164c66", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:b03fa94ce9d5126a9534a3833adf5d7f93c33c6eac73bd8f8dd23862e4250700", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3292d4cf4eef683247b41843837d805fc64be41e8c8a112162ee2ddf087da7ac", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:64a9f54fc714cc6300c88ab3f8e998fa8854a24fadc06ccb22b5bc3a318ae4f5", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2155778df0800f43df6d1626b220be753483ff64873194ea09393c503ee74008", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4373fd90bc1da1e3a439f0523b1748d54ef2982ab6868723e211d22694fe31dd", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f739a98cb4f2426354790d31c79ea0555e6fe523e3a1a4509b8eb758937ce68", + "evidence_path": "evidence/20260721T145159Z-19ad63b1/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 21, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run1.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run1.json new file mode 100644 index 0000000..124dadd --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run1.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144546Z-f4e7aa16", + "timestamp": "2026-07-21T14:45:46Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ad0b4a31722367a89a2300bbd0aa1f2525406f3a3745b550f2222c1afe29e10b", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4e9fa5d1436706872957655d7a8584b8f6bb4ef96b811dae550121c9f736cc16", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:aae7d1bab096852dee83fc0c1acf632deddf4acf0eabdcc03da403ff9da1fa4e", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:8c0bf33a9945ba615ed72fab6764024e3527c1d2135d622ff29d9cf0fd133830", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:35fb8b0c6e70aa216bd6d818a090e9e95c7c04688967f84df0b3cb1357d1e3e1", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:c52e21145dcfed71b413e75de48bbf8ed9b4de6d30eb11a4b0e395f04159f328", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:44ac55a8108ec141698025d99186793459edaa7372dc21cabd5bb07e34db848c", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:989e86055b5e540958c06ea0d9de5f0fc79ce091da364baeff55164f32de7d3f", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:b07ed9b83c6f7c8a67341dffc8cb8a2b95a910dedbd38df46a9f92c4435cfc39", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:338691e8c7b336f057f61c57cf4be48c60b102651e8182e4a23bd55a62f03a1b", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:454441f8071ffae74de30333a9fca22aef3348176ed9ae449170299a9b3c0140", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:27b04403f687376c5c3b6a0f8bc838c10448ef00f4051467167331df8f002cae", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:553490fa6e0fea65f3aa6ddafb3b4d85b3a1c24d11e81ac3b3b94e42b22fa312", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ccb5a1a88e095e87642d68cc36dd9ed28950dab7a66ded864a91366d48efaea5", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:0495bff8aa8cc698979c83b1d1b30c11f54887461a83d170b2c0663feed9b18a", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:4058286d89a46a6b42c03bfb5ce9cdfb081972ecda262eea2db5839601f565d7", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:0933e287b886052c3c0d8167b262c73bfdb6fade840eb10bcb8de8ca6aaabee6", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:6ee8cb09cd4eb31ca2bfee9c10dd3131570b14f9c683a96b6581f4aa5457a8ca", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:7d9c4bec8dd25775f8878bdb6f18f48e18e4fae2f13db1c962675708c927250a", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:97a36b02b9487e14188b6dab4ebabc89e48a98db3a157b93d002a1cac2378766", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:56121b6df70fa87df5442ed9f92569b96c1ea7b8c0aa08b4aaf903f40cb7507d", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:8f3752ca775e36da8bf238a218f7d6b23cd1647c032b7fc337d502c9f3a49baa", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:771fe2ef2fda7ef254eb977a61090562c68d90872e4adc32b8bc2ac9dc97fbb2", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:99b078cba6cfcc8a35435978cb608a43480133487d0ad0e7b5cd8902b3f3db0b", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:df93dd2e3405319d379edc5ba7b758c7c6ea42de1b4d19b2cb109c372db7e66d", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d2cbe21b20b0af31aa9cd8a6120aa94e0f46278486a03b87a4307039a926eedd", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8ac9f3e5f56ada7a431a9ad22c28ac5482afad1898de6843c60975ec198d2b9c", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:238e951319d7a6900e91fc851142d5e1f325451f66da65be2b45f17e7b106fb0", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:0819aa9143c446594792abea8b88cc630391cc2416b33fa01a19d579d06c7ee0", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7600e2480be70873a825833d714b2ab6cedfd420a22d36dc8375f9042a4208f7", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:607fa47d01786695582a0beab60bfabd468c5eeb070ed024a4d68871cef84e38", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:4206fe94b4505e7b7e16d8925f7c3a083f17831a1b59f697ee20f7ed2681ee25", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:5ddec2328f746b736656b104715a7e8031e28458b62f34535f076e65f3a395a1", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1b54a7ce212c9a42e8e1a2c154272404758f6ba98770c599852c151d9f0539fa", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f9bb8e4e34cc077c7e1fc28f02ab9280d3a77f93b3d5a3cb53f7e011f753281e", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:1acb362f36565357f2e9e5117dd6ee90e5a89cc3d1c5b035c16186e6a3ec4ba5", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:02a077511df6123a5399b0d863929ff11924356f566d429213017a6cda046aed", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:428fbbe122719be22d24e8cfc9710a5d3f69f032654ad9e270921446e356ab4e", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8a1b8b3145a80c2064e8b903e78e47f595c46cc8119ad271026ac197331ca5ab", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8770efb30aaf10d1172ad8899c6a13e780f171e6f6510d2d806bdf9598fcefb4", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:03eb4245108f96a5104298271f142ac9b4477ade1c997868047daa818bec2c92", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:7f3435e091813559d10cfd72829c38a2765eb8ab312d7445ace4cc11ba540587", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b30d2acf640c496497bfb35bb7124eb84e90980a184af161bd3caa3852789a05", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b69372f1617d78afa4f263631ea7a332f3fe215280a7ae02b46ae0cbeefa853d", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:24fabd9238f7cddcb524483fe5f916aaaa8f9ffd476ccc23bd17dfb191c67508", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:87b9fd7d50225d70ee5fb7593754798459a74616f055c477f07db8387b5d9236", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce562a8a3a8f2e2274b0029328fc51298c98721da66d7d2f1937ead6040bb770", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1ca921672a60db11d2634f01b35bd37b26b74640e3a50e206a2277a609e88683", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f1831fefa1b5f03c17a4f92c19d224625f35c7c2b4144bb71ee55578343e2712", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c54370477e66ffc3d2d3c2e3ae7d29a15d63987095646c3d399c949141c7af4b", + "evidence_path": "evidence/20260721T144546Z-f4e7aa16/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 22, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run2.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run2.json new file mode 100644 index 0000000..f240a6a --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run2.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144633Z-d2280c5d", + "timestamp": "2026-07-21T14:46:33Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a8b0561c40ab16bdc5bb1dde7a45bb218c9f7a50c8291c98f1c4fc0ad193093c", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:189e35440cb479659b629f9c729a539cee4a2c103d5829e3ed2d0e119a47a233", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:58013ad75903f632ca60ee489dbc0b20ed097fdc64c180f536dbe1a0e1bf4f84", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a65f6fd67ce98338979d4b99cd6a1d383a7ae882690afa754460e18f197ade40", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:55eecca1271cf3674df22e1767f2519e47e92f02b4bb71158d06a08005580765", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:899a7f80732df2bbfcebcceb510694e9e351fd00244150da67672528a3f664f9", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:78fcdb18b8bd48d40cc61b3b205500e1ca83006873b86eab6da22b0535340e61", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:e84f4a497f7b2c1d9141b652b03c36ee6bd9aac5ca4447ebf2ab6c41eb6a8635", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:52ec910800f5b469c45c9fd0aa365b4e62ec599b9709379ca3e4ac6760474c10", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:98f5eced10b8cfe3365682440c8938f3c21f33818077f03023429733f60df132", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:13499d945a34ccebc3f727854b1df59d11df3a1ad568e1810e5d56f472161523", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:28bc512d39b10bf2117afadabd4a9b8f532bdb2264d65f686c498c32002ce481", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:595383c79f939dbcc76b044910d2bbb0851bddee00ff8202c6e3ea0cec758fdc", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ff2bab5ea9e470bf2144066894c69f69dd9686108c28477d34635b38a7a525f0", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:df1f6041c80546a56cec6c62338d0f9dc05122650a0c4198ae4a6ffc1ff6d5b9", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:388f47f41337cd0c18798fa7783e10e0720a4adb989c1810dfad7e8b05f9d7ec", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:38b785d21ca83445fc719f46cff882c0de8f22bb6e98eb6b679e48f1a91f578e", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:04be027b9b74ab57374498050200f7821112c0d361110e5f6b611236132582e4", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:b4057060d79ce38ccc73b1f70c9c243071903fdfe56157ec93fc311df8a2318b", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:7a5b659d76a4be8d71b896716c1d4906c46a2780566b4f32aeaed5b5ce5d5877", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:7f45bb0202a1e2bfc8c203cc677e908e21151cd1ac93cee2dbe59058c21dc0b3", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ff83517e5943836a3e837c5cc8211affc9a635956c7cb3d4c702d3c318b09d0c", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:2db1c48df328ddb848f45632d4bc9642830ab9b20fd24ea983df41c7d6b102fd", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:ba85021f9f9310fba95ed738b3c28407ec715793507f87b3ab9095893e65e927", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:48b78684c5f1747a8f6d7b4d886795b7c94f5ec61e4d7df66b8df1071e82bfc4", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d05e790e92a36d536f75e3b5815faf4e47a4b86090f64a7e11b60ba8158fdaee", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:15417bf7fc8fd275433bdcbdc6b5ad2f2f50cc412cda21522650c370629fb5c1", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cb4a3f5ea0c63b7d71a38774a3cc0ae03b204e39dc4914a5158b21e41744d504", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:092bfdcfe9a821b3d94228e7eb8d306dac39781b3590f05168fb2cf6d0603bc1", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dcffb17b25263e050da84395241e5b092f824af65cfef8162f58994bd9d9d7c9", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:b9e50c236426e6529b3f866fe34a2b467ec27b5c3634593fe112883a855571a9", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:82d3e1c147a45d19aadac33ace802ecf6bd0631e6baf042bfd9151a6177570e3", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:b3078e611897dda7636faa8799c27272f6572b43c607d5ae312c622f292f78e1", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6344d6b0b18ac3cc05c4c1e5e21cb5633ce44d2533e943c67b60349676e1a995", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6640092fa02b2e76fa2e34ae7ae352cd6c189198af0e1402dfe9f0479f16b695", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:a0e2a5f22c02d86d12bf3aabb61693a97e34e74b4a80780e0ed0801ed04fc5ea", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:0456d839a4b54cc5705705b7e4b11b89ffa25c12ede3d62c59b76134996036ac", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0c5710fdb54258e73bd05b591fca7331706fc341fb493fcb1934317b576d63fe", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b18ac2eebdcfd45118562fc571469c2b4f02cfe5d2f4a2d1e0332fb77b4efd2c", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:508df53cf446eedf897e0291f90a907c1015936fd3ebe4b62634b1dd79534354", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:6c1cc5d2f52a1406f63cca060080f7adbb59138a6d0db1adf1fce9868b56db6c", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:b607b290ced7980a9c2f30c76e87b9b6d04ddf385ce36c4c3108226ba76e2deb", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:175fd848698e52ab57e2979a4ee2664576794740843621d3bccc7efd19d93e34", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0c5d626881be266c24afc143b1bf81658bbbb0f0fee4c908ff3915a0876d7669", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:c7837b5a3eb68eb56637384a5a3c64de20ea7adaddbc015613c59a119dd5f542", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:198a45ac0805ecf55f550a58fb447e38e46705afdfc653fd347ac3bd60ae55e4", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2a19e02d1b5087376e4e8f06df77d38a3e92cd1716d4f47c129d3047fcad37b4", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b9868731eb4e05566f3f411c21837f9ee7ae531def563f710090f8fe36724327", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0768cae6cc20c16d784917d82332ecc84a0bde87600344f9afc52ffc745c9179", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:48ead735def074f062dd117701e0e2fcc30886abe4267c685cc610ffec77fbff", + "evidence_path": "evidence/20260721T144633Z-d2280c5d/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 22, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run3.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run3.json new file mode 100644 index 0000000..3779096 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run3.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144716Z-94b5305a", + "timestamp": "2026-07-21T14:47:16Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:69eb9e8d160b67abc1cebdaca43762e9e4a57801ba080a4aea27a3485a68ee36", + "evidence_path": "evidence/20260721T144716Z-94b5305a/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:14a11259ac16cf8ccbf7fc8317f46445fc9080308c0677079b551f47119c6968", + "evidence_path": "evidence/20260721T144716Z-94b5305a/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:c11c5522b47eb2c04ef0939452441da2932f572cd1cd981ed530a1fa52d4a892", + "evidence_path": "evidence/20260721T144716Z-94b5305a/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:03c5b5050738502d984dd7b16a0392214875624cec15cfa3a860d0bce55d91a3", + "evidence_path": "evidence/20260721T144716Z-94b5305a/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:d3cf4044cf460d70675f23eaf662b4a1d24f771c6e3b5179c4c6c7cba58ad5ca", + "evidence_path": "evidence/20260721T144716Z-94b5305a/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:c3fb569547ff878d08fd9b946182651473c5c85e9d8ce16045357fd0e2e4bcf7", + "evidence_path": "evidence/20260721T144716Z-94b5305a/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:8b5b2f72e542365f8ff60c51113ee66b3f7407b19ba7190cb01b8bbb3567db9b", + "evidence_path": "evidence/20260721T144716Z-94b5305a/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:db82a59f9e57c18179726e2fa695d6bc5c494ea420ba1669bf258fcefbf7f294", + "evidence_path": "evidence/20260721T144716Z-94b5305a/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:6eab57de685edfb17d256fb8440bcaeb87a1f1c1b1e5873fcc9e352a50075137", + "evidence_path": "evidence/20260721T144716Z-94b5305a/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:281cda5b3fa451cdca5a1d1149257e9d454fae8a8871efbcb311e4ecb3a7d345", + "evidence_path": "evidence/20260721T144716Z-94b5305a/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:9ae89cc83a188fa19f079ceee7b37bb2cc889a58325b376a0364cdb9e2e570c1", + "evidence_path": "evidence/20260721T144716Z-94b5305a/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:77ed78d8b72c0f1443629734b69a6bd7a74d08cf33c4e5805e5910d29085cab9", + "evidence_path": "evidence/20260721T144716Z-94b5305a/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:0cf935c09620bdfbd3abfb6b05f488ed45c76483f5bfb94bcd8f11a5dee809b4", + "evidence_path": "evidence/20260721T144716Z-94b5305a/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a62bcc3b66268347f63fbc60b605c298a7e668ab792b27d391034b1c976868df", + "evidence_path": "evidence/20260721T144716Z-94b5305a/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:08046baa3f8598eb84762e51eed3128242f1a40c8f5dde0a87cf826ade1c243d", + "evidence_path": "evidence/20260721T144716Z-94b5305a/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:45b16c55f97526bbd4daaed90dfc304cbc34f9da4ceabea39acddc6cae9e0848", + "evidence_path": "evidence/20260721T144716Z-94b5305a/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:0022ad593cddbd9126b83614dff003899065b592b16bc63a9a1e6ee287442946", + "evidence_path": "evidence/20260721T144716Z-94b5305a/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:944c1a1e322328a1a92e16272f518960db86ea86c72ad9df9e5af4eb183983bb", + "evidence_path": "evidence/20260721T144716Z-94b5305a/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:84a21678d28154bb24fdcc2e6b660779b1979da1e8ae3e5e80d8055a565984d7", + "evidence_path": "evidence/20260721T144716Z-94b5305a/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:50d3bce61e2f8665752cf8e897d588d8c346423f8e5ef13c3388c574d411d357", + "evidence_path": "evidence/20260721T144716Z-94b5305a/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:cb31145abe45eec30ff510cc7aabf6b8b6e28bca5ea5af82286c9937a4d7b6eb", + "evidence_path": "evidence/20260721T144716Z-94b5305a/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:44efc563b95c14ee73c379f2ded5dfd16696accbe19d9ac358173b6a087dd59f", + "evidence_path": "evidence/20260721T144716Z-94b5305a/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:bc2c2964573f90273836f302ed05c14545718a472e8c54163989eb6126ff4d95", + "evidence_path": "evidence/20260721T144716Z-94b5305a/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:cdc6e8a3c79b97e510c62826c8b51367f598f4fbc18370dcd8a6b2eec48c0577", + "evidence_path": "evidence/20260721T144716Z-94b5305a/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f7dc63fee47c7d8273d65bb0d33d8bdffb4e923a37d65e152ffb46f11c1484fe", + "evidence_path": "evidence/20260721T144716Z-94b5305a/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3d61950ddaf26e9ce022dfabae792bb63b427808aa3a4b5db4fa1349ee22420d", + "evidence_path": "evidence/20260721T144716Z-94b5305a/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ae5c8ec34b951cbf1bd46099640b5ab46b3f988371e9afa4759fe424aa4afd6c", + "evidence_path": "evidence/20260721T144716Z-94b5305a/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:87415d062bf1499c7c426bc7cbc038ebe658749db3492a123d494ac932694838", + "evidence_path": "evidence/20260721T144716Z-94b5305a/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:6a597a3f2da061cdd466dc9251bb6261aa84ac71eb3923ed58960eb51f8b3532", + "evidence_path": "evidence/20260721T144716Z-94b5305a/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c64f19c3d65e538c4e9082471706277304a096b8030a93e06a808865cc08ba9e", + "evidence_path": "evidence/20260721T144716Z-94b5305a/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:4c8367972b5ecd0bd5f64c26fd937c97b7296cebaf193a412204a2a9590c791b", + "evidence_path": "evidence/20260721T144716Z-94b5305a/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:12e7b9f2258c8162dcc5b8bdc5fdb63a3e7352e19aae113b5909bb58d542c78d", + "evidence_path": "evidence/20260721T144716Z-94b5305a/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:418f2c79a57a7809331b4333bef304bcd3c117b04c64e1f89a786a94a7008bca", + "evidence_path": "evidence/20260721T144716Z-94b5305a/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b1c14b07e09e50be96ac16812cddaf099d6b7ab3c0dab9c734145fa95a7450d7", + "evidence_path": "evidence/20260721T144716Z-94b5305a/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fbf89fa68a8e4ca5a03c96e56425ea2327cada604b77451003a4c5afbe3a19e6", + "evidence_path": "evidence/20260721T144716Z-94b5305a/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:8fdc4744d19d9818ed8ac58b8b29ab144e093bb9bdb7b08d09045e0964a0f068", + "evidence_path": "evidence/20260721T144716Z-94b5305a/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:3be1226755bb4d4b8cacc15d26bc8b83ccaa1b9efd10881fb56b3527e9e3e3fa", + "evidence_path": "evidence/20260721T144716Z-94b5305a/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:51f7bd2c5f0a82ba8d8c8efc04f37f446bf1b175ccbf3359ceb22257165bcfd6", + "evidence_path": "evidence/20260721T144716Z-94b5305a/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:482d7e7ff84ff80468f18c9802a3c3c3455d7133613c068a2e8094e581458bc6", + "evidence_path": "evidence/20260721T144716Z-94b5305a/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c703895e1753c5e27363c13f429319d1810d3e42b7025620d18c7f3f9ce42e5e", + "evidence_path": "evidence/20260721T144716Z-94b5305a/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:08318c102330c68084fa373abece6cb42854c084c342c4bdf745a1719a37a1cd", + "evidence_path": "evidence/20260721T144716Z-94b5305a/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:6ef84014c52c0672deb8d98efa11995b251881dee1b160b6a3124a7fe14dca7f", + "evidence_path": "evidence/20260721T144716Z-94b5305a/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:53965026faf1f6023b262064f74424e8cd7839fd534439823cc41f23002cc3e2", + "evidence_path": "evidence/20260721T144716Z-94b5305a/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:873f4c6e05013618ff3aacc347cc6e4c14eef5a168f6334fe5ce9a0939be85da", + "evidence_path": "evidence/20260721T144716Z-94b5305a/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:cfc8d670e68ffec31d789474cbbbb7f538b105ee063306440f72ea48b0a3781b", + "evidence_path": "evidence/20260721T144716Z-94b5305a/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:789d906581405c5df33737ad0f9a25daf085cb792e15f8c92ed56b2a0fc7dc62", + "evidence_path": "evidence/20260721T144716Z-94b5305a/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3711ebe5991e3b29d0fb030d9f7294817a11acbb4ff1a66a2791a800abec5a80", + "evidence_path": "evidence/20260721T144716Z-94b5305a/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f5af1bac5b05551519d1dd9c8b8f748a77366963856ab5d055cfef5f52eebed7", + "evidence_path": "evidence/20260721T144716Z-94b5305a/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8f7ebae6dae33666c65082deff6a4e16a9723d13fd8d8e22b4d00c77444eb915", + "evidence_path": "evidence/20260721T144716Z-94b5305a/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f9049ab29ef36a38b873c4ca1d6ea4209edfaa49651d1e13a44ae810a49a6472", + "evidence_path": "evidence/20260721T144716Z-94b5305a/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 22, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run4.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run4.json new file mode 100644 index 0000000..2664618 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run4.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144759Z-8f9900ae", + "timestamp": "2026-07-21T14:47:59Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:09fa0c12c3c01127c6d7d218e957d88eb00929aa6e68ae3fd5cd42c32deb02e4", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:54bc42024bf7d20d661e167470d4cd1f725d2e1d414c2b0e37efa741728669dc", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a5fedcf3074e10a2577f9acbe648c7f9ae4c9b07b9b999e35badf48a5db11647", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ea4a4b8efc66f2bb28881e1128d1107c8e88023bc692495d677a3e0ff0223bc2", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:c7c3a292f9abbe5eb9145cb94209208314101d01485eedd6c394006fc6efa574", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:2cc766a1036f82243d7d19dc9df6bad0246422fb2a3acf431d5ae60305e8e1a9", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:237479ad78003bc381687b43831b7983306c14e146d6c656b5561d6572f7448b", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:670e0af45500e944fd83607e34097e24f072f095848d4eea4be3e72d39ad8564", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:e6edc616251e0eb2aaffd374cd3c81017df4616f403b60ea7df4140f0cc84436", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5aa5deeb98724ca93acb7d8aa4ecb0b98cc551d8e0c91aeaa466d1ffc22391ac", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:41e0b1de926bd27e0d9dcac42aad934a2b17b827e4fb084be3be98dbfd5f89fc", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6ab1cb4ce507805a8c740c46c4d9861966dd0f21056eb63569635a26483f102a", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:a68dd3140a6450486bd4900200ea00a214aac50f4aa8e571667b14e53309196c", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:27017285f9de4b1fa7fe4da2c3549446dce69964272ad6910125f83d39eee300", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:1e132d0066f7a528809ad41a5544f31e906d8469fbc026216e2c492da4b64de0", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:5ac15469f00bb42a8a76a402720aba640665c9da2231f7f670ec20db2816703d", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:1828c01e473034b5735f1505a1845e966766259c0d452710f460141c3a07d1da", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:6f50424cda0b7ac68bd5f5f8a6131c4d05c6caa4f864105ed6a666c7adc885cd", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:61e0e54bba4011ae7db9655cbfa47dd19f702d5e7aa51f7bee5528529327a81e", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:e651845c09e896014302d871887a6cb66c9b50abdd76e757ce6efa9ad1696158", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:71029326b89aa5a851e27242cc3962f0d96721b9b0e2838da842836f1e5a5ba8", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5ef340fab7cf830ae3569281a190d828fb7410232399f8279c9d1d1287730e68", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:b3638696128d14d44d4bdd1fb565138359978199d0d2be40f236825972317849", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:8e528c2835b7cf001260cd431baf50667c08d90429aa3b004d4b81b9c0964000", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ef1c22fd1b9ae425525724b2346c51ca2614a9a13d0ddd5a18c69d8d050bd7d8", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d053e727eb21bb4bae375941268c90c49f7982cddca9ab7a3aef24c6f5fa6394", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e2800d322b2558abd11701942e404dd7323864ac6bdd907324a6e6642ef3e7a2", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3737db7988d3cf8b93988eaf9d6f85c69854d9d0c0b2a1a27dee3b017a786969", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:965b1abe34cb79e9b829fbf630b56ef556a423e1debf2120e6b8453c34c68d64", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c1fcdbadeb660558d434322166b066934f828d78cb2c625c239acefaf7ab6014", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:6d065911d2773d3d174e60c47725e24addf463886b5b896c52ed60fd62af9527", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:643e77f86d0ef3395197f6d44590b148b5e75ef7d08ff10630b1efbaa077efc9", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:1530472d293695858da1a07b27cbe2e2127c7f9377ae13623811b0b27ffa6bcd", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e85b33325356dca6798f9134fad3755c0f9c6df95f8cd884529918c91fa05a10", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3fb47512e1ca8ec345d047c34c49ba2fc24ccf3242c7ef78e66d11019c9a3637", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:8381aae98abf31bc03a38208a7d68dd0582ae938b9611b56e12ae5b64d34a3f3", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:6953e61e2227ef6bef78492f093cb0cb9c93c4aaa20e02749cf9b435c31527eb", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1476ccbc73801d8042bf61e7e4e56dd4a518333e435a9e7b3d89c14abf031770", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:78fafbe53b66a959b9d97ddc4e14ee944934be1d49da92413bdb20b58a7c7d46", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7823dc5348c5d266ca0b734d25e3774e888ea0816d4d69878a9bbb00ecf90912", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:f64b18e9af474662347bee2f703a657c9cf64594b1f997b3399d52cdcc87e5ba", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:02905d53c7f531be7b6d280a0fe0e939a49c4923b00bdda0ffba6037353fe59b", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ba86153787f8ce86836964434f3cec51630a7a0e1f37b95d7166cb2ac82e94c0", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce85b531bfb1a6c8cac83818229460c41e5fbf368942f1c22d91d17da70a98b0", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:488e9ccb0588bb846055d8d06956c29516de03381397f710927a64c130aa916a", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7c995ce050de7546e2bab730553b9b33f8a706dcb62852339ea6123427f7c37a", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:074faba30c0b13ec3f1143fe62a3a787549c5ea6588fb9e538c5b2894e4272f1", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1f51b20e25feb86a74c0ba92661eac89ff51d8924a9ffb53fffea2cffb0c8615", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:59046de41a1a4523ac24b08ed09df6dc6731ef38f3140733bed20aa357d13122", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1c4234e7f0b9d33e6cec158de81f18d7820f306ec369e13149a0c9c0ec1b0807", + "evidence_path": "evidence/20260721T144759Z-8f9900ae/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 22, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run5.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run5.json new file mode 100644 index 0000000..6e25a5e --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run5.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144842Z-0fcd3a43", + "timestamp": "2026-07-21T14:48:42Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4105f6776269040b0ad32d3f00cf0a21b1ace90b55f84151b21a63dd1c7c1622", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a9dbf63832a5cb251d86624898efb49d07b0f21798e101dd7ad82462cf868277", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:2a978afa7722722b460bdbbcf40451e588ee47e40b9fd837461cbe3268151225", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:6914435690f55e24069576ca46fa7a633cea9ebaa48bc09f322a3e607756c2f7", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:98327186edbc239d262496d7698bd3410883293af1219934500c96fe6462da52", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:2441af3bd89af6127a4f0b2f3091d0b4c33ff7b7c0533b54ffe3c55f3d00a291", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:fb0ecb7d31c12c29639517e19989ae7cc42c4292d2bd9568d22270073bea5226", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:6933d10c5605ef29d82935356d9d329122668aebcb8aff33a999802acfccfed4", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:0cfd824c0e928d27e8fdc5494dcf6b4f7014c83bfcfaa2b0a78875d6f0bc170a", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8554ffb9dc52264cbdb40dac067a7b3de30bb2a6a5744440e768e52909d6eb24", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:1047479e03b4868981d5a9fe0fd51040ac035cb842bb61526243ef5b69d5692f", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:22e3f44c95e55578df9d5a4fea8238761de2ff45e25cb8c976c06380921f1a70", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:97fca9253c13193177d8103ed6e142b17b924ce7a828cca6b4ed9319dbcf418e", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3da4c60287dba866773f4c0f46b98bbbaf89ef9b0b53e1da57659a0c32464996", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:b6ecf40810232d8125aff2dd84ddd8ac6722be014f98094779b8979ff36f2518", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:d7e0073a5c79f7ded31079188c33c71cf9d15693261b208b4ced63765f2dcce3", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:53aeaa588950eec4097a80f103edaeb3d5726605f0ab308aad9ec683ec41e9b5", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:ede6c9d21834dc7f1e0bce76d0b7fd88677f568fd52447ba040a3dc607c23c3c", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:76272e6ce20812d1de7462cce263e646a1b6b0755c186660d1687c18388a980a", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:62ac164229fdd15359a0462fa4fc9581973dc37967cd4747d76262be1eea8f23", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:525608c29044e363ab0db13dc3cc48b130be35a9c81bce0b9c8a7911729dfe0f", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:32aea3ffb30fb9701ce001910d6acdf56dc8b334b06118da9762251f06d6ac4a", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:6ad560fc6c43dec2c05a3f29006583b48c5017705a90d6ca1366756a582572ec", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:bffd0d9b0cb65f3135ba41de10a852c7f6c7b79491a1ee80ce2e6196ab912361", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7a9f3a5f9240e60bf4c7aa093eb72ef2265d871adfe6e40330dfd97cd921fddd", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d5326cf2b94bde7f3749b06b501b2eaf0b1331afeee446d1b50574233d658a21", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:788cce4e456d0593bcf9cbb56f09fee466e076f45b34a41dcdab81792ee97fef", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb33eea4d6c13dd92ccbed305e7031bd123672386b5bff1899ead8d70e61b61b", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:280a79761679d213056d1ffd14a960eded46a9444ab6dd0b8357208719d3803d", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:008f3231176eca4a17d71a060da7bddcf20befe5e17bfceb942b78b0ac2d8055", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:6752266c94f9b95d7ee46d7c32a99890ba6ebc07089cd2012e4ad55f04f93eed", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:d7c0edffa6f05c4807cbbe05c684da3403c25b2b3a0a343359aae304d930d7ff", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:6b01beb7deabc068d55d4a1c03c36bafd2b6d28631cff860cc6aacc2b422c681", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4b1f5010d501247e1665ca2bc07d67bdd99373a09a82dbcdec3fa9454e7ec73d", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:83dfc4faaaeba2f80d3d3c15e34f1599ee72e763703d2c9f455e573b51350121", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:070485d933b4e19c8c975fc0fa621306f33632f32d213777b775b74d1ad195ea", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:b414c3ade058d8b0b45c30e9bdf76f74c2fa4063417fe37de2e49362c5773828", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1095999be938eb3dd589e1e0f370298116ca1b2c7166a350df8ba38a8bbf277e", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9464e8060d52d35087e14951d7e15eff66ba42a0c4229aab7605be4ebc16eeb5", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0665307dfe51e1117a70b66ee28532a098ec4c046166af2a3ef7d7c92b986af2", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:c19bfd259c0fa2bc1009d0f0b66d1a937cc3d7cebbf81e754cc906276c15f544", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:805d778db8083ffb98a9a496df9e387181e43b2902f0efdbca34093e2d8ed990", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:03bd4a6b9e1376292c0e5b86d0e74f3d1f2c79d986526dbc0d56b7fbbce31512", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9fb7c0c448a5f23032652aa10d7d0eba412b647205b890820671d779a27ae777", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:1ab3aff41f68b1b115e8abdef9c4e4c31ced4d1025d4f84c526f14a3f9826c8e", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6ba918f2df3c6826915add4c06c82b8656c6727325ffe4cfb882b2b0233f576e", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d87d8b19036c2dc0c2cca3bbee3275926569c55d3f7b0817970fffdafa3e908f", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:489492c0277eb34f2ce916f7db7f909db00ce430597646dd4af54a5a5ebeab23", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9e8c2e1b0aefae9b2553e3f6a7a1a3b33da1b75288c9dde4b57ab5ebe860bc16", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3df40333d9fe06b8e67cd4e3af7c63a28e0d546840ece4cafa1a4ec2fe490af3", + "evidence_path": "evidence/20260721T144842Z-0fcd3a43/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 22, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run1.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run1.json new file mode 100644 index 0000000..0d9030f --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run1.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144405Z-d06e4310", + "timestamp": "2026-07-21T14:44:05Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:8e097d8814791ac6a6042a7b06ed7cb58d43281a07c90e8de208c398fbb17874", + "evidence_path": "evidence/20260721T144405Z-d06e4310/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:0873b994b8a6fe051ebec3daa5e33c48e0ccb61727ea2a092550ab1c3eab64c3", + "evidence_path": "evidence/20260721T144405Z-d06e4310/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:7b7f2ebc1ea84f6066992744cedfd50050f794ed5c00caae869837815b946901", + "evidence_path": "evidence/20260721T144405Z-d06e4310/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:6f7ba974c4efb52ae953e689d35cb15c980fbe30c95a4e9d50a506dc62174011", + "evidence_path": "evidence/20260721T144405Z-d06e4310/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:9f0901e02e49d05ccb2985f6467650d5956b21f5a55f95878f7056c8e2d4e59c", + "evidence_path": "evidence/20260721T144405Z-d06e4310/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:02767efa6f3a2481e9bb4881b746cd61a3ad4a274501dc7db32d62bedf334a03", + "evidence_path": "evidence/20260721T144405Z-d06e4310/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:add22215a98ee07ae7358868bfca4c113b0b3a809e4b4b62831be8e54f36285b", + "evidence_path": "evidence/20260721T144405Z-d06e4310/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:647b87f053a6a30f11254dc481a61fe37e2e8448de43e4786a0f3d0879f23573", + "evidence_path": "evidence/20260721T144405Z-d06e4310/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b7868019f91eb58abf113646b21bc02ad9db179f2d977259c15564f5598cc479", + "evidence_path": "evidence/20260721T144405Z-d06e4310/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ae96f51d5ef4b88f6e8e34166fd72aeeb724481c9697d7b1b7df652b46306b01", + "evidence_path": "evidence/20260721T144405Z-d06e4310/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3ec7cdd013c250654846f388950c980036cc71272f5826b519c1333ba2045e0e", + "evidence_path": "evidence/20260721T144405Z-d06e4310/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefgijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:ba2fbdd1be14e55da9543bb507decf8f6413a19526e33a178fcd779bcfd0f93f", + "evidence_path": "evidence/20260721T144405Z-d06e4310/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a357d1cbf6bf493c8cc903bf3388192a7c4fd6e2fb4b7f465f6f15f5146b2b06", + "evidence_path": "evidence/20260721T144405Z-d06e4310/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6347cc56ec1857d337f9fa36efe2318796b22256a6a00a0a7f401e91ff893622", + "evidence_path": "evidence/20260721T144405Z-d06e4310/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:289f770514faad5b231dc3b33ab4c6b86773901622f0b3aa084ad25628fd55b7", + "evidence_path": "evidence/20260721T144405Z-d06e4310/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:019dd65dcf2b67a65f1fb8e52ea758bd0635a2ba5ab319b688426fbcea79305b", + "evidence_path": "evidence/20260721T144405Z-d06e4310/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4497f140070e3d3baeb4cfc5e23a83c9f282cc9fb54593db7be6938955ae0b22", + "evidence_path": "evidence/20260721T144405Z-d06e4310/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7c44053fc607bb302492bf84e1c7eb6910081668d6a9d2c9dac45ef701222390", + "evidence_path": "evidence/20260721T144405Z-d06e4310/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7d4a31428621922065cf7161fa12cc2bc67ebe8cf758a6d6a0725c0e114d6070", + "evidence_path": "evidence/20260721T144405Z-d06e4310/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:375377e1fdd33ab635fd974ec54338914eb901b4552b5ce6924c8fce720ea1d3", + "evidence_path": "evidence/20260721T144405Z-d06e4310/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:bb1bf79b78dfe5485c9fd814abcdaf56acf37b48d7c68270e89550c41860a601", + "evidence_path": "evidence/20260721T144405Z-d06e4310/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:87660fd01f0340959e04c284b2b4dd23fe9632d94f6047291e17fa08d5ca2037", + "evidence_path": "evidence/20260721T144405Z-d06e4310/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:620bc385138bffefc2d0629116b4759040fc802d54f9082f138cbde6ecd53df0", + "evidence_path": "evidence/20260721T144405Z-d06e4310/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f3bf0de574d4f24eb515696631c2389f3cb1b0eb90a984f85dc52be9b1b403af", + "evidence_path": "evidence/20260721T144405Z-d06e4310/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4234a55d4a231b2f17388a95c677352dd32abfe65fc888795892ab113f265dd9", + "evidence_path": "evidence/20260721T144405Z-d06e4310/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:712c6ecda49cf584064d77216b4efa14caca3960e84f080806168a6e932615d2", + "evidence_path": "evidence/20260721T144405Z-d06e4310/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:724ce873cb1d43dba0147f60ae40c458b4adc728ea4d273245d06816af06168b", + "evidence_path": "evidence/20260721T144405Z-d06e4310/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ac874cb436af518fa156bc9baf34b98ddcd8913dd43bb9b94f04e16b04f9657e", + "evidence_path": "evidence/20260721T144405Z-d06e4310/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:174e09232237ff713c69c98d636a959d2a0e2da3cb2699ecc696eaf7ef2670a6", + "evidence_path": "evidence/20260721T144405Z-d06e4310/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb530794c4407832552d7eae0b5b6d3c384acb8da40c276eb5a7e95b31075772", + "evidence_path": "evidence/20260721T144405Z-d06e4310/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3d0d5f939760115bfa54f5fe57703fbaa4f4b844de1fb8e3622f3dcff69cc0a5", + "evidence_path": "evidence/20260721T144405Z-d06e4310/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3bdf311bca542dba5dcd342702ef398f081afc465ccb83024213a2afd2b623ea", + "evidence_path": "evidence/20260721T144405Z-d06e4310/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e61817e4f279ba27cf04084acdc76acf0dfe4d06c5e280507aca5511b1fe34f0", + "evidence_path": "evidence/20260721T144405Z-d06e4310/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:899548036c0600a3338890a615dad1fb1db9c94e98c6112074833f2ff149b057", + "evidence_path": "evidence/20260721T144405Z-d06e4310/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:065be8f8721e695eef31e6fcaf8a12205fe6fcc588a00b54ad1df5bd84eb8d6a", + "evidence_path": "evidence/20260721T144405Z-d06e4310/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:794773baab2add2d856c7e76cd79dfa7df448478cea633a1abf6a4c82c15a242", + "evidence_path": "evidence/20260721T144405Z-d06e4310/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1fbd5563f464aae8e8aec13ef8b502997bc5fbbc2fd5910f9e14ea65ad2c9f72", + "evidence_path": "evidence/20260721T144405Z-d06e4310/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e7458395d3605d91ec87e19659dd89f6af1525d4998cee351612e069ff780663", + "evidence_path": "evidence/20260721T144405Z-d06e4310/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b450af1ee6148b55349e820d4b02f165951770bdee108cb43a213367ffa16fa7", + "evidence_path": "evidence/20260721T144405Z-d06e4310/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:570233d49a4d68735bb1bdf14e9eed0151b320cbc381cd3c710ae721b025f3d8", + "evidence_path": "evidence/20260721T144405Z-d06e4310/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:248a589fe2f04e19bcf0e60dcfb354fe193dd57e9ad4e29f00ed055a9a1a477b", + "evidence_path": "evidence/20260721T144405Z-d06e4310/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b3175e96118be02d0baa5776abfc73e9178fed99a5893d9ca62839a96f19e11e", + "evidence_path": "evidence/20260721T144405Z-d06e4310/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:207d1b6b913e9f7a68437a3addbd53b75b8cd5dfd2d5d14a6857fdab2c3f8552", + "evidence_path": "evidence/20260721T144405Z-d06e4310/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:161f4fd5084154b75f9aa9376e623ffec199973d6669c050bbce4985749abffb", + "evidence_path": "evidence/20260721T144405Z-d06e4310/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1f44ee061c320e244528afa692ab3e1149b4c676e45a0d18c9280c967d27ba46", + "evidence_path": "evidence/20260721T144405Z-d06e4310/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1f338b884a603b74a7da008940417d39507377d54179d1a8819b83396ebca498", + "evidence_path": "evidence/20260721T144405Z-d06e4310/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4e14186406086d9594824ae3a9e6bb3177bbfbe45948f8d62c99e56b2367eef1", + "evidence_path": "evidence/20260721T144405Z-d06e4310/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:af5c18194d8c86ca14d25fd8dc5e1e029ef7a2fb87b7c300ce1be422e2ab0328", + "evidence_path": "evidence/20260721T144405Z-d06e4310/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e1b4550588322048ce0b9c79d1978c012ce404f62aa0d3139cbe3f2ed438184a", + "evidence_path": "evidence/20260721T144405Z-d06e4310/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a45316953c87f76516d551aea5d0801c7f1e04041c695e194c00f5c203f2f0de", + "evidence_path": "evidence/20260721T144405Z-d06e4310/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 40, + "failed": 10, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run2.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run2.json new file mode 100644 index 0000000..77f25f5 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run2.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144421Z-93f5af91", + "timestamp": "2026-07-21T14:44:21Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:5b0b0f898f116ba042024ea29fd5139a737a1f33a1544da29b42485f9c3bf800", + "evidence_path": "evidence/20260721T144421Z-93f5af91/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:f47559eb62fd966a61be8573179d3966dd2f71fe5f45bf65b1346cff0b109d38", + "evidence_path": "evidence/20260721T144421Z-93f5af91/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:082a28f5152b99ef6fa545dc04d0df1231911691aed75ebce6bcb9f83387dd9c", + "evidence_path": "evidence/20260721T144421Z-93f5af91/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:374d53d6e3396b2ab9381ebd4b6a6658c158b28d5eb94aff76ea643515360cad", + "evidence_path": "evidence/20260721T144421Z-93f5af91/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:0ed1a35eb3e634fbe1a64edabe2f667fea329d45538ccb89305e3d7ba58f4204", + "evidence_path": "evidence/20260721T144421Z-93f5af91/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:99e87bb86477cab90db8f2c51624f80b005232a93967c9ba780275f1713ffdb0", + "evidence_path": "evidence/20260721T144421Z-93f5af91/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:228811afccfe15da5686659478abcd8783069ff78d576043b14447c8a10a3eb0", + "evidence_path": "evidence/20260721T144421Z-93f5af91/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4b6f7e4b599ad2b856425f2325e8b642edd1f6e065175f67ec8ee43e167adab1", + "evidence_path": "evidence/20260721T144421Z-93f5af91/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5b045585a6fe81f53f45e4a4c075a2d6f536adff5d2dbc9a00c79701826c8348", + "evidence_path": "evidence/20260721T144421Z-93f5af91/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0576a5170763ee9c33507700a9e0eab46fd80d43c4ece9ad163b58af280bad91", + "evidence_path": "evidence/20260721T144421Z-93f5af91/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8618f88f15312261ed1bd195c04587a340d5c2fb99330d54329568895bc496ad", + "evidence_path": "evidence/20260721T144421Z-93f5af91/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefgijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:69b356993048247071ef8dc705680b1bc2dbf181f336414c2d667eb931787624", + "evidence_path": "evidence/20260721T144421Z-93f5af91/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a9fbfc4cac42a1541cfffcd449533de678a6ad0c4ad05586990d914f56963d23", + "evidence_path": "evidence/20260721T144421Z-93f5af91/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3a69404795ab33c414349850dd97ed1b750c9e707f25f2eb466002f60358b7a6", + "evidence_path": "evidence/20260721T144421Z-93f5af91/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:debfc07833375b81a0f1f87dee4dcc150ca91d2b0b75a4dd0cc9564939823a62", + "evidence_path": "evidence/20260721T144421Z-93f5af91/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dea0846cfec6d7a5f93bc1034d008c66bf86ae41e7ff946e009a50a82fa8258c", + "evidence_path": "evidence/20260721T144421Z-93f5af91/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a4695c446e69286d1a8e399e90ebc4f1aebebc0f4148e54fe7fadefae9b9339b", + "evidence_path": "evidence/20260721T144421Z-93f5af91/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f1f321ef628e718a97b11424f94e290afa43c71763427c77fbc3fb95618dd4a7", + "evidence_path": "evidence/20260721T144421Z-93f5af91/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:00e5fa673e3c3290df75ee6c56ce19a0ab1b3461c8023bfa9a53c9ec8dfbeda2", + "evidence_path": "evidence/20260721T144421Z-93f5af91/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0335c7f498e17fe2b572911c32dc457b58ac2c961d29aba0eb4805656d255fa3", + "evidence_path": "evidence/20260721T144421Z-93f5af91/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:56493d10074e5db4638d306475db6293c448e747052d52f3252b018c31c4b6c7", + "evidence_path": "evidence/20260721T144421Z-93f5af91/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4b84b662875259fe7f6fd73fd3fd6cb54748beb655ef20367689601d51bad12a", + "evidence_path": "evidence/20260721T144421Z-93f5af91/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d971433b7cb9d4264d91c09b0ec1a33360c63ef1d6c064be8e21d4aaf317f204", + "evidence_path": "evidence/20260721T144421Z-93f5af91/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a750b7b19ac3325409f91546d7baa940d4ac068a9aaeee7a1ce87a7ec14f91f8", + "evidence_path": "evidence/20260721T144421Z-93f5af91/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0175715e380a541c5ecfbae8196812a7d38730b67932fef015b52c88e32379cd", + "evidence_path": "evidence/20260721T144421Z-93f5af91/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c473b30da062cfab5490b8ad595f939251083dc8e7773265f818f60d8106ed97", + "evidence_path": "evidence/20260721T144421Z-93f5af91/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6645cbd42e1147b6352e0d9d4f2eeab33b0601a7ef4852f93f1c5e12bde4935d", + "evidence_path": "evidence/20260721T144421Z-93f5af91/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:68433633fb475e327f2d0009e0054e27db6f35f2918eb88ee6d5bcbecb4a4d57", + "evidence_path": "evidence/20260721T144421Z-93f5af91/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aaa1d51a4a504eeb510d0c3ecac4fea7cb160ab2bcc652e4484a0ab4c4bb7a48", + "evidence_path": "evidence/20260721T144421Z-93f5af91/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cf39f93b915cbaa332e7b72f5333f7599c10a42943e41896788679371d04151a", + "evidence_path": "evidence/20260721T144421Z-93f5af91/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6e0077eb86bfc64ed3a02911769b10e73d757e0472a0c20bad72ce32b3163f03", + "evidence_path": "evidence/20260721T144421Z-93f5af91/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:62c9f58fa6ab529e49549d2daab25670443390aea7ce3b6a226a1eac0e0855db", + "evidence_path": "evidence/20260721T144421Z-93f5af91/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1e3c9f023a457bbbffc05bc3c1d30d79f87162886458f9744f11f7f390883dba", + "evidence_path": "evidence/20260721T144421Z-93f5af91/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3b0f75b68ce0552d8dbef444c1d32cea93f6f68f27ce7bb184aa0fc2f3474bb4", + "evidence_path": "evidence/20260721T144421Z-93f5af91/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2657d8516f04da4390d395af298eba3c6cf7966c3537376dcbcc4a301ae9df3f", + "evidence_path": "evidence/20260721T144421Z-93f5af91/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eefec43af60a0d2a2039d7454586b216a3c33eb37a5213fb313b6430f92717c3", + "evidence_path": "evidence/20260721T144421Z-93f5af91/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cebda7bed55db6f96c3a3270b7e0b13e82288ef12a49054a0f98ad74481ffe4d", + "evidence_path": "evidence/20260721T144421Z-93f5af91/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:10c28420e8e36aebe123028b8b2fb68a32a521f3d6f2ad9e654a5761a0ed7650", + "evidence_path": "evidence/20260721T144421Z-93f5af91/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce0759c21170012019e0a4b6cdb8310d55a11d7ac1814dbf46cd6146e2bdb78c", + "evidence_path": "evidence/20260721T144421Z-93f5af91/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bffc1f400f90103024bf485ab6a349836cf15afff459016a6b2586e08e13893f", + "evidence_path": "evidence/20260721T144421Z-93f5af91/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3d0724cb64e5b58da606c4bb2d1b1bd2f76dcf53e10b9cf01adef51d9e38ffbb", + "evidence_path": "evidence/20260721T144421Z-93f5af91/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d6c4096b2dc3fdc9afc099be4b3dd71fa1837c84eef1e06dc1ce890f91e39a69", + "evidence_path": "evidence/20260721T144421Z-93f5af91/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6ba0e0ffd2a6f13739a04440984250c4443f6be2d1d2f76bc641c82f9c8e1fb0", + "evidence_path": "evidence/20260721T144421Z-93f5af91/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:132f54ffef0449f8a20ac54b9f8eddb8545b24f5e515626c9a21f46dad382982", + "evidence_path": "evidence/20260721T144421Z-93f5af91/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:755ced792f707124d71c8972d73be12b65665c0b5c28303c91682945519b7cf9", + "evidence_path": "evidence/20260721T144421Z-93f5af91/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f4318ec21f1dfaa47d7c9e9488666e8c4061a038b0785fdc228e8bc5b3b070d", + "evidence_path": "evidence/20260721T144421Z-93f5af91/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ffa5a76494f69ffac46c5b3a17c63c34af8ba27fa6d441640a831dc847656e11", + "evidence_path": "evidence/20260721T144421Z-93f5af91/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ec609741705a9142ec6f57c11c4b826ff4f0c38e405ea8d6a87a4501b954ec1a", + "evidence_path": "evidence/20260721T144421Z-93f5af91/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c38e399fc67058a76b875fb8c45870ae911ca197aff8afb89f8ec91c3fc18d2d", + "evidence_path": "evidence/20260721T144421Z-93f5af91/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:26eac8aa92abc39cd13619f81346e3d237ccfcc123f14604f8166359ed976f22", + "evidence_path": "evidence/20260721T144421Z-93f5af91/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 40, + "failed": 10, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run3.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run3.json new file mode 100644 index 0000000..e2963b5 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run3.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144437Z-edc49bf5", + "timestamp": "2026-07-21T14:44:37Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:598bf1bc8e52767c526a7a1fe03fde2ff6d0927791889c225900bf57df614c57", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:3806f09f2987d94de700183488d630c1d345c59940ba31d5c4fef8251db2c61c", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:e4773c16e546be79128f51a014c3bcc88eda70639f5bea1e37e58a46a6890e25", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:61194e50141800064584c9786985553d0c536f5ba71139c4435314698c880f1e", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:2f74751ba7b5a8bfcd5867b07209ff7d1281cfa4d292b8678537cbc4815d9d09", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:dd252def558f719fed66e481cca5f44dc53a348ec725344c9a4831eff7653c88", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:d4a62330031428592a50a892acd05ff86e7e91b558d146d4da1a90b4b9c8d6e8", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb9c95f9c36c257e3f7dcf65726b8a18f748d5841f82201c9eabeca6fb0cb107", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b5360da4d1409692ef471d8e5574acfc9ca66a38a423938abd78109de57e9b12", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:08b0c9242e74b5b22e20c9751f0039fc08ce1ef51792480b232a4b742cc5f583", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:901a55893aba30ce26ae7e897f0308c2ad580d99ffca8510bf29f7e5dca05421", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefgijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:c93cea1bc63805a8c0e216f0a7d30150849db5f1f4c1ede0089cf2f137e714f1", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1484a4ae1d9af190321b49f4b6073a269221e261d7ff349fb03e7b978de99a20", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b1c204dfb3243ef0a7a9164b77e16e3c9c1543bb4762b567c17d4dd90aaae360", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:4087964c111e437cc3756f3ae9724f71ed62dc097d5913d0debe4833fda58116", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d958c7a562fc4c3b8af140a42542da6fe48da16440976ce44a275ec1d6907326", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6f7d1db7098069803edc4b3454a787d42633b347ecd41e3af4fd93be893707db", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:07fbdccaf122b99517e56f7fc780e7e3de95de3c38860e1a0a82f575a2de5560", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0238a6bfb72aa0c653256fe2b196825c48e12a1645f1c7f8a746751a90393205", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9a5d86f9514ec374ac71eed93d40721f8d8ebbfb1e978129d4cba0d880224345", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:6e3ce31b0c6300179d745cbc8176a3dee1ef2eb7c9d5a69f45a8581410c3945a", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b85596a50a04321c0026b78f4bda7f60aaaf08e57e52be15d945329534a2de2c", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2bd85a2d4603cf2d234cc7fb73a7b283be4c9a819b9039af2e35cd33c8355113", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:19a8a0b457f7d7639b16ca1cd5256303fe33ee4e2197699099eb56782a0b041b", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bbb7f546b84a75aa134c472bccea80e328c24325b2989f4da1ca12dde2b1ca0f", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d338403c1fad029ebc7f7e362d40263091bfaabbadf232561f7ddb3a8c6f045e", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a7f819da9357fac36ac17e1012f57a8178ab31d34a52ca62247f7605f42983ad", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:09d5ee298a9f879c7fd2f9b678453bd48ea9779e9e136e1a88887cf2579e485d", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5192640f0d51ba555b689d6016510344262624697256ad87012028ccc69312c5", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:976b137eaa9b4056d7c4881a6108c4d8ceb00a90bf3d740dbf8c4a1ea97d9d0f", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a24707cc830d4e5c9082208f7681c4ff29906e248a133026fbbc77a76f012b32", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5933fc5a7cfbce2d2ca098994865111b39daae05b1ae7c34512eaed07120c4ca", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:10be26099efb1e3199e0aac17a56d96378fdebeba6bdbecd265d7f05704735c9", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f0754e453ee413a7a1bbdd7bdcf9e5c0bc27d594d0f47c0293b227f49215baa8", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a3ec39b13458c1f2a54e9cefbbbe4390aafe65605dea4ed971798055196aefd6", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:453ab98195bc231c7b96e88e998c81bccddae33d038db67d852d2144e109c84b", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b9947fc1c3f5693795e59a1f5b3631245cfd819c956c9c63deb93114b497aadf", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5a53bc533c3a7af47d23c86f95e9516daa0b902e096f5de4cc29c6293d319ccd", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1e3fb0ffc478d2384548167263f1225fb3a1125e4fa41d72bb22fd80b222cf19", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c7b34a62b5350de07ab7e38a5bbbc86ed654d98c56153c589d49446de4e11e59", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0ae09e5b463bbb0c0439f0c0d0b350536a0943efe2f695d70ccafccb3d981411", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c301fcf59df7ff0f10a2f908c31039131294ffc7e831a516a971ee388573e7f9", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d2f254839f80a5c49788608b9281cb22a087106c05e3873a0ccb88ab6b3f25cf", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:faded4fe3cf4a4cfb62331a7eec13abdee3a070ce6eeec820cb1e01936c78c4f", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d0baefc62473ea7ae7a0433f7bc299f8bb99bf0f26c47ff9c21eb1ebab5597cf", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bfbb3aca1858d9ac72d4d33b1aa64573b3cf465fa0c7c0105539817dcd29d4f5", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6537ca1172c162f40ec34bd4ea318807126f044ff1e7de164a122c7d5008e32e", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d15b98a47d5472b1cd50434f9926910d75a5855f80785e8f2678050c387f953b", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:320880d2053a8484e6ba40d706dddaccca771a6649797d328c0ffe1ab992f28b", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:79a3af2aabb64705bfdbc77e57a7e54aabcdd2496198d764929ad25c3420cc1a", + "evidence_path": "evidence/20260721T144437Z-edc49bf5/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 40, + "failed": 10, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run4.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run4.json new file mode 100644 index 0000000..388056f --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run4.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144451Z-d518cc58", + "timestamp": "2026-07-21T14:44:51Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:d3f0fd75f32453a72546a01136a4e6f123496ec04109509228f677d758baf992", + "evidence_path": "evidence/20260721T144451Z-d518cc58/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:90f5c4c79b38c95598a879d4b97a449d062be0b6bfd091a73119f451e624c6ef", + "evidence_path": "evidence/20260721T144451Z-d518cc58/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:c814268e013a77000d504f20777de2984b7c581395343d6caa4c77ab923f83ff", + "evidence_path": "evidence/20260721T144451Z-d518cc58/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:680f6053bc95afb44aef0b49e05a55e126b3ff065607e8479182eeab5dcac878", + "evidence_path": "evidence/20260721T144451Z-d518cc58/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:e18b729ccaa67add8367399e804ba47693ee40ac024b1f2d0a4ebffc016b9307", + "evidence_path": "evidence/20260721T144451Z-d518cc58/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:af5c94728393cf467ae0757375b021491e0954e71d951c0b092b2c916232e21e", + "evidence_path": "evidence/20260721T144451Z-d518cc58/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:3cfd46709a15caef7d1d9bb47435cb0f9164009dcce191508fad417c275ad9b6", + "evidence_path": "evidence/20260721T144451Z-d518cc58/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6376d37cc358421ad5eb73274c60137b755922ac4b70387109d64a1324f42e0e", + "evidence_path": "evidence/20260721T144451Z-d518cc58/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7056755189f3c5efd2c0a329977204076c5701a9787f3e34a173dbf092d58efc", + "evidence_path": "evidence/20260721T144451Z-d518cc58/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2c78c9676aafb760461ed11b37332b5b655eadf57a72dc7904f98004063bfa89", + "evidence_path": "evidence/20260721T144451Z-d518cc58/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9bb294602e6da9f5c258f8796c7a249f05e40ffac96be350fcec188d8c1ea41c", + "evidence_path": "evidence/20260721T144451Z-d518cc58/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefgijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:7eff192ecd8587705a6b3682b30fc22c4ac461e74bf9fda515b5f900bad92c3a", + "evidence_path": "evidence/20260721T144451Z-d518cc58/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2c51d0e36911b8c5f6ace9e218fcdd152af01a61d58ff48b905f94895ba6b4df", + "evidence_path": "evidence/20260721T144451Z-d518cc58/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3dd9386505bc11b0dc51fbe3f1939e5f070390b33a4a125c2e7315a509b70c09", + "evidence_path": "evidence/20260721T144451Z-d518cc58/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:e3ac3aa74c113794736d3ea96b9a97ed07bf9dc739d09baea45273f28e60aa04", + "evidence_path": "evidence/20260721T144451Z-d518cc58/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f1e6a86f04026b3d971a8ef6148eeb8b026d027cb94c56f4e7fcde33f6c6b07", + "evidence_path": "evidence/20260721T144451Z-d518cc58/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1a32e1f1584a8bf944a9cd0fb218fc56f58e4413757480238d2b364914c454e9", + "evidence_path": "evidence/20260721T144451Z-d518cc58/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:29dd035092ba7e9107f825be2868abe907455dc32450b6bbc992dc3670ad4644", + "evidence_path": "evidence/20260721T144451Z-d518cc58/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8670aa878e81d1a61d77e6d07aeca67abc0cb90043c1ee63a87d25319d89b015", + "evidence_path": "evidence/20260721T144451Z-d518cc58/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e8ca1af7cd8fad4adfa5083ad5a5134ddcc8678361dcc81fdec1fcf7ddcad677", + "evidence_path": "evidence/20260721T144451Z-d518cc58/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:27b9b56e28e8232df8150083c698fae6470f88f524c7fa5b44a78fd8f1f8afcd", + "evidence_path": "evidence/20260721T144451Z-d518cc58/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c59b1f3eea21199e913c3824de7c225694cbe916bc6c299793a63249d3e30c8a", + "evidence_path": "evidence/20260721T144451Z-d518cc58/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ca6dc9f2cd75142bf63efc97a856a7d063a39d4bfe72c2cbfbf46e1c7171f547", + "evidence_path": "evidence/20260721T144451Z-d518cc58/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:19aaf964c5c228294748ce518c2a1189c823239f8a2963a5df3486918c4c81f7", + "evidence_path": "evidence/20260721T144451Z-d518cc58/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:db34a07caf2c10724aa59905a9d458eba4fe9e2ce431c60cf70ef54da038d0d7", + "evidence_path": "evidence/20260721T144451Z-d518cc58/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3cf9f5bd8bf4a3d1f27ec5f967051184910c6ac0860255c879aa825018d68679", + "evidence_path": "evidence/20260721T144451Z-d518cc58/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5b005759068f55886c5924b91c296c2c3e822e6a9e52c50438582e2dfefe55d2", + "evidence_path": "evidence/20260721T144451Z-d518cc58/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e850724754aef32c1646232d60f7250cd91758e1858abadcf3deb8c79a4df675", + "evidence_path": "evidence/20260721T144451Z-d518cc58/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:73d3e56742c9ba3e6baa9802128e4bb57fac66f83a899fe267df81841b381f70", + "evidence_path": "evidence/20260721T144451Z-d518cc58/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:679a3ea614fa907efaa93cdc13c75df1b3d6f402d2b2c2d3f4e6e11e9cae70be", + "evidence_path": "evidence/20260721T144451Z-d518cc58/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5820c9213535f433369263a33a443555b6a42b69d7747e09989ec66a3132d2b5", + "evidence_path": "evidence/20260721T144451Z-d518cc58/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:590f455e719beb2b674e1926cc2cbeaa2303998e3de16acd884f90a17f380181", + "evidence_path": "evidence/20260721T144451Z-d518cc58/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:63d65493571dac08fdefd1b251927a21564aa29c9c8f13b6af1814c949371456", + "evidence_path": "evidence/20260721T144451Z-d518cc58/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b37712a657bfee393776a4c0c598e5ea2f49eafea3434036ba51f207d047d11a", + "evidence_path": "evidence/20260721T144451Z-d518cc58/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f8329cd07449ba140423d4cd484b41a443752de898c3dc10a316dd24f75b9abf", + "evidence_path": "evidence/20260721T144451Z-d518cc58/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ac871d5b4a69e0e6bf05f95224664513a3704435d014e6c63f6050982245e221", + "evidence_path": "evidence/20260721T144451Z-d518cc58/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c674b0475f91ff373af54c1b2660fdccf48aa510aeb92ab6b420644c4f0aac64", + "evidence_path": "evidence/20260721T144451Z-d518cc58/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:61f53d238c1089152245c8df6a97afd57ed365d0fa54f9121cba6951601f3d17", + "evidence_path": "evidence/20260721T144451Z-d518cc58/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a99757773ba07c885ff00055f540e0b6315a44a0bd221fc0e3720a9677589a7b", + "evidence_path": "evidence/20260721T144451Z-d518cc58/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fb6d85e37ac0e7657387f98143ea4038adec4209934e4483506f61353b29f363", + "evidence_path": "evidence/20260721T144451Z-d518cc58/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d0a904e60bfdacd51138a20ecd4fcb7deb3f885a3b0a63d07f3ddd713b99180c", + "evidence_path": "evidence/20260721T144451Z-d518cc58/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ad85c4c9d2f7b2ce09e8cf354f50415478e7982dbb0ec44ce2caf1f9e1e71dcb", + "evidence_path": "evidence/20260721T144451Z-d518cc58/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f6a14d89678876bbb5a38abc3cd7be2b44a344b842b590a48f8a58a5e37aa8f7", + "evidence_path": "evidence/20260721T144451Z-d518cc58/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d6ff860ab298568dfc8e55091f36f5329207837464874a44d0642e1b9256af18", + "evidence_path": "evidence/20260721T144451Z-d518cc58/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:012b9fef98f54bc2a2a662a23039535427abee212318bc79b3bdad3177b3bad3", + "evidence_path": "evidence/20260721T144451Z-d518cc58/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ff0688373fc4192d306a5d87ed51936f41db6ff227ad8bbfe08b21b5451f170a", + "evidence_path": "evidence/20260721T144451Z-d518cc58/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:437c478e53935d9c6ff006f1baaa22d1a2cdf6f516f076a9fd5492d47b199fe3", + "evidence_path": "evidence/20260721T144451Z-d518cc58/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f1aa5051e7a9eb4c170ade2b9f191f0c9f079240d1df154f66dbb74ccbe8f7a3", + "evidence_path": "evidence/20260721T144451Z-d518cc58/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:051ffc6185207f57b350fdf99b6ac140991d7554c37a7dfc97d06142a0c38813", + "evidence_path": "evidence/20260721T144451Z-d518cc58/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6a27ee4ae28727774bd61e11fd63ff5030096b9b174946ec3a3dffbaeb684232", + "evidence_path": "evidence/20260721T144451Z-d518cc58/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 40, + "failed": 10, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run5.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run5.json new file mode 100644 index 0000000..fbb99f1 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-run5.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144506Z-2b8cb55a", + "timestamp": "2026-07-21T14:45:06Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:965b923ba3dbdeca3cc048cfcfce1dab04aab00bfd7ccc305d6eef4172377e44", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:33a7ef40241999af9305398bd0860171b01a0708e69c8b553015dae037938c53", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:7196aae52e92c9e5fbcd8258d780712bb64ffcff9a21b7ddb04edaebcf76abbd", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:d1f8f22c1814036cec63b2b18599f55827d0903819be05c0b966841949fd6c74", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:00c542d8f599f798f2c2dec0a99cf2bc4c6970e55b0f4521c248f3e03e9e9660", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:f30d1b85ce07fe760e5740ba745e16e5cd83e5cba031fc17c2c9bc8190c6c95d", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:66901b40837c84eea8345c4a4491393e50bdc6d13cb050debe476732e6a6926e", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b05fdefbd75780e20c04ce923ee35bf3792e5dd885268a2b51640b34587b8836", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2c513bca92367c73e142ff22337ef6cdde6d3e187b499d3eb9685961011f691c", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3caf66e8a1f1de01d36ccf6fbff266206eee5c8cc538aa1c4866269f30ea7fb4", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:74f73886e4c70a9bf260a1007f949531d8432bbf5f4c18a63223e0a10b612c56", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefgijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:e5e2531763ae0bd94b808de155ccf940b9aac0c788c7d82588490be21a0232df", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bd084a019199723d34ce8b4b4db6696a651cbbc4a78de7098b47888f20274b1e", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fcb2b8bc12c61ec5addede82210f4ecfef658a2e091bc51cd066a5dfb8e7c9bb", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:e4a7ee2ae0700ce932ada1d66df0ceecfccd29e5d77653201fe74a9c8645f6d8", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:818f0f45c0ea4a952837aae5bcada977affc8ef8fcef1620657779a6218222d5", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0e54649ae1c6f88cc50f44a342d4312a253ccdbd6c8bbabe01a2d72bc1aec3a6", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c8d3b9e22c9d35f52950e5317a9052c5bc37b8a859ae3606b4a7c5ab850d8ecb", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6c57c4c96c4bcc4c543a62e09bc781294921d5a37cc00c352c20a6a97954ff13", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb4ea3020104b3b6ae7dc1b86d0222cbb52d1a54e50e4b72f8cbeda0dc0cb6d8", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:5dc88b22d5953b9420c9a1c6830de955a1f9378c7faa87f381afcc6ee40f7645", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8824e59ca34c60f69ecc5139b0e229e75d7e16b4ce3855c5947b46fcacfcdbfd", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2beb479dbde39f092a7a3ee05448446b8ea17556b6ef52334aa6a1043aeceb74", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4c96ec33545b31ec5be104227e9d943eb7525c8be62f784d8e0fa3e368524e5e", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:405568ffd7523379bb2ab57539fc08e3405a98f70a5553c0db7d0ce67c9372d5", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8f58d9d78d12f874ff54d6f1797c426b5c4941577017a6719c05b25f5648c7aa", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4a6de494a96df1dd03cac1c8256458cbbd8dc830a360cf76dda63d58fbe46c6d", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a7cb74781c64d09ccac031465d92f795239dbcdf202403a5f2439adf946a2987", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:86ccbbd644cc93153365a50f082b1fc52a480c77310972c15b2fb2dbc133a981", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:41e5f417ddbe288c67a44e74b71f187c2f651d91158d819d1074b932bf6f8b7e", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f8d128ac244044487d76d0b64340a1bf2e02cbd42553dce994e69571897b150b", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8068e1dab0f5e7888759e3f6919231e3e51c79d3953c52771d509fefc17eabc8", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fdb22f2bce1c91974e785cb9f96ad809acbb22ec644023b52a76843c464e7157", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8a2011fda22c5ea4a3f433a91c9209321481a40500c395dfd504e80e5bfe6fc2", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e4e931e43ac5ef35b0997a5cda38f1c67866dcb3e271f5c9f85a5e7a567b858", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:59345a2248f2b65f150b241d096709036ed625849f1b5ae21b881157269422eb", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cb30f63a3b667ba2ac4aa8667b8fde6f76c04dae18fa63ff67907a26df575c82", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d02b087549c98ee65aa687d71e176de2815a5c9d9e3e9d077c6dc0fb84be34a2", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de48c598f6e3fb0542908c519b9c77d59b01195d6e864673756cfe7fa532060f", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7b3562e6b98ca6bd1349f7e13afc86e997394fc84b46ecb5f5c6435cf3f93f80", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:568edb81f6a1e6aa442aaa813b9b2dcaa01b6fcce74ae2e754726a930e920e55", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dbf8445b9371d24bad11f4f1daf0d65edb28defb186dd0d92604c50f39fc771e", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ca3124ed827a2e6d3d7eeee5567b79542ed6801a87bcc4304dfce99251b4b25a", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8ec09b94704767c4d7200e9c9d13ca9cb08a5f68dc1101b5b69484c920adbe2a", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9b4e268a4f780cbd9ec8704fafe3b9e9242bcd5028aa5b26279da36be0217603", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bc3d79d546e028d709f2bac78ea9dae33dbb55292f6f400102fa93f34d8dfc5a", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ba8c33f23b3b75f905d536fea4dcd6065add57f9b3367f33f3727e16cf65ff69", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8c94b9b9555d95256cc4901190e87e8bfaae8b2127ee91f98e5ebec453093b77", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8120335af5ef9d36d56e2b5148b312522b7941efb818670c795de8643a5af14b", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a4c67b5b0d9218f8b4d8534b8bf6a2c62c633b089c60a20fddce2608affb637f", + "evidence_path": "evidence/20260721T144506Z-2b8cb55a/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 40, + "failed": 10, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run1.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run1.json new file mode 100644 index 0000000..57389da --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run1.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144230Z-2ea44aca", + "timestamp": "2026-07-21T14:42:30Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:83a33cab2b58282b24f0e4ceed551ab6f19f867b8b3cbe8426fc07ba54c15264", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:31250773595e79635dcc5ede2d55b9fffa4332d5ad4aaf694f5ee406337e6822", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:c762d220bedbdfd28da68f8811df1a680fbcbfdf4090423999166ee5673bb649", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:a9fd8298f66bc0c6152eba850f40f861b47f5c0ee65e0712debeb729890f2e2b", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:f1a52c995740d734f113936d3668e2006fe8024453ca9575b4cf454cc5515020", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:f38b9ac5405f0390e62e19402c235a1f30cabb7edf9e754b6f4298c676db6adc", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'archive_account'", + "evidence_hash": "sha256:2965326ab7f093921ef73f57d5d3607b8fe04825d337f6e299499dad36ed18dd", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7ffc4922ca23ec14f6d527b875548691e8e159062be6d1b63c323acb1d0c1773", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0e3db6a7ec5603dd7fb24831523eeee147ba597ecef98b8562ec4c293c16ac20", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dd721ba63694d0553266a80536094e599590d40f1d78cda97fda02b491d3fc72", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b113e998805d0cd9c75c7986e086c3d14c25749989318ed1b115c307bca18744", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5e720d2a285c331658f7ed61a6337a531651591f2cdc440bd70ba69e800f3f08", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:23c69cfbdc88e3beb8f3ff7c3fb37f6ee1621f4759e3a83eb703f166978cf8d6", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4c2ddafe60cf2e04ef9b5fb8fc8a28a9dea190e810c0556b26800eb0015a16ec", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6db3e956ee5b46539eb588d92ef3a62d8103d418ef70476d3d4aaf057c184b0c", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d30d3ac912beee7cf9bb7c79b81a1d433eab3feaff974985e29b7e4543f26b5e", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4b00eb89bee13c0705667725fa66ccb236eaeaf2b0997ae819d3be8c3313d001", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:47d2a5a1453db11db6d6a77087d1f2ab65c8d0f1bf68a32a39c95534d1497c05", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:40f56b6708a88fda03657af1b78a8fab2971fb9cdabba700ecc2570050913250", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:203956d7c050a5a8ddb8ee778ae3290dcb8598b8d0ac2b7a2be9f70b9f3aac15", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0c7bbf84753f1400eb55e5984047ae66c7cac4da861d7a14fef4dbd767eac9d0", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d943d1802cda9cb3fa0ba8cc20987db9441600142b959b81b7f3061c53d99fcd", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:000b700470412ddbbce734652f27a170802836769709420493602f3b69943e9d", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4c5bfc47fd2b75d2c04623613005742bde2369d800b7aa4b6851a7a01a3ef7c7", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:51588cb8d6ef2e78287d11916729c46a86cb8750d0475c0dece5ac79010bdcbb", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:135237deacbbf77fb8ac3c5f2c23dc75787dbeb6744dd11639fff5a72a4cadbf", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:700a179e235c2372c895cb0467293edfbd1989fc41ed6f326efbc1e535d9e940", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:618a26bde6f941e57a1c6634eabce108b9c052cda5fb9162740913e4f4f925e1", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb1077c4c29214a772f8f0b9a83c3d50fc7a04dc64330f079b73d9ae72cb7fd3", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d72625df18319f81ba3e0a6448d02d5d020c1bb0c774b46127d9c38efdfa843d", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:40a6d357ac53921d26beef151fe1c4aee47f9d6208f0b64e93be0bd62c3f141d", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b6e3e7d0ac0d7c8ae22b2abb3981c8495e94fb1ac6e13056a8cbdf40e787ce22", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4ac9d95db9eb435f9f2f04916cb73cbc55b6eb049a27d1b044eb309a57fc8f37", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:67259074763b2a4bb557922076217ca299583010d16289b3bf55fbfed950fdb3", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:33eed8c0680ed2199750af1089f4f6cfbcedaca394350321351b2bd364167754", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d5190b0d1c94298b4d5e942e9f2ca93f898f4637792d8b85efcf037709c4def3", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:082af80694b04d0340959cb5af1cc02595f2107cc044b2bc0723358bdccb1e67", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6a2f034b95ba24701be2b499131fdf4ce40b481985aa34ebe4c5ddf7e13d5457", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:759a506aa9701e265957c39aafae4a9eb3310d4567916ce254bfb4e19eb60c78", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1c99c47d33e8a96769deb49be7aec688a1a7830c7f57048886b6759c3a39326d", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c279c52a2e0e9923e3e245bdc82277ab9a0c8d414527679a88d2f64b8ce6ae28", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:055a668074a8a6b8000b801efd38364f7f496d99d43a5704ff46f39451de4762", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb2d79938e5ab65aff63e52f59534a71ecfa1e099d13e1b5663241d9ebe146c3", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d290789b41cdcb5a85bcd9813af38a55467060629dcb8e142a234dfaa68904ed", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:d028e9cc204469c2689457b158d29f1538b277dd2ef341e10271623892dc0205", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ed8db1354c3c34b26ba2ed75760aea770cd4c4b013c6044d7ab8f289d7dd0fa5", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:216d84dca814f56ebf7224fe6e7ee047b3867043dcbfd3f86074c96504b8f9d0", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:389fe9bc66153cb0c1416a0087d39a7736c3d7e37678eb88307d3e4f96451de8", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a712704ccf38beae1b66b3703a04f064242d7870122bad5bc9e2787be31dd9f6", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e0de8682bf2bcb7f2a44123a06566273a9aba1fcd28881545b29c5037232e005", + "evidence_path": "evidence/20260721T144230Z-2ea44aca/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 42, + "failed": 8, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run2.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run2.json new file mode 100644 index 0000000..f6a6ef1 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run2.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144245Z-37628bf3", + "timestamp": "2026-07-21T14:42:45Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:22045ead440627497ffddf5e54fab3417327ac5965790dcad90c8709c5ac56d1", + "evidence_path": "evidence/20260721T144245Z-37628bf3/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:38231067e81befff04d8da38197e1c7339699f04fa5ca5e7781fe391075cdcee", + "evidence_path": "evidence/20260721T144245Z-37628bf3/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:c4b593171dcd31ec4c0245fe13490982f30e1519e30cd5413c2758be0679c47e", + "evidence_path": "evidence/20260721T144245Z-37628bf3/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:29ebcbe61a7ffe054bc03ffabcdc12685eef8c131170330382e7e451503a3e8f", + "evidence_path": "evidence/20260721T144245Z-37628bf3/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:2b764ae4f2f40ea4221e8259f2493baa40edfd067e16d2d1a9d7bc87934fff82", + "evidence_path": "evidence/20260721T144245Z-37628bf3/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:eb7ba765a5bfce8fa0b5370a61e54e5c017803baaba41d3d09f94c7c5396220c", + "evidence_path": "evidence/20260721T144245Z-37628bf3/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'archive_account'", + "evidence_hash": "sha256:cec3b33d022624f1e61d05ef9ec67ba8935135f850b022c7483a06422e190c28", + "evidence_path": "evidence/20260721T144245Z-37628bf3/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ca0cc1fb1f4c582c2058042c243fb5db7f103cb85e121c1bc8eb9d09ed0003b0", + "evidence_path": "evidence/20260721T144245Z-37628bf3/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:26bdaee1006fe3e41550c49a95bc8243a32c4a561e7057f58c9c3c5c35b46f1f", + "evidence_path": "evidence/20260721T144245Z-37628bf3/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:096486beba333360f62d2f70eaf1b66857f71e86c8db83c59a9ab632ffe53377", + "evidence_path": "evidence/20260721T144245Z-37628bf3/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e4649a5ecc0edc76e79fbcb7b49c7e13826f5061568cd0b2c3de6cbc70d573e2", + "evidence_path": "evidence/20260721T144245Z-37628bf3/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:72a8af8f284cb271980855002d1b2c7deccbbd58a6e51789e82afd0f61d77773", + "evidence_path": "evidence/20260721T144245Z-37628bf3/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7085fece7d8e4a2a8b3721b0c2b5eabddcac8c6aed992dd616ef99bb0da5bb94", + "evidence_path": "evidence/20260721T144245Z-37628bf3/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d627d3e50179af873cf5f17b6042392abf3b8d5bb33ed801cd3ece97e99b43d3", + "evidence_path": "evidence/20260721T144245Z-37628bf3/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5ec5ff98ccd004f8907e7dde71010723769f060be4eb4ad6154b5937a27454c9", + "evidence_path": "evidence/20260721T144245Z-37628bf3/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:715c0c7cafacda0221f05dc5fd4dcfe8c429d368272827c6a3fb8fb1ab1dc1e1", + "evidence_path": "evidence/20260721T144245Z-37628bf3/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7beacfffeec8ff6b00a325fa393adeabdffea95b1f48e4be99089909f96f3409", + "evidence_path": "evidence/20260721T144245Z-37628bf3/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:942a0d19ca5c123a82c10ca6883b284028444ef58e3534e0806100154c6ef788", + "evidence_path": "evidence/20260721T144245Z-37628bf3/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:109d48620069d0e94286371266140f29a74d81de7a6d841e16fc0c5a3742cd29", + "evidence_path": "evidence/20260721T144245Z-37628bf3/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:972486c965b87a0b42fa187dd7c2ab01fac1cbb1db08566dfdff6b4ca74c4b75", + "evidence_path": "evidence/20260721T144245Z-37628bf3/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2ac13dffc4d0d05da65d61d318fb6d13a9bf60c6b0617ed5c38c234d57d7c097", + "evidence_path": "evidence/20260721T144245Z-37628bf3/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f5ea1cb9bfd444bf84ef9bc131c372ebaa735b55686545f52cda7d26afab9337", + "evidence_path": "evidence/20260721T144245Z-37628bf3/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3547eeaa5e8f88afb04c5cb28485253c39c330e07db7facbd3e56dcaf27aa01d", + "evidence_path": "evidence/20260721T144245Z-37628bf3/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e0c923769f7f8d160d76ea8b46d752dfa7672f7482e11cc32d462e73cf9957c4", + "evidence_path": "evidence/20260721T144245Z-37628bf3/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:84d0b1397f2520bda9f4b68f2bfd5fcd83e17f179947b1d52d007c1976ba983d", + "evidence_path": "evidence/20260721T144245Z-37628bf3/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:894448ce077d403e7c87635edd856076b2cf9a8bc3aef74116c09e127e3a9f6a", + "evidence_path": "evidence/20260721T144245Z-37628bf3/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4419832cc4c99fa8c629251cb93e4e56997d5e53f43aa60cf7c23e9d4d4dad66", + "evidence_path": "evidence/20260721T144245Z-37628bf3/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aa53a94abb1172cc05912e1d6487284489e627c0f7d340c88fc3e89c31726051", + "evidence_path": "evidence/20260721T144245Z-37628bf3/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:02609312767f121dc8a1d88f9f3642616b0899cb7f93f1aa614c8f35fa2d17d1", + "evidence_path": "evidence/20260721T144245Z-37628bf3/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7d22ea5ebfc205f8f2d64659400ccd55f6d696f5408c591d1c9ecf53c4369066", + "evidence_path": "evidence/20260721T144245Z-37628bf3/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c0cae343b9179088024e97b4cac5c6a465f633e57386a86572ba4dcf8b4812d5", + "evidence_path": "evidence/20260721T144245Z-37628bf3/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ea1a10cf6399a2c011534d286ae92ebc7c38f8a100a1e848b266ca89b86c73e5", + "evidence_path": "evidence/20260721T144245Z-37628bf3/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d8505e7be53d11604b22f4f466f8f5bbfdcd4c63c80dc35c49e54450277a1495", + "evidence_path": "evidence/20260721T144245Z-37628bf3/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ec89cff3b82245e69b167febf40bb42a982312d62c1d0879fe449cf12c5cd468", + "evidence_path": "evidence/20260721T144245Z-37628bf3/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c84c91807389a16791c12fed6043b46f58937b7f36e8c4c72fc42a158bf95308", + "evidence_path": "evidence/20260721T144245Z-37628bf3/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:73ac7752572e7a786fba10af1ad9fcc037779872d43b57e643654dee367a48aa", + "evidence_path": "evidence/20260721T144245Z-37628bf3/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1429680e07eeb4a2d24c1a592e49f708304aea037cffef62862bc2e65918c914", + "evidence_path": "evidence/20260721T144245Z-37628bf3/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5f7df05332d864222321364f0d1ad8e4381fb4cb4698452dc3f6b2d65cf2c4c2", + "evidence_path": "evidence/20260721T144245Z-37628bf3/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9e9e44f9ac8f330eb6c1ac69658fbba9571c9c5a146489b14fb4103ead725036", + "evidence_path": "evidence/20260721T144245Z-37628bf3/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:814631e146f0ca30b7c575bd6e7a9ea351f36f02233b4ba090939cd12148cbda", + "evidence_path": "evidence/20260721T144245Z-37628bf3/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f299686421863b155280dd7b1aa912d5501c5eb0fcc762b5d5b598dad1d94007", + "evidence_path": "evidence/20260721T144245Z-37628bf3/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e68d345d12344e67a517c638fa62c19ed83b60c2fb28ba6400a4622062de4e7a", + "evidence_path": "evidence/20260721T144245Z-37628bf3/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cff5de6b7a99d0d045226ab44c76bbf58b2f487c1a51cb1fe35f196e1386d2cd", + "evidence_path": "evidence/20260721T144245Z-37628bf3/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f03d3ce3b181194e9ee1805724baacc15021b627dacb373294587c089fb277c4", + "evidence_path": "evidence/20260721T144245Z-37628bf3/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:239eaca27eba1f3ae612ba9a58eb2224f5bd1a3ab498ded560a657e7fdf66d93", + "evidence_path": "evidence/20260721T144245Z-37628bf3/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:12d34beb0aa0c54771f750ad04c63a38a7af3ac999859e2e7cc648db0830b2e6", + "evidence_path": "evidence/20260721T144245Z-37628bf3/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e70adad7ef59edf7d5440659f27a024a2cac26149b707b3812919260465b89fc", + "evidence_path": "evidence/20260721T144245Z-37628bf3/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f7b2b2abae2ac725711ffe62c34ee98d8f0fb53fb39acac2b5ad5de6e9ceb23d", + "evidence_path": "evidence/20260721T144245Z-37628bf3/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:58ad0ff1100d8cde76a7cccb5b2db632efb5c33dea9ccea077cb7b80769888f2", + "evidence_path": "evidence/20260721T144245Z-37628bf3/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f94388d17b2363a9db1a3f311c190253ce35516140204c18d1ddcd91fb0e62a1", + "evidence_path": "evidence/20260721T144245Z-37628bf3/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 42, + "failed": 8, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run3.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run3.json new file mode 100644 index 0000000..95e4e37 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run3.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144259Z-0c77a05a", + "timestamp": "2026-07-21T14:42:59Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:a03a900aa1fde3936caf245aed87b7cd25b5fd158c8977d34a56e62bfdff7c78", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:92701d4a43c6b6687fa26b3fbcab6c119aeb74ea5553888768f5183ff162dd43", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:7550105fab34962426aedecb89cc92987049d955567ad3a826294276f3549ca2", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:e6ed1a4135c4805f0221849787731312b6264a320b6247091bf22eb8afcdf7de", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:5fa949a7ced7fd777ad226bf0469bffe2893a9432f8ee7102f7695d3fae77d0f", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:b3f94b59f61b1cda79f502453aabe05e519788b22354815f40bbf2320e3d7373", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'archive_account'", + "evidence_hash": "sha256:18ebed38db42b726c578957e7feeb2b1c8ca574108824272ddf5ee9965bc4ee5", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:574df3bd622cd2f641f35afe08229654b32aa5e4798dbb26603a0e5ceefcec19", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:85e08b38bc9f57411f242baabb1c845b18786425202d7c1e9ddd69f7ea2ad92e", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cdec66937557f0c00b8386ae97faaf55427b56ddc5e6e37cacd16899b0c96366", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1cc9b1aa27f83827153ebaf7e3295c9600fc683e112f57d7eec5778adef7494b", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:224129c787ea0786b2cb46eea78a0c56e26c5fb4649d5731a1caad1b33389b7f", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:36465fa959914cbb647d37d40174e25e6dabaf315accd4d6ada2b34bb4a501f5", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c0e65399f657264429339333fa9325095213db4d8710bc02aeb230ec4b070fa8", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9507e0de462406c4b2048200ef8bab45c0f92fc838159668d66e4a67b31a9340", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:41d89bf6c475a9b826975923ce58c37b61fdc1fd5acf3a18eeddaa88b1f1f363", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:25d72a03cddba3f7c54cfa8e151de4c60c2fa9eac30c07cf2d928c711d5f6750", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:03be82955e382b5d7772d62ac03054a60b60f47ad52d150e8a9651aa846f15e7", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4dc4585031d1b6ca05196c3a3f6b352aa0cea48ad5e625bfe40a640207024a5c", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:40b5bca511076c4c140f4a175dcc81b33d82f9d219be85e245bb2ef9b63914b0", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d7e1eb66b96132a374d8f6a46a617caf4ec2d9cbf047fb98a8473d74dd519778", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ecde4ce2b1a08a696b790b04f723858b9531910b703d4a4d1785ded78be2c4e1", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:676f014aca40519c7c10a0a5f1eebd40d8419a3af4920eafbb3f7dbda06b14f6", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9d24ba0b84b24822efea5cb6cf62ee15b4dd07cf4212e4e74bd9aaa9b2205e2d", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c2b2c5487e56b047ac496290847bd28eb331f4d397fd0845dab98caa70d6b99d", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:69291fce32439934d0aa9eeafd61e0161a73855f54fb236ed46aea43560f78a5", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d054c38db45bc5ebdc8ea2c43bf20a6387d5e3671bf469bfd46a1e6cd8345a31", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2c096e83457be89e7979d0d8b3801f1e95f346d12f9a13cc03a0eb5beb236638", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:01130765960b9e852461a2863d7d4e7b8793ea63e117a7dfc8b4ad5b5efa4379", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6ae25b1573ec9ce0234a6c16f60510e426a39488a239cb35b5fd5246cb0864fd", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eac47ecef064d140f8a586a7f01851bd0e40c7ef0bbaf24193f64a67bdd3b6c0", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7d0159a36631ed682cf6cfe48c087dd5074c51594d7a4c324b232c3684b8fde1", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f8557e6a817c8343c633a4002e7d1cffece53dcd9380770af6cf48570dc06d0", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8747ca4218571ee7d6ef7ea2b008d3c0f580f242523c571d710d1a28c11421fa", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:86d316d371c7e61d5473f966ceb03a5c44f9d9d8a347a50c83bb544290c4c700", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a2564c0c8569e5164e345f9492cf22c9298338bc83e270d3fd8c170c66465b50", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3ba84d145b90d4fb7fa88c1b427ccd875f78ca81f34910d37df20217c97d1460", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6133ae6eb1e39bc62447b5f285fc1c366faa4cc3f583ea42afa07e632df5816", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a8bf3a169542983e90c1921e20966447ecee61fac6cd099dfc8603e59dfe9400", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0d53c411a873509cc73318a1aa6c6f6669243951ddafc1c5744fe384fa6648c7", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5399bf59032208e1180c1f252b36864f844f0ec7af7697944b1609c72e681147", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b8be4a4494907fd096c30db930b844b2e102162a2d4f12d3ed83a00a45aee6e3", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de53905d98b4d37b13638753285dc33d9783002a6ab6704df95055a3f252340d", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7c1683f737a728e5a6b1a4a1ca2705c7fb6f7b9107623d4eea4d5392fd578cd7", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:9ce73abbb94eff7ae6f2ef7439e65cadfba5383ab083133d9b44f9aec1c74445", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:53aab624202f6d2f51068ee803f56d984b21e616f1d1b647740ce5875aadedbb", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:63a5b31577f0f1ba61d430919c58d475fc90064601786dd405087dd1d1ddb24d", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0b0528c50dd4f3c625a0ab6f768500f8aa251fa76d9cf0ca2db27d015dd63c59", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ee41a0c7e6123a3bb7765f44b20371545ba1a8cbfe3063086c37607f73a32e25", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6793c3d35631c9b6bb7ca4c22862191fd7a4084751285843576895ab35612af5", + "evidence_path": "evidence/20260721T144259Z-0c77a05a/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 42, + "failed": 8, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run4.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run4.json new file mode 100644 index 0000000..80c2797 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run4.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144313Z-89e3a681", + "timestamp": "2026-07-21T14:43:13Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:5b10e9006ecc12cb935f65cceef5cd8620b85b6211761bdcb46cf1ace0189d1c", + "evidence_path": "evidence/20260721T144313Z-89e3a681/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:3da787a5bd333ab8e3b7df80df275e3ca2e4db2794a39f1c8f45c0807a826e77", + "evidence_path": "evidence/20260721T144313Z-89e3a681/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:669684354f031c8d838b406e6fcccd370468b70e24d1e3e7dfaad68e809f7804", + "evidence_path": "evidence/20260721T144313Z-89e3a681/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:bcdcfc9bed91c7e94e97ff7dcfec8e8d98fd612dab0fb17b3ef78ad8a82c39bd", + "evidence_path": "evidence/20260721T144313Z-89e3a681/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:21c1b7b1b353ded9b682ed74ecfac14ff728c997681a0c58a23a4952ffdfda8b", + "evidence_path": "evidence/20260721T144313Z-89e3a681/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:7c3adf48bdc3401df9d01bb46faede5d788ee7b67e03f00224d5f0a4a4271c2b", + "evidence_path": "evidence/20260721T144313Z-89e3a681/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'archive_account'", + "evidence_hash": "sha256:8a0ca17b919d42596a8079668796899683c5c3f28b81a8c595a8c0cdb3b95532", + "evidence_path": "evidence/20260721T144313Z-89e3a681/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b2484bd0d8cae6edf0e39305ba94ca933a7d6041643f262bfb4500cbffd525f7", + "evidence_path": "evidence/20260721T144313Z-89e3a681/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fe6a51fcfdce813bdc9d7e7ae35c8c0bf4da160d7a0dc0959ae8a087d6e8e2e9", + "evidence_path": "evidence/20260721T144313Z-89e3a681/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:89eb4b0c367d2c65c09736d7b5384fcbedf36a5e54dbbb61289e4f6cfb19a0c6", + "evidence_path": "evidence/20260721T144313Z-89e3a681/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9064cc2dc256f7e22bb091fad2c1f8c3a304ec74a98b59dbb8d909da212dd9af", + "evidence_path": "evidence/20260721T144313Z-89e3a681/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8514e3d8b4ec58d6145d9ec046f9a03467b2311a887797f1da30191775f95ed8", + "evidence_path": "evidence/20260721T144313Z-89e3a681/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:080da018779598abc1a2062db5b88adcf2adb83bf30ef18d854f3231af45dfaf", + "evidence_path": "evidence/20260721T144313Z-89e3a681/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a63bba15ee23d4f2dc40001d80098e8e858f5e1e1d498df9e8564ef1d4bb63da", + "evidence_path": "evidence/20260721T144313Z-89e3a681/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:94b23d08972b32b7c2aca6761e63fa89b9ca4ca44a2141c0d4578b635dd9bb33", + "evidence_path": "evidence/20260721T144313Z-89e3a681/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0a72e170ed17a8418c3e3231c41756296d200ffe9960659806bf1272f619534e", + "evidence_path": "evidence/20260721T144313Z-89e3a681/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5d476583a61f13b603183993151e34f5661806cc5c191fdbbd30c984783a6fd6", + "evidence_path": "evidence/20260721T144313Z-89e3a681/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b5742a52ca866796f2a6b5a09949036ec4ed896cc87eb23ec62308181bca6c2e", + "evidence_path": "evidence/20260721T144313Z-89e3a681/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2473363a595d87b381ddfdaaff49e2d26f23a867dd59369c13b8bc5727d52033", + "evidence_path": "evidence/20260721T144313Z-89e3a681/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4a5911637f31a4e1520aa750d900b4ba84e5cfdf0b197fc414470ab1f29d77c0", + "evidence_path": "evidence/20260721T144313Z-89e3a681/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a2418379a57b9b9dd61cfb51a27caa82136c3257dcff9d7dd9fd94a2790e9843", + "evidence_path": "evidence/20260721T144313Z-89e3a681/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:17f08fab6def62e946042da9d5dd37f7693bb327ade7f3d37fb218d78d025c98", + "evidence_path": "evidence/20260721T144313Z-89e3a681/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ebc56f0df2dce32400ec5b020255021e5e6d9a3b0b46703c05a3234daaea12ee", + "evidence_path": "evidence/20260721T144313Z-89e3a681/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6413d9d6f32a13b513a9689bd7799b2cba42615c0f3fd5632efe009bb3affda0", + "evidence_path": "evidence/20260721T144313Z-89e3a681/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3137863f69e6360cfd3000b29cb221d640fc34fbff0b5fa707462c556df561ae", + "evidence_path": "evidence/20260721T144313Z-89e3a681/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6656ca7e579f8ea327d26e4a2d2125d823160fe3c6f15af39d5d69b374c1328f", + "evidence_path": "evidence/20260721T144313Z-89e3a681/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bdc08e893d1e9bbe46c0355785c2fe0ff2bdb8ddb52a824102a7d88434170b4a", + "evidence_path": "evidence/20260721T144313Z-89e3a681/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1dc0f3d4eb08f6cbc841a16f2acea72c4a04fefd2a8f6a3ed8311de6187ddf7f", + "evidence_path": "evidence/20260721T144313Z-89e3a681/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f70dbcfb4b1e4b26397991bc5e1f0cac26b6114a4e17a50360bc59265d9b732e", + "evidence_path": "evidence/20260721T144313Z-89e3a681/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5ad7271d9e6dce86c198986a750f117fd6f800f88889a30094be69d184af5582", + "evidence_path": "evidence/20260721T144313Z-89e3a681/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e7a706a45ee7a553fd2fc586958b7fa3ee19f85af616a68dfd42d8c4d850495a", + "evidence_path": "evidence/20260721T144313Z-89e3a681/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:056ca72f51270a512755d1e801b308f7896f12329b0122f0dd1a3cf94493b38c", + "evidence_path": "evidence/20260721T144313Z-89e3a681/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:872f05b25cc6b610942d4457eb3ad36b9e4993a8b895a8e979eca5d1649bea8a", + "evidence_path": "evidence/20260721T144313Z-89e3a681/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6610e6adcea4271cd276cbd4582211f8a3d36c51ab4e99dff0d1ab023bae9df5", + "evidence_path": "evidence/20260721T144313Z-89e3a681/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7cb4804e26a8011c0b19176e57d14a09dcd59eb51d8fb1fe227d1ab09c82f514", + "evidence_path": "evidence/20260721T144313Z-89e3a681/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:636bb84832a0d656a8ea281ef6bad043dddd86921885a4ed2f5763fe35d61374", + "evidence_path": "evidence/20260721T144313Z-89e3a681/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a63adbcb083e6d587192e231cdd69e637281735069b46fe47f63f4125fc53208", + "evidence_path": "evidence/20260721T144313Z-89e3a681/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fc3d812b72ae7bb36540dc30c0329d0f1b2cb17fdfc634508c57fcba241cc2c0", + "evidence_path": "evidence/20260721T144313Z-89e3a681/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b1f40d3776d8493e8bcdfb07ff56f0d6a91d635a841a4714db81a19695747890", + "evidence_path": "evidence/20260721T144313Z-89e3a681/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2aff63ac3abefc532c1eb872adeda1d88adc8525c922fe4447d4778da2d1a346", + "evidence_path": "evidence/20260721T144313Z-89e3a681/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3c17c98796a66483551c0f30f1b01ec8f9108d3732c4eff83968c54a541be562", + "evidence_path": "evidence/20260721T144313Z-89e3a681/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:85c57516ff83bfc5765ddcb5a0163ace3035f14e3a4ad008e4e483ee2b07ba8f", + "evidence_path": "evidence/20260721T144313Z-89e3a681/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d9db3216c1e2621f7d256fa5e8da551defcc3d4c9f814a4520a4d98460a94690", + "evidence_path": "evidence/20260721T144313Z-89e3a681/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:78b7f7655255b912d6f1a353a36fa936824c843cdd4fcc4d68d0e2bb36cf3f03", + "evidence_path": "evidence/20260721T144313Z-89e3a681/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:d7444d09ea9a3cdd571f1309863fc9c8353640c66dcf5a3551667de04281014b", + "evidence_path": "evidence/20260721T144313Z-89e3a681/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:690c43b57c79adf46ce6051c9ae1e298088deb52319c28cae0dd50e24c0c77e7", + "evidence_path": "evidence/20260721T144313Z-89e3a681/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8dd632e5b96a4a6c3447605ff87e97770e7953a2aa4409d4ecad3b6500ccd13f", + "evidence_path": "evidence/20260721T144313Z-89e3a681/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:819765ed5c09e5442f85bf949662e521fca20989aac0da763c3239cc1baf2003", + "evidence_path": "evidence/20260721T144313Z-89e3a681/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6bc3a7cef74072e7bcd952a04f85699f785ff45302781b2d639f8861bce9c9fa", + "evidence_path": "evidence/20260721T144313Z-89e3a681/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d8014c800e0d761422478ff06de00d6d79f8c8db375c3ba4b02cae15676fb4a5", + "evidence_path": "evidence/20260721T144313Z-89e3a681/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 42, + "failed": 8, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run5.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run5.json new file mode 100644 index 0000000..cb9edd2 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-run5.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144326Z-35a2ee51", + "timestamp": "2026-07-21T14:43:26Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:66e31431d53f75c2927e612409e9b7407ccf422ceddf3713cff78daa5f21051d", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:99bbbf9f0a006b3cf042988e6c09135395134a8dcc256009698179546e66e983", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:cb238955081e62b5eb75db2d0be46dc428ebe675ccf40d1fcf5b8bb621ef0208", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:b9fe8608c54dae066dd5d9858603eded98545b54dd73b960417c0e704ed33a67", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:5eec8e898c8247cefab6906806a417be8a1adf4d74c59f63e97a731296d3682c", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:d49ffa217f442b758c0469c3d32845c59c79d1193e7661537314782969dedc27", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'archive_account'", + "evidence_hash": "sha256:a98c123c171987df0f920d6e85035946ae3d82a3b2a4c8f8ea0297f009fd12de", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:799c5a067a87bcced4b28bb39471f0f2379e08d855273de9f97f0418fed57b04", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5cc4c400f28858f6b36f1c820d80a00c61c3ea053fac6937d5a830ebfbf9606c", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:610bbc3d232cef0421e5cf9d636739899f9f9f643419d4a16aeea93b23beb747", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:abe34365e3733408ddfe93fd01a9701f377baf3293443310ebf937fe2852dd30", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f78dbfa82f9653b49f2cbcdb194801570b0dadf38de35de613d658145a898f34", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ef8a8336ea0d2072dcac8ce51b38ec91c91c24363fa09a3c3ccffbcaa8665a0d", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:da0cf0be7f08162f63b4ea245fa942d131fb1cbbc42c4de5941057b498429052", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f146276e981f364ab2a92fde9c4f3027fdbdc6b6b2d0462988686a1c194b9b60", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d3cc8ca0fc779a286c72a829f3b9818bacf77c0d6a142850318d8a9d21b506e4", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b890fb426f2db4e81c97ee54d022ec66ccabbc53b20ea54bef564a00d6ac801e", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:94056cb32d4c3d0c57b8a236da8edc21cfec62c3ff16db7fccc8c52904652434", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a4cdaa0254e0e219ea428def8d5a89e9a8721a9141a9b8af7740c01e664002b5", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ec62f36e179b4a3d9fc888e8b2075f407e733722a20e5d72fe3c39c7f4263bd6", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:df331f9dd2b1213d144961dcb40512184f45db7c01a074d044f6fcc21203d5d8", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:49617f9b413f29877cd6952608682186efb5f60644480e8eee28afde04e816e7", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a1728ef2071b033284e2b84f1d50c253c8748156045211742efdef763a529ebb", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6e2df065ee9d8de1b6187fcb696420bf2ddafface9d2aa19c48f471dad58b67e", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9564ebb710721c72dfe315928653dfd81ad5368089f5c67c307cb045f5c4ac06", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7c09e6f99684d7aeb4e10de02a22eaa7dea046c4dc64dbf17d74aa702eb889a2", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d5ca626b0efbb06341e162064ab7a2cdc0b58eef5547b68d0bb3e5225b3a4a43", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2e6a4c51e5301ad411be75dafa3c56ba2d65f01753505245d729c7f0158c0241", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:96d3cc69c2bc429a5d3a24f5c3766ca7cbf765ff2d5cf72bcbfe415570149bfa", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4ced5fd1cec55a79afe68577edd67a3aa9c7b9317e33299fff02c50ee48c16e8", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c5cdba4293b9df85175db40755f9bca7c65c65f0d634a79de5547f7e3b57369a", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c3f90438e0a02873944f41e8098a8b026651b75445567aca5771b87499f97f8b", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb1704bc8b45535c237b175708dbc6f2f111b62d9bbf58eeb06889197b5b6023", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:36e7b40e3234ca7fdf7523eee9a6e67ecb00583a7c50d285685aa93627888796", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4b58423ee140bf338229df083ad486964231de40fa299af79e3b527fcd8cc02e", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:700d87fa5353527f62abcad7cb43398d850e903be715661aac4f24854f1c9de5", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:69bf50e593e86ad3e91f88977e4ef757bfc0ec21b55dfbaf57a91804e0f65356", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:be19802234c224e1da47032a40faae2b2b4df4d1ea761d1dac9c9ce933a10f85", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:df942d208fe2ae56fd9501ee7b2e5d026000eca44a7a249469e3315a5384752b", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fc5f71d3145301ca78db68c8f94eb1719453010885dec4049cec13933dab9d76", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:12d0a468182ad749af470fe1a9169922a13bd434468ffe91533df4b5a255311d", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8552a63379f8a125413d008aec6b1d6c58f3cf62a3d61199ec1bb0b0cce49a76", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:52d03023501f3b1c544fe75026f0a1e1b67bd062458ab6570d06ca6ebb8e7eb9", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ecac7de3acc6648097f4d323837534decde7b7685ffda35113d7661d7aafc84b", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:e13bf440d79fa87497008558e1d24b3188b85c7e57c02eafbdaeacb6e7d90261", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6731899cbdecc8e01b212f40eadcb45dfdac79952f0aa430105dda81f73df08b", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:19e9be061ce6f0b91cfcb6df7cd4c614a115c545556ce7f5b6d2ee58d0ddba92", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e20b9ad03f99ec623dad6904643793c0c9440c9204f31a30a6ce0d0f1f70e5e2", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c601a75fe194df23863ea709d56c2b3c2f168ccf05a6f516b934f0752ee22b72", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:38c6436eacf51bd150df2d4d52ac150dc1c4d5c2b6ab864cea527b66f4ee733c", + "evidence_path": "evidence/20260721T144326Z-35a2ee51/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 42, + "failed": 8, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run1.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run1.json new file mode 100644 index 0000000..4979587 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run1.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144048Z-3ba31d85", + "timestamp": "2026-07-21T14:40:48Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:01619c849586a6cae5b4d9dadfdc86af8b60d97be8c406465809a2eb44997347", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:e0bfd3371cae64876950f65de1ad181f368add97c70c16b210b4f27f50e1a696", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:16e72aaf09b18985050d7fcd56fb8ac14c5784bc551770eabff3b733a8b5f2dd", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:f61b416297ab7cdda5646516c1233d024fd1abc01c95b1162868fc969fd87559", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:c4441db216ca38241f462dba352efb23d9fb04a494e2aa5c4beac038bc778e01", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:bef1095f2e4c7fd4c2a64280ed562dfbebd43fa0846c2d4f982f8f9114488fc3", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:cc996a72444d09c4833585955bef0d211a698632bdb5982895aa43d38e356e0e", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3dee038f29934bd5fd4a3116f14ff229626084dcd309cdfe640a8898cc1a72f7", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f00caef4aef2e85dd75aaae0a07b9043904d03d710623d866bb3cccfc15a0f06", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:265c35763068fc861958cd8d15fdb5b57c83e6a749932b91e3eb08922fbddaf0", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:206765e0e0d00bb82201de787046fb594e616bd332c51e6978aaade5c492c23b", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:0ce33edd4fe45786952d38c93fb7f17f882aec640aa7e8f57d35368e4eb9393f", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b5ef71a1563715bb1081e8feeb5b05417a463eaea78d8a226f6c61a236d26739", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:49fce40258d30b51a69025cb3d6f6da3c0e8ffd25e76d22884ece027d864c9ec", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7545559f0cc2b2446a60d4401104f94bb21a2df1a37a934d73187998ee679cc4", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:74932bed83285ac7401b9c635ee918a76c7ec87d0b66b80f6ffbe6ed571b1824", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c647336d9b5d8c3021b3a1cd21b9b2f317d1def3b85ebebebac7bdd2800690f2", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5fcd70a87ba78db887cbf87bb80ea975e5d7310353ca25ade119275a884ca5a9", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8fb1610c9e501895aa9a8c1b48badb1392a179a664ffdc022ca6cc90c33dd4b6", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0620b8b7ddd274733696d5d651d7cb0897948800ffd235d0b69da7b333d1faa0", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8e532c0b4031c8687b1a7a4c9683511a4ca1cedd60218538cf7248599e098e05", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c1656309d8c765a35d01a6de70d73e01239c28dec89ccf300280f9747deea9eb", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cfda6b69ef555cdd382d2ade359434dc6dd246ddaef1a47eecd46da920544188", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:59a3b1fac57ce746d71a51f8477a3d9b6f70db75b842876045d0b07e9826a141", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3be24a6386721224085e9f5ecaebd54baf27584fe82201bd7e0b9e09e5b5729c", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2266e910074d9a6486c3839be483447c8d41cdc1ed37176020b7394e00650db4", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0d7570b05cee6d40a9646c8eac2245e26bffe2ca8e974ca1b571b44ba83ed2b7", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:187f881448ce6b094dff584696f880ba5f819c588adf6f22d92298f74ae956b2", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cf85f8649380a853e7586c02eb4f06a45c827101db725887b38885e94f00ef22", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:943045ae63e70ab9ae4c32d79e21cac994e43d6b7ce81986586b5afcb084f06a", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f81f9441ca2b204d8655af9c3eb064e1f88f119513b6c725621b476cf087dd22", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:05beeecf3bac31d16a10f8fe31bbb8ba4eb4f6f0a6954955d7c1ce36b0ddd0f4", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f9a13dff758d8b85e26f100eb4e8a23ce04010aad89781df73e72accbc1d698c", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:74a511e9a5ab32ac1eb2bd69982d592c80c43dbbe7aa8ca65115056fb675f4bf", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:31014bdee06aa46f01cec089fc7c73327693caa9af3caa9fed798af38813824e", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ef4532caeb746847c26c3365e33701e591e2a7682722ffe446951301467b6834", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d865273cdae1d93a892af52b00afef601011881c4d779d1bb4e269fa75e80e6c", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:417df3eaec271ca2bcbc26d82718ee5ee217e42edfaa09cdfad10887ae0cbea6", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce5cd71a3aa222caad4e1681038e2925a5ce951181dc22b284589be2ab715eaf", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:51bc39c26e8ff6879c80d2b248787b2a646a260fb8609d783a557fb066aa6e37", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:68ee1bcf1a0b86f576696812e4965fbd6bd070db8c78887866773439ab6a4af2", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dae39a953a44fbc02b8da34548e93cfcb9744d7d79c28311171ae1e9d1aa466e", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:18cf6e11c5ac5877386dd6a7286dd3a48a3102b362ea1278dbbc054391e510b7", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:494263b1ffcb0ec1ce995aaae9b598eaaa5051da3c8d5f53ac5761da74795901", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5abfeafff1b26f14ca26690d61e10b2c314322baa6d478984e05f59a559b242e", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dca3ba0539edc2f97b8ed6670f7d80de92091ff961b6502965215b9cec82758a", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:75d470907e77104dbac6bddf80807f9c7c39eaa7de6987c00b1681a2caade3ee", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b74cc96c3bf0bdb84515e96372f6c23772ca83106b46cd29ccd3406478f3b72c", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9d17682e895f3f676bee1764bc8888ae1879fb85f6d7a43b0c583f07e5f10517", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7d4ae5ccd2a7aa2e2d919a5bf0c2c398013b080b51d4068289a927f52ccabdb1", + "evidence_path": "evidence/20260721T144048Z-3ba31d85/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 42, + "failed": 8, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run2.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run2.json new file mode 100644 index 0000000..50fc8d9 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run2.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144104Z-874d236d", + "timestamp": "2026-07-21T14:41:04Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:87fb0eb24e836b67c507e28a57ab9da876da5cc536fdd755a9da353c0db0762e", + "evidence_path": "evidence/20260721T144104Z-874d236d/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:480d16b64bc454f36d2155db77980722ba899355ac315adbee29692479612979", + "evidence_path": "evidence/20260721T144104Z-874d236d/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:43d4d6bbb8c8d5921442237a7e71ffd056606033c7c32a8214bdc8f9e2b310be", + "evidence_path": "evidence/20260721T144104Z-874d236d/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:00c95c62e4c5d38e3f29fe4381e0b1d38b1415608ae2604e54e1a9846f64b60d", + "evidence_path": "evidence/20260721T144104Z-874d236d/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:41173ccf073cbd33eff5cc13a4f8fa2622f3bb915a135d6077e77843edf19e76", + "evidence_path": "evidence/20260721T144104Z-874d236d/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:cad7e6d4d34fdbb2625d6980c8e9dbcebbfe02def1c8a6b6ff490f4d9823f823", + "evidence_path": "evidence/20260721T144104Z-874d236d/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:00e3ec110e9a4d49de88c4754d8541e7f2ef017ce35e27ffd3370c67512749d7", + "evidence_path": "evidence/20260721T144104Z-874d236d/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e335fd1b47d3f076af3b8f500c55508ddce996ba5691d2b6db30b94d919ede84", + "evidence_path": "evidence/20260721T144104Z-874d236d/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:42a0d9ac708781616cbb8315929389c0b9ea74462d998fe07e92f4966cc57033", + "evidence_path": "evidence/20260721T144104Z-874d236d/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0c7b2ccde6c30d94216043f61fef95fb45da137a6b53239bd091893ebf45ca33", + "evidence_path": "evidence/20260721T144104Z-874d236d/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:05cea1e253754ba516b51adb35e5984c66317aca8dc2052306c5434a1a05cf89", + "evidence_path": "evidence/20260721T144104Z-874d236d/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:9bfc08b82695510cfbfd2bd535cb54f6967777952acba62eb46233599f7d9c70", + "evidence_path": "evidence/20260721T144104Z-874d236d/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:813ee22e5e3340b65e127d1b24a511ecde3a6e5c755fd0e565307130661f1894", + "evidence_path": "evidence/20260721T144104Z-874d236d/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8b5462db4ecac882a14283dbc91ddca3e689d34559c15f4d4c5fbb8df39d0025", + "evidence_path": "evidence/20260721T144104Z-874d236d/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:768e5dfaf8d40c3f7a1597dcbcf65fd1ce7bd2f5bde07a480c1e274b63ec5851", + "evidence_path": "evidence/20260721T144104Z-874d236d/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:df048985d51773b49ad1a87a8224ebe13bcdc09ee320fb9ec1cd806351b71c98", + "evidence_path": "evidence/20260721T144104Z-874d236d/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7eb330b41c21bea8a260bb47b912750408d57f363d6a7af75aabde643e73f6c1", + "evidence_path": "evidence/20260721T144104Z-874d236d/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1d4af993ed2e024d48f9e9ef7c3ce169f268bb00633dec4a22f0535d65534c88", + "evidence_path": "evidence/20260721T144104Z-874d236d/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e17d0cf9deb1988cf133f8ffd82e52f4b4f5535529bb29ca67b728a773c6a2da", + "evidence_path": "evidence/20260721T144104Z-874d236d/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:43798d2634e48c5757485a852d1384bc53ff4486299c109fdc547677fc4e053e", + "evidence_path": "evidence/20260721T144104Z-874d236d/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fc7b1c3f479021a3fbc707a10791e339474d4d479d0742acbe4b66eae13e55be", + "evidence_path": "evidence/20260721T144104Z-874d236d/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4507b723ff42c85c7723318ef1e916b2c6996cfa0c80fee0e1288de6030b992e", + "evidence_path": "evidence/20260721T144104Z-874d236d/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e899f5082064608fbfbf9dcb4ace457afeb138de9b2f689073ab6a499d96ae8e", + "evidence_path": "evidence/20260721T144104Z-874d236d/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a8918ee00da58e36fcb75cfe836a6dc576bb0c5bf7d05ee82b88712b52d8ba94", + "evidence_path": "evidence/20260721T144104Z-874d236d/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1172be6be611dd1c8e48945f408935952e2db1ceb875cb51201352332d981a39", + "evidence_path": "evidence/20260721T144104Z-874d236d/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2d351355624295a6220b271527afd8dacf792c2721161c1ff5c281350cfb91d7", + "evidence_path": "evidence/20260721T144104Z-874d236d/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eeb327233dee693a75b0646e508f5b98993a49d114cf77ba5b89661f13b0d452", + "evidence_path": "evidence/20260721T144104Z-874d236d/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6353f7b327c212f543eecb6ffe7032dbcc105c8dc2ad7dd616bd7551fedf7551", + "evidence_path": "evidence/20260721T144104Z-874d236d/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d487ba7d0ee35428ffd25684b0e59a315fdb2131b2c45eb9054396db9ece3f07", + "evidence_path": "evidence/20260721T144104Z-874d236d/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e6f21bd74622ad41083b0a8ff71988cdb59d7eb3b2124f69c34d9c43dd6d8145", + "evidence_path": "evidence/20260721T144104Z-874d236d/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1178a102739ee4a37813c5a602185172b5b104e282713eea026d7368c86223aa", + "evidence_path": "evidence/20260721T144104Z-874d236d/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0224cbe5c9b5ff93ed4117d1656a979e4d644af5ac0966e06c9f60f2b5bc579e", + "evidence_path": "evidence/20260721T144104Z-874d236d/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a8e1556742954d7a4d04ba9ab3ba7da2cc0113fb96f187643109309f20951773", + "evidence_path": "evidence/20260721T144104Z-874d236d/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5f03d3ba17d480159e102779c513b1c5cba0702d420e7c77a74adfa08d07049c", + "evidence_path": "evidence/20260721T144104Z-874d236d/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fb24a7afed60c41ad82fa2c6f259b452928c2cb8758e8859f8360874928cf885", + "evidence_path": "evidence/20260721T144104Z-874d236d/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d1da690eebd136830ebdcf70b3abebe52358f815a2415002f32b4ab025c83cef", + "evidence_path": "evidence/20260721T144104Z-874d236d/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f53f6e476f79764f3b6623c9ee77dc3816370b9dce49e26666a568850a8d0f58", + "evidence_path": "evidence/20260721T144104Z-874d236d/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cf1ff1b6377361b5a5e569c94fe60074509ff51ed065ca80e6a2b18ca4c2c662", + "evidence_path": "evidence/20260721T144104Z-874d236d/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:768ba163786043a9413a854561a42a23be0f16772cbb734df480074ebaf7f47d", + "evidence_path": "evidence/20260721T144104Z-874d236d/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6f7557e9cd36708bc3a61cc5ed11971b593695294c3d23c946ad60144a9973a1", + "evidence_path": "evidence/20260721T144104Z-874d236d/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:94a0f831270948d85457a025d894eb22bf4f9a6885751b13a66d733ca1bd17bb", + "evidence_path": "evidence/20260721T144104Z-874d236d/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:27e3526089d3090f66c520591ed26b666360a67b5d7c7455616ebc329ba147a8", + "evidence_path": "evidence/20260721T144104Z-874d236d/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9d5515b6d0039a11bc424dac7d5d9685fa3929d3d13b695d4b365a5174641063", + "evidence_path": "evidence/20260721T144104Z-874d236d/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9b838cfb779ae2952d154acbb5bff4fb30ec2ad925b8e307aaa4f2336d240841", + "evidence_path": "evidence/20260721T144104Z-874d236d/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ba5e38071ede53e7ae63a719d506a294c772839a5acf7ae9e01a501059acb053", + "evidence_path": "evidence/20260721T144104Z-874d236d/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:937a6d97420cadc05cbdeb6778752f4e55cf6a84008d138ff33d5d0c1c66041f", + "evidence_path": "evidence/20260721T144104Z-874d236d/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:023c07598443b1e67e1bfe4ac71d4033873860bd3c5adcad486ccd567576f2bd", + "evidence_path": "evidence/20260721T144104Z-874d236d/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:79fc261079a98ee479a5e71e5c41d6d6ab0b74340d2657bd214b2afa7be7c932", + "evidence_path": "evidence/20260721T144104Z-874d236d/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d8ad387bcd8ddf25c257ab7c71cd038ccea8838f504a1c50be3c400ffe00f6a6", + "evidence_path": "evidence/20260721T144104Z-874d236d/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:57efce569ce697d54837c60a67bfda6ebc4a215eb9618591f32378d4b850d168", + "evidence_path": "evidence/20260721T144104Z-874d236d/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 42, + "failed": 8, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run3.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run3.json new file mode 100644 index 0000000..8d2b9bb --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run3.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144120Z-b2086f36", + "timestamp": "2026-07-21T14:41:20Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:6ca550afc199fc0ae1be26c010344f57df0991a6fb2392983047c4276c001d91", + "evidence_path": "evidence/20260721T144120Z-b2086f36/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:29091f59ec8c0bc15cc7ec81107ffee6656714f07ed088984e78f623b5a80038", + "evidence_path": "evidence/20260721T144120Z-b2086f36/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:86a705c3bf40f60d7691c74ae7dae87dde481bcec60f7034c702ebe614a74131", + "evidence_path": "evidence/20260721T144120Z-b2086f36/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:d0e839bf1e81606a65f20989aea17cf98e31ac13b2a07ec83e5605c30b27b0ef", + "evidence_path": "evidence/20260721T144120Z-b2086f36/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:7fcad15427a15264afb05960969793f406949af708bdf2e31ba04f7ec8ae60cf", + "evidence_path": "evidence/20260721T144120Z-b2086f36/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:ca23e2f4e53fe1d4b84292c2ecef12d13995dee85a576b6004a9c49a6d63d9d5", + "evidence_path": "evidence/20260721T144120Z-b2086f36/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:6fa1d019484f17de6af405fd0f336c0e70a4cd74bc40f20756dda07867c24a01", + "evidence_path": "evidence/20260721T144120Z-b2086f36/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:40f12138363f2d834edee6821d36d9fa4cad87abf1449e6a0e8b9a8c8ea1eaf7", + "evidence_path": "evidence/20260721T144120Z-b2086f36/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9913839440fa98d16e1505e22315f8741b9e6003a53e23c7626cdf8edd52cfb0", + "evidence_path": "evidence/20260721T144120Z-b2086f36/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cd81260b9958fc3643157c53f9df983a9478ed527a4843e241ac20b624f80874", + "evidence_path": "evidence/20260721T144120Z-b2086f36/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7ddd49260c9d3eff6aa3823431192d69d823408fdc2910b879f973437af36ecc", + "evidence_path": "evidence/20260721T144120Z-b2086f36/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:2d4fa21a00212cd2874e9fb62b5fdb359e8511a853fb778f3f3ad4089a28b7f4", + "evidence_path": "evidence/20260721T144120Z-b2086f36/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9a189f2d39894f1c57eb8c570358285bee5c36b322b78db6672734259ef876e5", + "evidence_path": "evidence/20260721T144120Z-b2086f36/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9ef9c8b01aa0d15f5cd6f474f275a29c51febcb1d54add2e516c639bd9ba9f5a", + "evidence_path": "evidence/20260721T144120Z-b2086f36/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:da296cd45f9767edfd9a1955b79ede887bd0177e8b2b3e7e82e2b1e655612b2e", + "evidence_path": "evidence/20260721T144120Z-b2086f36/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dcfe7628cdd6604693900676b60536866af1d879f9656f7a08bbea0efd278c73", + "evidence_path": "evidence/20260721T144120Z-b2086f36/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bf7d9ee8ec5338da4584927e4b76a07f0d57a1d95d1d20cd16ac86bb86dacdb6", + "evidence_path": "evidence/20260721T144120Z-b2086f36/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:26813d180a97e6948cc5b03f3150fcdb574749c32d6658c6f23ded0b7570aaa2", + "evidence_path": "evidence/20260721T144120Z-b2086f36/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb5645cea653a0a6d38df46ce23061cb76fbe9d740fda6c96fda36af2d6b9b95", + "evidence_path": "evidence/20260721T144120Z-b2086f36/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5a11885462fd4cd64711f2f916276ba628b9ca72910d07d6363622864fb5713a", + "evidence_path": "evidence/20260721T144120Z-b2086f36/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2e552192cc91a9e31b1d0fb3636267a25044477195369bf342fa8384075336dd", + "evidence_path": "evidence/20260721T144120Z-b2086f36/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ee374891f96ecdbded759a2f6935b9f4097b38bcca02a8716c0580d80e4e9ece", + "evidence_path": "evidence/20260721T144120Z-b2086f36/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7d3872808246a4e69b04e50b683c7adb63f28324400e254deff5ff22cbee440b", + "evidence_path": "evidence/20260721T144120Z-b2086f36/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:44aa3758a39401d3bd44cb6eebda302e789c5e24ffc35928e0832413b1a26ffd", + "evidence_path": "evidence/20260721T144120Z-b2086f36/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f0aff595b55c94fdc0c94e5dbeaa291f40319cc84ce0f703d2df769a207ece8d", + "evidence_path": "evidence/20260721T144120Z-b2086f36/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ac5e89ccea61aa8e65b2445a82129ced439ac4cd4ab73d2e244e39e2a95153e1", + "evidence_path": "evidence/20260721T144120Z-b2086f36/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1565b80230aa3a2ad59e495a922bc99d32ce4d4ae4d8aca034815e7747bc789c", + "evidence_path": "evidence/20260721T144120Z-b2086f36/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:461ecd5c6e84f95114591b3b91c6cb3dd76561e4ece9a67de765e1f7338754db", + "evidence_path": "evidence/20260721T144120Z-b2086f36/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:20f3d2c1589b01b9e3ee4f5db58ba08d1bad8bf35110297e7cf267af36482ca2", + "evidence_path": "evidence/20260721T144120Z-b2086f36/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:76557e6a2ebb7200a3d4dc989472518c48dfdfbbe836f1a5192570ea14c5e6dd", + "evidence_path": "evidence/20260721T144120Z-b2086f36/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:08375f1aad97f6ff6d4482958904f3a9573864ee98565b680af0c314a73e3a52", + "evidence_path": "evidence/20260721T144120Z-b2086f36/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eed6c33631a55b08b344e16eca49908e70e419d4af08038fed4eb50e9dd7a289", + "evidence_path": "evidence/20260721T144120Z-b2086f36/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0f6c1b05e27652a1c4954895fb42a1ed87dc7168ca676def351104b810c5bf91", + "evidence_path": "evidence/20260721T144120Z-b2086f36/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3dc6902313be51272ce7c663e4bb36b69dc6dfcee1cdd7e5ab303a9a857400bb", + "evidence_path": "evidence/20260721T144120Z-b2086f36/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8247152991ee0b40441add1551439c45ad038bd6edfb47c11913ddc0f9a4b589", + "evidence_path": "evidence/20260721T144120Z-b2086f36/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:da76271ca9b1c7fc72b239569c682f5c4d40f8f6d26ebf000c2cf40d0ac43b85", + "evidence_path": "evidence/20260721T144120Z-b2086f36/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:61d7164303bd91f7d64e70c5e1fbb95860d955798a6bec122f11ac0dd6be5074", + "evidence_path": "evidence/20260721T144120Z-b2086f36/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1cc8d8c3ebe57d382c7ec4c0bab7fba9ba7f21b9228eeba4e0b7b8951d0ed55b", + "evidence_path": "evidence/20260721T144120Z-b2086f36/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d4c30d6f1ac9cc8a171647e0a2e100ed3db3d9939d5600d5cadab6a49f1586d5", + "evidence_path": "evidence/20260721T144120Z-b2086f36/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cf22c97686b24ec87aae999fbf22f2626ea72b6312cc995498021ac6068d7040", + "evidence_path": "evidence/20260721T144120Z-b2086f36/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4b3150b15cafc18fe3d7ece0a410e646856a010f5f613160189baca39edbad63", + "evidence_path": "evidence/20260721T144120Z-b2086f36/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3423413b66eeee73b6a835ae7df0087eecc6256fb4815b4d10f0fd6fb19cf62f", + "evidence_path": "evidence/20260721T144120Z-b2086f36/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:280adf25697402d0af5d8c5893bccf8a17de7558893821e0b5d0714e3535c30c", + "evidence_path": "evidence/20260721T144120Z-b2086f36/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95156d04d7d1d8142456cff86f8715826daebc55e082ee33e5d1684ae50493e4", + "evidence_path": "evidence/20260721T144120Z-b2086f36/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:24d5149bedce33a6fb4baa9368591fe5eec0e5acd6f28c386b39e612ac556ad4", + "evidence_path": "evidence/20260721T144120Z-b2086f36/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:53edd015d9fb7825369008069dd8726a277224df2ecc07f0b23c489844a91d79", + "evidence_path": "evidence/20260721T144120Z-b2086f36/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8c06635f506787306164760a07fd2dcd440054b63013fc375969c55cd98372f4", + "evidence_path": "evidence/20260721T144120Z-b2086f36/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:252a4610537e1b7f05ccbed1d823f57ed71aa54d045d28904b689fb8071f9487", + "evidence_path": "evidence/20260721T144120Z-b2086f36/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2c933416c181b9ae6df02a9dbf69a067789cc1e84ef427730a80f04dec916de5", + "evidence_path": "evidence/20260721T144120Z-b2086f36/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:103ede963b40a31c7f84e8ddb1405f16e29b183fc9e3478c61b83ea3f44ed612", + "evidence_path": "evidence/20260721T144120Z-b2086f36/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 42, + "failed": 8, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run4.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run4.json new file mode 100644 index 0000000..a8dfa7e --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run4.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144135Z-8cb0fd21", + "timestamp": "2026-07-21T14:41:35Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:955a359ec294d4f784036fa9e7f5dd3c06fef5519893c784dd2b7af42136c719", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:376e528d29b58c3a901acfbdc000d094d8b9edac68c6ae8dd9d2b7fa8a92594f", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:be2c6306f8ba07ec03853132506d49b06439a9deb731c8d6d1a6ac2d7130d7d8", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:07d76b9395700f2eaaa27df0fbcdcebf71401cec06ff2958bb2630d6bee49e68", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:1deb1b7c9d062ebc55811a1ed67088a88a829e797e7b84e4d3ffc42aa952713d", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:35ce3ec050ee75c7f09afc67ee5639a8dab9f1e4781089f20bc89be3d3840ed9", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:d78fa1557a7d5bbf5d89be0943accbd53e49a9bb57342f48030e4feea3554490", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e97b7cf2540d68a679cc9a3f663eca2c76d956889b39ac579ec1a4ad0d782895", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3dc202df1aa5e8bbe562a54a196ed3801b6f269b28b9cfa4a4e4698283729875", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eed55d4d7af59af0b0cdc04b49af38754d7b9a8a6bfd709e15f775223a4016f7", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c0fa8258ef76e8255aa6a056e5031d79330e4cf2f0f3e1dbdddf59b45ed6918f", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:b1442a8ecb8125c49118ddc3367033b2bcec31b049cf7c59ac9c739201e4105b", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0e7535d3d7366c506864827277bc0bf090432a82580da6108e2cec459f323e56", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0a7ed767b964ab59b3590dca7759d56d0a001f9be9819958a2c8be5181c861d7", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:668a64acd18be8dbf8af0a213535492cbbccf25bbd1fd4466dac01119dde9182", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ac13db5686f13ddb61cdcb2ba7036babd717de53057c741cb81f2e4dc46f01e5", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:752f8ad23edfab8958d05a3672fb78153915e9b1dcdf529a1e06bac56941cc9a", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:39a5d911680f5d20a6a7757813f172bf5511ffb350fc7b17131e429f6d353d95", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8412c1ad00c78cdec4cdc8f561dd680556c83ef6cba72cac46d8f814c6064ecd", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:073ad6f68db9d3d3a5300058ef6d1eb809cd5d0cad424d21d4f2a43e5c478d59", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d5ac8c1f3c092c02232d3f07da3c657a0568abb89d1ed0b329143f68939ce49f", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:13c154c8e7aa8649fa5234f64a648337dc99e3a5bc9bf1934c3b9bf055dcc2d3", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6c5ec803b9090cd989440cf0b7528a6d023b3ec6c1535464f95155d85101dd74", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7e29c8a4b9d18185f1bbaf3ebccaafd988f0b893e1cc3542f4d240e21114bc20", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:53fa55844fe7b66f95de5aa709a743c6b2dae2e6d7be2d083c2b3b1b26a7af8d", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d2196023915eb25312eb603cd461039526e7a200ee26a5c34ec058a10c4a8ce0", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9ef75c6d3ed692dcb4f26bdca308262b230bdad9bf9a8f1d9931d74e3c038171", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:be4570b2ce1acab2536853d4506b5c6aaa616e6695bbb06997751deb2f5ac25e", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:55dfd6747d6b821dd5a61710c54329555750c7afab3d7de85910573eb72738ad", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:45a893b88795f2b3c7806a3829606e236bb2082a0de7a64faef34f744b803ce4", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:af7c6acb3b66983e99c2ce4ce44ccdff8cc9519cafe809cea04436c4adefdb91", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bbf0676f769b53dbc78885d5ef30afcd51ad110a84fa0c7b8272f8096e5f1c8d", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:171aceaa99852a9e0dbbff69a679908c970eb109f62254fde9212b9d1d53b03d", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:87a2e184bbf5a3d0fdc4c7a36ac4cca1301accd5499a3f6086468bcc77d96197", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cf1cde594a3a132fb9350b16266f4278b7a2659e965e7afc3f14c2f7e86df871", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b50d6c7d36a29a21df4f143e873bf7256862f4eef3f562acb1ff0899f4c36467", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:496f765cab2b07680f5a64a0ce199f461ae061eda5a15a5cf0a573e514c4bdc5", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:832c846d58865e6e4fbbbdd16c748dda494cca648a26be9d665fbeb743560eff", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f243a61b0fb3f1a23aa7d14abcb7c8813a113c2e5460ca88457e33ec9f691cd6", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4483072a52bc17dfa990b2fced33dc7eca7eed5441ff08ec4f0785cd09193f8a", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:813fbc5304f20ff89d052c44a43a0bcd454494d30a0253b8880185369782e01d", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ef26253b0a85c559663ec2ca40b8adf3217468bf8024246d0c2f07ec7751e037", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3323e89e08b9ae747c036d6e6c6476ecb53fe646b6941442ca5e32b8920e8741", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c54b8793a48dd9f5b2dde7543e26a0d455a16cd793c925dfdbe12c903df12fe5", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9fc1b89992089496ee684f8f1615b6daa8aa0171b0bbcbbe5d1e9f777c5d3889", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f543e769d8a472421847d6f027cbb61a9ed4e1fd2abbe8261413e76e1578064e", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fd03c741ae5be3e6ec9604cd2e27b0ed25c9ded64ff2f8e0c40dcc7d2a4cb331", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:41adf22f0aa9a9fe87ddc8299b37d00870452b797297cc6aa65c1995e4c7e34d", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2a5d8d8685ed1a6d3e4b1e4b45c8a85612e00e82303e9c815aae3eb6dc0c4849", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:631ebb7f424fd8174e91e0a2872e45f6b72a6120c3e7429336bfae93747f1354", + "evidence_path": "evidence/20260721T144135Z-8cb0fd21/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 42, + "failed": 8, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run5.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run5.json new file mode 100644 index 0000000..eb7494d --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-1.5b-instruct-q8_0-run5.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T144150Z-b2ef0178", + "timestamp": "2026-07-21T14:41:50Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:59b3dca2e224fab30859ccd9d1e4fbe84a943aded63812689a750838eaaea0d9", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:699c31851e4475036d94abd156f527f87bae941450d1ccd83068ebba0076d347", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:c20d7112a85d7396ec155e3459dd2c3424ce5e88d8b42c5f643f2d1072926546", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:a22b04ce880eab755cd3731e1cca5606b6fd734cd57cf2868d858ee11c7d210d", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:cb0e2d720c62e9452cb844d4a998b2d9ff3d917e59bd1828a3ec115e9ed07da0", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:a8793b679387388ac53e01d35b2853aef4d8ac1e0b701d72f7ef17c546a2297e", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:72719b1da994b9d474098ab7a3b8c60adaf3fed2cb37bbfeabb939f61896f93e", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b903bc0a022ffdbb21d769a38daf115b3aafd739585d5654954c6f1b5b5a8eb9", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b8d450e6bc1388a5040dbf346d46c2840f36342d07c2086e1802ec28b20f9d63", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:79981848ff1a6632c907008f98a9eb3d804fdef1990fe1c760c26c54f0ead287", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:83baa961cb6d31286bf3066e63079188600a0a9e93bc895376ca263eca5fe863", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:6a05f33d257446b547ef1af93914f12f9a4cc24eed04747a6b0c529803cb5bf3", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c094184765e081bc30598c53f28ce29c83672dbec8badbec77d4f9596a8cdd2f", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d609af3fca6117e7a05961b8a6d746d6cb91d5d0d69573e84efbfd8344980082", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7444b3c6521ab0935fb7a63a59b5b64d156eef1c1e4f316e30b3a588f9d6fdd9", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bedd67a313a5d92afaa347731af7503340b915509b794613deb162225aa87a78", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e4a620f65955686ad532c3df8e234e329b105a2961ba5364d1f0c9f8ab8fb1b2", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:98b04efee0ee7d9fb368cccee857a24dafb1c080589a4910838af1072805d33a", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4bfb5f95e400fb9e0e2d6a4245f8a927ae95d2ab97aa179530420d2806fd3c1c", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b196e9c24a46115731a45b196dd6ef773a9d855d25b6420c3b89d270b4c67480", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:07e1e75632128c9182913e4541769d9c53c98a3fa52d6e27cc48f7972e28f722", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f0eca7a58f8ee9ecab74acb1f532412b1a8a33b4c334dc1885e7382f6c380caa", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:06c4c742459c06ea39017a2637c23c1e064d11f7e9fe18227a132cbbab52ec0d", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f76ab049aaded5a6da8e8a143974b41f1cf57f20e7de20cab965abff1486757f", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:05a5b8679ccb5dc2760190da97c3b05f13d26ff7d198c9f1c2c5f2dec86cc817", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6f83c5f4a2d2362cd586a5ede2a20668880468db4ef4a7932cbf1d29531dedc1", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c03d19e04036ed7ea4d6c7ee17fc4824a0d753eb7e9d700e82ae5f004491f398", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6ecd04cd0026f6e868615e3d73b6076e68f0263e432bc0748446a26965c148b8", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb4a743192854f5ccea50d1b9c44dd941cfba597fc49231b6ee6474de4414746", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:97a4bc83e06a5e33a94d8afe769b7b58d38729dcf41a4009046caac5a1f0aba9", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c269f27b9b69cce41ddd1827434e119e65b084ae11bd8f95b4d36a81089e4a7d", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:277c1080986521eb8ec4c7328c07fbc13e998c368a2dee9356ec3e4ec3f937db", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:909285f415650dca5e598ba8e8c52ae7886775df5a63a8204018a31afe8b63e1", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2961b39705d97d0c30adcda16c10c010d53f713ff113bd07c930b86694a58857", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:15fb7d72ae987881454c5eb8542854108b9ab7648b12ebb78cda80cf879cd147", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5bbf83dfa3bbe6c5f4b88fcdd24d8ff001616cb6762d24011f0949e20a30cda3", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6d3d2f37470e1831c608731787f2a8bc2ac76633ff000bc0a6619e54faf25497", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:34ca9af0bf343d23dcbe49758de83296f26f7d0562b2404d0414ef70e2d3f912", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:91ec35c09d65413a46a931d29a675209c9e2fa4d74df9dc95f4583096f3861cc", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:684f24b414639500cdb75d11cd0b142de8450426379d0dcfd6dfc94d16e1bcce", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f02e3d0f46fdc0d62bfeb8d0002786cf62f953ff0dde96442610ccb9602bd201", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:44064fc291b6106b9672a793b8e203f14e0c4f480a3e101fc650f729996f9ac0", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c29fa1426d465c053423e7fb242e2859242351ad51830e61e0a8c02d799d30b1", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3341fc5ef3208cddc5337f43ce9358b643ffe53cbaac188f19333220d676a7f2", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b3006592975bcdaa8214f90f9ff5e950b6d9a9081784a9e44e40937c8a401bbe", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b14e1c67cfbfdc914e791c4a4d6d666a159df267301a94ad2de6526d46c814d7", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d6a5bb39c9ef7086d4368fdb7802390601180b3f0101001e76a431447cc9a236", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aa9c844684619caba6f0f204d89b968a3cdc4c33e77bae81e6ae21aad990d7d1", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8276834823f75c784f4ef205c117bc75ee38545611ffaac2cf7b666a80bc9048", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:77124de1070c2e6507def130b4f0a5157f56e4e48030d22aaf131b10389775ce", + "evidence_path": "evidence/20260721T144150Z-b2ef0178/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 42, + "failed": 8, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run1.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run1.json new file mode 100644 index 0000000..03a4743 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run1.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T143658Z-d7bcb992", + "timestamp": "2026-07-21T14:36:58Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f90086400f8b1403b147efb65edd708d847a69efe6ceadc17e8c8b2ade4f456c", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b363c2e5501c52efafd69ca8f601045cdc07578f210be8769cf3d4d1e7e19f72", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:910a78932c2e546dc4a74c58b8bc4295f555c71a25e3a58e9016326583d5d183", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:e9449b5a7dad2ca227a87ed4b8be591a91f1a81e23c03787480ae58f6b0fd508", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:33831ddbbe2d0aefd098a91811dbdea1e812c0593edc2d33a39d71c33e7295c1", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:274f61e85d9875dff142489274562ddcaf6f10d446d5bf8b03b36c4b5e1d626c", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:df49a98feb500de6e4f272bb0c61f62be1ede002d755241002466e7f17665683", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7608ec13f70fca3eb6b6bfc30bb224133996541cb8f286d7da88bb6bae288625", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:44657755ecbec609bc516ca29322e2f252f102aac11d4fb8ad9a27c04d875c2f", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4323efb0ba5bef53be1ff691bf8d69b146264b9190f1b03f7ab7ad3939515bf4", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b8ad8aee1bba4b608621957981c4de565b5406787c50c14e53475f69fb1ada1e", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c0237bf854ee23c44277e400e6d5bd026fa7022cef246651807bfefaa9914ee7", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4d39d77761e5876bd602721b6d578961c908d55fc0af38c92819e287b72ce6a7", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0859bf179b575245a7d527ad2f5729b127dad11d7d4742919c24a6faea0f7fd2", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:413d823a7022bec3210857e567cbd13eddf13000f39d67e652684ed4d5eb10c0", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3247851dadf1f696dab61c29db4fc675e3011cc82191334f5a6ce2ebe3d6a210", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:94ab05bf0abb2a1574f27691b3244f6a4f06d462904030fa38cff64aab4f801e", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6b185f62928fac8edc44f58c4b4177fe9b13f96f98b206f95920c5a6edc56bf9", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:17097f6e4c03951c93cd8a158202b1eb66a6da660dce4753c9d88485d88bd014", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:07e4a9c6fa656a021ebc5394234ac5748ce3f25e650580dabee5cea99b8d91c9", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bf243c441cf6a13be8d1bd300ff0255a2451a8696ae427217e025b906eabaf78", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ebdc33daf90461d4e3b59a0bef0cf47795ac163f0bfac7b994d72c1ab530f659", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bfaecb70626ccdaf99ca6c1ef1deb9e7a4982568fbcf9d5918a5bdbed4c25483", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d7d3820d499a13b19b6f6ce0107002bffce7d7691f0a1bd1e403d25779066e03", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:12a12082e687ed24d336991b106596d263429fbb60beaf54532b7fb4c38461cd", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:240cbe2ce4cb453085dce858ea77d3f7d3cc5fc47a410985e7d3fffe01bae6cf", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f3330d085b6dd8031c6b79cd66824769921701c802964a02e23a1a8adb547fa3", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f207e8416dbe372fc15fae150037b06cc4fc2179c4230d8bc492844487b32b55", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:200f1c98796862683a899f8409ed86d4e3c6a95e1bd88c5e07c51f466446a045", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1672bba4317c1002559982f75ae0ab35624312d76703979757406f848ea90d0f", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d964ebb385cd9cb60d592a5cd2859e4965c2e4fb3ac1d960d9f03ef23c5e9a98", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a8ebc3301350ec23e1db43e846baef16c7b1913a8da35e86439e4d2fc78d2ccc", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a4392e82b6f8b10f7636694c47325708c0b4639402a916b11063f1b59f608887", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d69dc3254e5ce34882b722019676ba0af53a16fa67bf1f34575ee14501ab48ab", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9dd7f5f4709da3dd519a58afdce3aabdcb72f99aab984ee6f3a96a4e3b0b4997", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:62b7ac1b4770eea97dfa2d44f51607669e0ff385d6c9560b1686ef9c793667b4", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7231f0c36ed54fed20d5dba032a63d8fd2b639e8e717bf31344a0aec249652e6", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7f8258999ffe83cd83591f16f26edacdd129570e68b553db2829cfd3ebe23d1b", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:55c7bcecdf54831e0c80dbbdebca7cf34993398522788d41a9aeaf4217c5825f", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d5ec846d05a520db83807422b4c3dfa0e0f6b708042afc832a197391e4814fd7", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8f24b441484c8eda2ce5d16c18b87517e9de06ef4726e8a450daf44125b4d6f2", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ffc56d12e57b49a2099efb5ed53eea8b2db52ceacbce45c70d2616ec8576170e", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2f3cface150fe79827927f0e1680058ed00c950c293655db291046084923c4ef", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:017b05e083f658ade7f5d53e6325eb6d5195ade4b2c8285af9e3bb0c78a80f29", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:981cbb6e39fe7cbf45b97e42677dfdb676334b52db54669d8347abfcaa595067", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fa1b4d5bd4ccf11323e9a396809c1e01bd4595139d12b41fbc5e44d97f45df59", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:01cd90def88d984808537956b8363f5dee898e9ca566d8af4406e282714e946f", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2da6b9a92937b66ae14132d7b302505dc9aa0a35a46145664c6fed95ecf7710e", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bc9ec342148ad8f7e6208480f73b4ffcfd4134887fd22dd71e14cb85e44624a6", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:05962cd3616859cb89c73670ab93663c951a1f782e60b4609363bb87af80157a", + "evidence_path": "evidence/20260721T143658Z-d7bcb992/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 48, + "failed": 2, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run2.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run2.json new file mode 100644 index 0000000..57f11c5 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run2.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T143744Z-32a841c8", + "timestamp": "2026-07-21T14:37:44Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7278433d20f443feda8185bdb69eea1112046f5e6b94d958c5d4d3c711307f89", + "evidence_path": "evidence/20260721T143744Z-32a841c8/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:671ca3f2749b5c6032dfcffba25bf28fe77e06383fa501065127339a308e9ca0", + "evidence_path": "evidence/20260721T143744Z-32a841c8/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:22332beb0e54b38a4d5a0301ca20317bac0c9e2ab15374d7f5dcf987fd2f072a", + "evidence_path": "evidence/20260721T143744Z-32a841c8/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:ce1c0a3a1f3fbcc1bcd1f02da23a9ff5ee12364717d58d5aaf321a87da662c84", + "evidence_path": "evidence/20260721T143744Z-32a841c8/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:63c5ad40ab2c706170fe88d54a3cfe4bf46f8e3622ca597af46488949f4e2820", + "evidence_path": "evidence/20260721T143744Z-32a841c8/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:75c912d636ad3fc26812bf6fbc4fce06abc2bacc82b382fe07e540ff196b27e1", + "evidence_path": "evidence/20260721T143744Z-32a841c8/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5c82ad5c5951a3bd641de35cefa20f8acaf9b287a469480519dc23b8e0e93b16", + "evidence_path": "evidence/20260721T143744Z-32a841c8/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9685c07f4f9b234ed06d4996bf946b710a2e8740a7eca57ccbbb3520d77e3965", + "evidence_path": "evidence/20260721T143744Z-32a841c8/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2dcde4ca6f8c6110303c0558637134dae7f919533ff09371c266d2c293ca60e4", + "evidence_path": "evidence/20260721T143744Z-32a841c8/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:065d5e0b682702e1098dab70cccdd71082b01ec4c5c2206f5d4b93a8f85ad31a", + "evidence_path": "evidence/20260721T143744Z-32a841c8/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:157eace3c2bc6ace116da3028f442c976ed3caf0d575f03e356ed39b1bfbedd6", + "evidence_path": "evidence/20260721T143744Z-32a841c8/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dc4fafbcd2fd741607d586b264fd90171397576bfe707723cf6a43a5f9cfcc14", + "evidence_path": "evidence/20260721T143744Z-32a841c8/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ea169563eaa6688170ec21f5a4691546f8455d84268dcab91a9c8056a138913e", + "evidence_path": "evidence/20260721T143744Z-32a841c8/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aabe7a788645fbd82effc4a9d22ad90b6878aef25158e6647d22ded643abbd3d", + "evidence_path": "evidence/20260721T143744Z-32a841c8/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:212579cdeb6f003f3ccd6b33d5938639d8e1e02fc89c257815a180dd055470f6", + "evidence_path": "evidence/20260721T143744Z-32a841c8/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:febb99f4fa0289cb3588f468b0857a7af7cb5eaec9179eeedd77cbdce1febf44", + "evidence_path": "evidence/20260721T143744Z-32a841c8/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7bc2675f91933c7b75e9ca0eb5cf216ceb039c31f4f7582558ff348c0651c760", + "evidence_path": "evidence/20260721T143744Z-32a841c8/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1a950f27739c503e076caae461f10b6494ecdbe38b3f209f08b1388ba66fa7cf", + "evidence_path": "evidence/20260721T143744Z-32a841c8/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:346ad6833649e55016f26c4b92ed551a5ed49065e07aaef103d94a598c8813eb", + "evidence_path": "evidence/20260721T143744Z-32a841c8/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:86e581ff1a8394bd2c206a088153c2fcc91cbc8a06b06c8ee2d269400fc1a6a9", + "evidence_path": "evidence/20260721T143744Z-32a841c8/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:36b605de3fd40fa0b41e8f0cec5565c85986957838572d853988853d9e16317b", + "evidence_path": "evidence/20260721T143744Z-32a841c8/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e88a19d3037ac65d40b04e5e830fea7fbdcfbfc73197542ba4056d1c0e64b7b1", + "evidence_path": "evidence/20260721T143744Z-32a841c8/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e1c13476d37e327c1f2e44160e4b3a24ac66d5efdcf22b6f59f0f2e1201586be", + "evidence_path": "evidence/20260721T143744Z-32a841c8/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:23e3feb3e9c389e5e8be4401b44e071c7dcf00bc60cf217de5be40b0be1e96dc", + "evidence_path": "evidence/20260721T143744Z-32a841c8/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e32d114af2bbbd3aaef39f8ee11a2cfcf41f88d9be7d1222dcc0d1adc65c768e", + "evidence_path": "evidence/20260721T143744Z-32a841c8/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:da165ed995318c9f93a117c3fb409f8fbeb0e2971f32f3c3c242dd26a678a3c4", + "evidence_path": "evidence/20260721T143744Z-32a841c8/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:551474a660d505672e57ff844e8832898fef2f146745e5306d44c28812e6cc1c", + "evidence_path": "evidence/20260721T143744Z-32a841c8/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5cfbc1fce1156a3e41aaf35bdb1d220e0e9c566da3539994aa95717f479ca312", + "evidence_path": "evidence/20260721T143744Z-32a841c8/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0174154682f4668b5b72446050211481c40f0acaf6980c0497482b3797f6e55d", + "evidence_path": "evidence/20260721T143744Z-32a841c8/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:244fa979a0f0971635b58157b2e88479530ad4e14b8a021e835e4641d03ff3d6", + "evidence_path": "evidence/20260721T143744Z-32a841c8/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:004ba5bb47caf9d5266890bff8f4a1e5e9aaf3a89f5c01bca1b17147b7e27874", + "evidence_path": "evidence/20260721T143744Z-32a841c8/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2310ebbe379e08fddf29f0d82f0a34575d34af994efa93bcea1de8f54a21f788", + "evidence_path": "evidence/20260721T143744Z-32a841c8/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8791998c53363b3096a77a502198f8f28aeb2423efca5b70c2c7350300339598", + "evidence_path": "evidence/20260721T143744Z-32a841c8/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:957605398495127eb4d814a15c18f41f03bc80b875198eb1e84d58c5400636da", + "evidence_path": "evidence/20260721T143744Z-32a841c8/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce795eb81a8c3c87a02b119a73c2e14847f2ea0f8adeabc95547967fd71a1f04", + "evidence_path": "evidence/20260721T143744Z-32a841c8/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d290f30a2a79722bf88061ce8bf14c23c309c95fd9414cb61262facabc3d1408", + "evidence_path": "evidence/20260721T143744Z-32a841c8/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:67197878aab1432b90e580f54ef62d64148eacea4f5ef89aa02d0d9588017806", + "evidence_path": "evidence/20260721T143744Z-32a841c8/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d679c742beee67e9f092e0793a9cc20e5e8ac16f6624c92857ac88a1fc04ee15", + "evidence_path": "evidence/20260721T143744Z-32a841c8/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9c6c63f97a042f224fc256842c733780f70bece8808ab7c11b20203e32a0a958", + "evidence_path": "evidence/20260721T143744Z-32a841c8/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:051b7ff0a2a06dbb208dd16b6bff5301ccfb23803e570de02d190bf5e5f7af37", + "evidence_path": "evidence/20260721T143744Z-32a841c8/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8fd846ea02e17dd86b6cadf2ef63b6ab961904e89ba2d28ce6d87da3ccaf5d2a", + "evidence_path": "evidence/20260721T143744Z-32a841c8/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cfc0417095e86d04118b49f7b06e7de273f046439157cb324e6700cfa2ecc170", + "evidence_path": "evidence/20260721T143744Z-32a841c8/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6e5d2dcce43ef0cd71ae2897549b864126993333db32405b62253aa941c4fd22", + "evidence_path": "evidence/20260721T143744Z-32a841c8/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d1e62fc22499f2f7f41922af08308a59911c78213192d20fb94b4bbdef98b581", + "evidence_path": "evidence/20260721T143744Z-32a841c8/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1dc98a301b888006398429fb97e76a961706f055ad4d9f9da467cc1382ff0759", + "evidence_path": "evidence/20260721T143744Z-32a841c8/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ec6d8653ebb8a2464e12a9f487a455ed523971b06bde9eb7b98785d18968ad06", + "evidence_path": "evidence/20260721T143744Z-32a841c8/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cd056b911b55e2ce4cbda28af2c284656aa18ece2a07ac0f337935969217b626", + "evidence_path": "evidence/20260721T143744Z-32a841c8/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5bab4340a545d67318a4e144638f7f57e4380b843e4e3c22e64aac61956f50aa", + "evidence_path": "evidence/20260721T143744Z-32a841c8/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:709bfcbcd723aec6bf093c7611b3cfc53484479d10d06a8bab5e3691d36991ad", + "evidence_path": "evidence/20260721T143744Z-32a841c8/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:586c5acbbb7f2cb429e1874ade3b729270bae3613357f2eadcac7607046ae012", + "evidence_path": "evidence/20260721T143744Z-32a841c8/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 48, + "failed": 2, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run3.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run3.json new file mode 100644 index 0000000..3162466 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run3.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T143825Z-d0a409f6", + "timestamp": "2026-07-21T14:38:25Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2a91e8435bc807aadf12ec6a8b34cac95c5c8ff2bbc76ecbdad2f8827515a731", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f4120d5e6db698f26f678c18c18b18280ea150b159f71ced186c99b70fa8f11", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0bf366669449fea31efe923e26bc67902c577d36f28b0167df870a90c8f6d00c", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:b5be721e621060ca1b3f050ea32c36a3c5a9d8a277f981ac808ea266d4c49d99", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c52ff27b014da18c3f3e6dea7cd379fd7b5297e223ea9bebf7fe026e1566b2c5", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:f3d2671237898e05bfa9e9b327654d019ab386a17947e03ffdc7601828a3a2ea", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:911a23525cc3698a372deeaef31cf750dd74bdf12bebae973bdc9150f4822563", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1d1829a790738fb9aae36b022cdec4bbb9515590683519a1b678c574e4bf575a", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de924f6b3fef16a403cd6018bfb88a54d55f84a620c67d3f86af33a2a6ba3f01", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4488f9f8669abd6549d2a5b978d203472f3b13a270c5ec81fed885b923a4340c", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e87bec9f52f97edd161f9cb5581655d1b1087baf2b9565aea01b8b91207fbeaa", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f9e875c261baa30427a1b7d9291721791619b82e2906ba62baed5b9aa2e4463c", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aa524ae7b414a54f4980a4eaebf2e9735b66be79cb18a41cf557a1e41b90f149", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1665919d6d83e1624591f9a69168cbca95a151e881232e21a3f4d2c7c3b78b96", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3c19baf7afdcaca1abb960d00c47ccdd387157a71c947c7db8882e36d2c57eb2", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8f698221fa43112c54ffaa1caa3ac8de5f0ffd0170da3859ba8d2e9dd4b21b6f", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3d16bffc2ec3692d02f6131e544b4a2d3e083af53c4e1067e98ce3aaf23ff085", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:630e84e984a6dae5b9942f54e7e7a8d16eb807159bd583b6d6044f01095f8a17", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:af41b43db23fe26f3a0b081e3c0c5d8ba094831dec33fc2adfce46b0a8c4aa85", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8872c3b87a707d3f4942a57037ee666a73c934d3caab48c1604493c5245467b3", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c3f10411e3ab7e9441c07df121c4cc8b6517c866a482071e33f3e28dde99f022", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:61f87223200b1e9b086b700a8a6242e4f76bb3c99af05e5b93ec4f292a0910f9", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3bdb333c3a4a6878e569ba2312bb996b0d7a44cf40471b366a9f3edb4f1057d6", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3b3c01463d494aa36638f72ec432d70fb88fff9fa6a47a00a6b610e0a1a08bc6", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7fdf237b53869c920f9eea2130e92aa28e538834d1597162005f752022fa36b9", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce9488211b96e416d17ea79ba9ef5cff7cb3316e7c3683f1e0820f43934b7236", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:77190ebc4f2a3d565551cd28342a53fad3b522b3e1176130cc55fe0a9082111f", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3c96d4633328a1ca3f8c75d6e96b561a994a816eb105e2e92e8bf66905454bfb", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aded1898df1824754217247458f8234de48eb6b3d232e709f101c1035515dd00", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fc8bb82d46c969f64226a064a0d12f5b9eb194e4215e984645e659091f482d1a", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:138676c6558343b7160b83229a67294de92fcf114230a990a06265386e901b48", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4f532bc64cb167a58ada7a43dcf87855d61e54e7702c6ad33fc0ae807c33d53c", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a58a5b7e80af6cf0f725b79f99f2209f1ac0deb33d822969a839cfc35e383b0d", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8b75a738f14dc3e0eb04925cf303b52cf43c442acabcde796972dfe6ca2ac916", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:551f07c189bf5069a8d3f0bc9fa86f8360f6d57fad40ab7777a949759c0035a2", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7dcb1737589cf99d247afa294e19fb2828de43911752473dce8e2257a2657b05", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0f84abd7f28380e40a5d48648d1b69dad0c6f6a00b46e6a15780d9ed98091347", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:51486e2812a959e304082d6b34cc52cf634274c8ae0b78782dd426d5c521b90f", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1217832e53b0d958b7b0fe5941959f2ee1d5ae1a80b5021ee81a634b73348571", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:41ac8a1d32358fb0f5fb3c80a57978ebf3dd248ad3500f15dc02ebf7c341367d", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0b6f7a5a7ea8482bd0f96781a84ad9e3fd0aac49a2a14a9605bbcd18a2db9b31", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de8f32da849f9b749e320a34dfaa82faf9b6f8ae56162e0893f4d7840f2bedbe", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4d751200b2f77a6d264cb3472e74400c1d2ad0cd01bced99be31b74a234f8de5", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:32ca27dd208703fc532efea1b6564061d93f0102bb523bbf7932f2c1a1af5423", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:71ffbd36dac3633038bd0c0da99082acad029fe87e19c6c857e4c73e1aaeb4a2", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f66698dc7467cacf9951dfb8bb667012657f6affee25d05bfb99da39826bf4d", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f77f25e1bf75f93c75229cb1118b4378f3ae9b3bba7c1b8890d670623ede6ee0", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5ea397a97265994e0c188ef0132ca57640d33cfa335294d4b2ff925a9d6666f8", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c03538d4b4482ddf579aa21d15268fc1206b7ad25b54f0b5ec46f6fc9506a333", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0ca174e261945bc49c557e8b6600fe3e222e8c2601b283e3bd80dcb97829c7be", + "evidence_path": "evidence/20260721T143825Z-d0a409f6/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 48, + "failed": 2, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run4.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run4.json new file mode 100644 index 0000000..4700eb2 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run4.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T143905Z-16485630", + "timestamp": "2026-07-21T14:39:05Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:281c87f6a0d9d925e9c29a08d85ac7bb2e6f530023498082898a31171356015d", + "evidence_path": "evidence/20260721T143905Z-16485630/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3d623296d96ba51b2717a31ea839e88e81b7962ba8a8f084419ec3478f137de8", + "evidence_path": "evidence/20260721T143905Z-16485630/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:37f954d0c2bae65a88bc487b4f606c8aaf15c448757b48eb874534bbee6c3da5", + "evidence_path": "evidence/20260721T143905Z-16485630/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:187350cbd6984a303bb456db0b12512b7a40a71a9a48f7ca9dcf63130b1edc6e", + "evidence_path": "evidence/20260721T143905Z-16485630/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1bf02514ddcbf471761bcd44a41fb4901301f4418f45b950567c3d51f4433760", + "evidence_path": "evidence/20260721T143905Z-16485630/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:3a35a6730bf1368496825ce5aa2aa4d8f2136a65373bda8220cbd0ae37ca4660", + "evidence_path": "evidence/20260721T143905Z-16485630/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d72b2c1c11ede4c84a3a6b1618d833431789c18d2bede327b43aed98dc039043", + "evidence_path": "evidence/20260721T143905Z-16485630/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a4adb02ab071a34417ddb423d94ec707218d6eec37bd8ce4f4ed56e0e36170aa", + "evidence_path": "evidence/20260721T143905Z-16485630/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:948657afece733c1795b28cc30e85ae08378df2c30cfc125c0630ef7f6679dd0", + "evidence_path": "evidence/20260721T143905Z-16485630/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e33da23ac91a0d86eaaae5429c49c9d354e3f96bd00c8ac7d231c6f1ad87cd9", + "evidence_path": "evidence/20260721T143905Z-16485630/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f96f399af78ed57819bd68d8c65decea138f4e52b49feafe145c75a39681978d", + "evidence_path": "evidence/20260721T143905Z-16485630/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:46463e575df8128317c26fa1d75011fa5b3585d798e95e357430e7dc9f821ee7", + "evidence_path": "evidence/20260721T143905Z-16485630/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e7901c109a544ecb8084910727628611be557db595f400184a694a23978b554f", + "evidence_path": "evidence/20260721T143905Z-16485630/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c20db793b933191cca6eacd64cb1d86579d2c9dcda5b7ba5850be93bebd68a3e", + "evidence_path": "evidence/20260721T143905Z-16485630/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:733ccced5b154408ba0cd791253ef152c9febb76b44e5540d9bcfe682563104c", + "evidence_path": "evidence/20260721T143905Z-16485630/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d6b2b2db00e5e1b6f2dee337131b6c38ad23883fb9f1174f137532435430f6cc", + "evidence_path": "evidence/20260721T143905Z-16485630/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e2a5a9b94140e9dc2d0db703aedc9508f773230857ba83dc69626533736838b3", + "evidence_path": "evidence/20260721T143905Z-16485630/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:676a646e3b32472781f159700dfdb516a7aee97e9b772e8cf6c924744fdf657d", + "evidence_path": "evidence/20260721T143905Z-16485630/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ee951a7f96915089d2cb1bb492fbdb7b8f4f234a3c91c3827483f3f5558a09b9", + "evidence_path": "evidence/20260721T143905Z-16485630/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:818a0b3a9c8ebc6b6849c424f2ddd4a78af9c4e4214df8f33d48b4d33d64d838", + "evidence_path": "evidence/20260721T143905Z-16485630/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:83267dea67643efa55e9b3d396c8052c20a77b4b03170929fcb93402e998bbf0", + "evidence_path": "evidence/20260721T143905Z-16485630/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2929b013eef114622815f85eb6b20a871e5eae4a2d76d5db2984e974037264cb", + "evidence_path": "evidence/20260721T143905Z-16485630/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fcbd30bf59649ef6d4312a9be85d51445dd69b896552e79ca61000e9138521c2", + "evidence_path": "evidence/20260721T143905Z-16485630/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b40381cdf6e40ee6174681bad88557cdf7bfc12e04c4bf9009c1ccdb5545cc40", + "evidence_path": "evidence/20260721T143905Z-16485630/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1cbc4387fb767981043e107b2941ebbde2c86e15cab104bde687c4c671f6b767", + "evidence_path": "evidence/20260721T143905Z-16485630/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:198f5ff9765016772e3ed6fa2fa6e4b776e92623f8363c13c32730b386a563a2", + "evidence_path": "evidence/20260721T143905Z-16485630/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a41579bfcc199faa2b5827055ee71a4b1c0b86d9cb72c9c52af8ceae9e7a2864", + "evidence_path": "evidence/20260721T143905Z-16485630/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:052c264ca25ba67737e50da6a6647c5197ea89ed4a44927d3659c516b184f64f", + "evidence_path": "evidence/20260721T143905Z-16485630/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5c4a6ccad8d6f123c2dd427de100e7759c8185c669592144c0cf4dd0cc890849", + "evidence_path": "evidence/20260721T143905Z-16485630/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:18775851fd4e2e0fbf6a72a458eeb83517d41c8464d80370082a0832d16164b5", + "evidence_path": "evidence/20260721T143905Z-16485630/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cf056d09f9b8d314c707a83f7a68ca0c388b45bc7f23ab84965c18599fcc2235", + "evidence_path": "evidence/20260721T143905Z-16485630/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4b643b19e22244a88bbe2eb23a68bc5b4decdcdea87ce5ddd6b190d7063f5879", + "evidence_path": "evidence/20260721T143905Z-16485630/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9ba4421ccd752ef01a624f1117f57e12497ac74db621de7e1c0e06f6e251e58c", + "evidence_path": "evidence/20260721T143905Z-16485630/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9728c35cabe6f895a0d231bba95f09bd61120b378e9494e7c2151c2be3961b3d", + "evidence_path": "evidence/20260721T143905Z-16485630/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1c89c7c44c4ce15aa68ec55a73f547094ae89b843b70507da82e99c984a1a3cf", + "evidence_path": "evidence/20260721T143905Z-16485630/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ac122fe775416c461eea6ed61dc216339b802c16793fae7828474938d28be41d", + "evidence_path": "evidence/20260721T143905Z-16485630/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8a8f9db7406b03248d5d284ce9648e577aa8510d6c51b026d227805a122f0f7e", + "evidence_path": "evidence/20260721T143905Z-16485630/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9e2bf9677abea7552ac48b2aab16b06e77f0879af1006b8ddde03564b26c321a", + "evidence_path": "evidence/20260721T143905Z-16485630/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7f0be045b14620fb9b33ff26f3315c5ccf9a56b818d13df0bcdb8645682bcdfe", + "evidence_path": "evidence/20260721T143905Z-16485630/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9e60b8ea035f9fa148e211f28b3506695ccaaef05611a50b973f22541798682c", + "evidence_path": "evidence/20260721T143905Z-16485630/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1e8d9daf80dd952b6a522a80099012a324e5774d89cda54756b58c831a2b71ed", + "evidence_path": "evidence/20260721T143905Z-16485630/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c3c849733b5d913a8a2649efb380e73cd02330a64fc04e6f7bb7162bcd223f98", + "evidence_path": "evidence/20260721T143905Z-16485630/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:74608f5c90e29fbcec860b2131c2f58ee4ecd5be1f61b3e209921ff048edd1a6", + "evidence_path": "evidence/20260721T143905Z-16485630/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:049c021abcbe30a7187c946f172674ddb0a41178b8b733fe786da818a5b41661", + "evidence_path": "evidence/20260721T143905Z-16485630/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5246b0fc39caf5092902fca3ba6071f69eb9ac5194bf655729d430411f7eb530", + "evidence_path": "evidence/20260721T143905Z-16485630/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:706c68e759ae2c9a9f74986f72f8c2f14d371ba0bef3e6669c940c33c7917a69", + "evidence_path": "evidence/20260721T143905Z-16485630/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f1504f54210877c1b26fd4787d1b012421c4ad52f86937029e7f05b72e540f1d", + "evidence_path": "evidence/20260721T143905Z-16485630/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:43c6e5fb075832412af0ca609614642565867599bc89bf5d779edae2929d6307", + "evidence_path": "evidence/20260721T143905Z-16485630/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:45447c19317a75ecdcff3a17c97b5dc7084ff9a3a579b7414eaa308424e3fec7", + "evidence_path": "evidence/20260721T143905Z-16485630/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:82ea3f4150673236a1e1c3cfe53a91fed95fa190644d1f539f9326a64fb9ece1", + "evidence_path": "evidence/20260721T143905Z-16485630/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 48, + "failed": 2, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run5.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run5.json new file mode 100644 index 0000000..6bd9181 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q3_k_m-run5.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T143944Z-420533fd", + "timestamp": "2026-07-21T14:39:44Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e9031285d940e770c7cfc735925eb76e65f03e21c5a1a5aa28c9179d2ed4cf50", + "evidence_path": "evidence/20260721T143944Z-420533fd/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6b2afb548b179295d0fa9c4ed798e26fea516ad8fd24e4714a20408001004b3f", + "evidence_path": "evidence/20260721T143944Z-420533fd/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:52e2dcaf39638bc6596d8237e4590b0c384690c62ba70d166bdf394093b47444", + "evidence_path": "evidence/20260721T143944Z-420533fd/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:64420670d940b697a5675e053f8da8a8f6c54826c8bdeeec6566d5d512251f71", + "evidence_path": "evidence/20260721T143944Z-420533fd/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2737ca03b8c120da3ac6bd854ccbb27d9c8c1c3394c212a6b3ae5008f42e4aea", + "evidence_path": "evidence/20260721T143944Z-420533fd/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:b2c591dc87a9309beba3beda1a1e32f3171a67b45552241531d3f92ffd0ac480", + "evidence_path": "evidence/20260721T143944Z-420533fd/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cfcd2321464d7081c834d176fb7183c38ab66128500509b040f009c464e7e637", + "evidence_path": "evidence/20260721T143944Z-420533fd/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e4be47853ef787af6d0d3a571dfa6f824e1069d0e5362a82bdd10c3899ac8d9", + "evidence_path": "evidence/20260721T143944Z-420533fd/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:65be8991b157c980ad96f9d83c210b0d259942624a48ccb673d4dde6db5e4ad6", + "evidence_path": "evidence/20260721T143944Z-420533fd/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4d6b34c11e3f884485c66fabd437c16dee311d1872961d4b09e224f0eb9baebc", + "evidence_path": "evidence/20260721T143944Z-420533fd/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c7df26baed98effbc6179ca96ac52c27889405d144e8e10be7e236bd27f634f5", + "evidence_path": "evidence/20260721T143944Z-420533fd/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8b424f49cbdb9fef0f49a7edff0e534c1506364cd8050d749a93e7e6e597552b", + "evidence_path": "evidence/20260721T143944Z-420533fd/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:10fec414aec81d9548f55efa4baa81af50d5cceb35e70b206b91abd05a374ad8", + "evidence_path": "evidence/20260721T143944Z-420533fd/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3cf4465f3e1c9899071c4b74797d1c808d28578e23b07373f3b195115fdf29c9", + "evidence_path": "evidence/20260721T143944Z-420533fd/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:61cd03d50da38ea62dcef3d88e73948fdca20417844125ecd0e87987a3d33e94", + "evidence_path": "evidence/20260721T143944Z-420533fd/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fe2e437fe9a4d50c75e7ff822653bb564fe1e2dfb9d6721ed400ec234d833d3d", + "evidence_path": "evidence/20260721T143944Z-420533fd/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:667026f6b89d829c4c857d071fcf5f6c0f61a5383f566f82fd184549f01b01bf", + "evidence_path": "evidence/20260721T143944Z-420533fd/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cbc400519078b876e944f6bc36d13d114c794f264a27d6bc1be9a6a9e419ce68", + "evidence_path": "evidence/20260721T143944Z-420533fd/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b5cbdbb10d1a422d4068b77455456f43d3cb9c4378d417c049d2f29e8aeb0b9c", + "evidence_path": "evidence/20260721T143944Z-420533fd/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e979673050a40ced6e97bdfb14e69eaf18aeb7c0db7c62d83ddc06417066ebc6", + "evidence_path": "evidence/20260721T143944Z-420533fd/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6d82bc42b7c13134a4a900ba633d7bdd2d16f269344b1e1550a926f64fdd1bb0", + "evidence_path": "evidence/20260721T143944Z-420533fd/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3c2db30fc17f7891381f07cdd15b5bbfbd3546885ef0f0a8754d80325165489a", + "evidence_path": "evidence/20260721T143944Z-420533fd/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:357dda6408078c304af29ffdd1a654cb74ffc0c20fda8b42c7094bf4e96e62f9", + "evidence_path": "evidence/20260721T143944Z-420533fd/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:027bc7dedffc158c2a02e5307bbeff3a0f9d5e3dba97fc2987a173bbaa04f7c2", + "evidence_path": "evidence/20260721T143944Z-420533fd/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cb20ebee804ca647f55444a6578a9370d6df19e015d30ab33da19546fd4b8b50", + "evidence_path": "evidence/20260721T143944Z-420533fd/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6e9e935b9262a667eea8c8f73028fbc8709608bd3485b110e908a837a456c867", + "evidence_path": "evidence/20260721T143944Z-420533fd/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:76178c31c26f2421a17016cecbc47f95371e52322dbc17ccf96aa4722a951a1e", + "evidence_path": "evidence/20260721T143944Z-420533fd/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:87f38bad23f3f1ffa2671fd76b21c19e8c3d5e470913435cb6b4ef6598dce2f8", + "evidence_path": "evidence/20260721T143944Z-420533fd/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:25a7fe92851aafe2fe9a2538ed3bef15ec5212ca2c9b713f7fbc083dd86a24a9", + "evidence_path": "evidence/20260721T143944Z-420533fd/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5b1c2104ecac887ce88f795c5476956434ba3c0d77d8d419b5c33f36c248a747", + "evidence_path": "evidence/20260721T143944Z-420533fd/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:408e6d451b2dcef79aa929776616c3d85c3c03ffcafc0b20cd72108bd0b6822c", + "evidence_path": "evidence/20260721T143944Z-420533fd/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:870bd1ca931d2dcd07cd69d48cee9fa92fd8acab9b78c75f3a2f8e1b35d2441a", + "evidence_path": "evidence/20260721T143944Z-420533fd/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:da173ea9ba3eca0fee8eccd8d2a86f88dd171ce4e2747e8d47c3a36fa18ee55b", + "evidence_path": "evidence/20260721T143944Z-420533fd/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ed4031b7fdef375ec2306459e307a45240781ba672ea40c44f5a4a5b8503ae55", + "evidence_path": "evidence/20260721T143944Z-420533fd/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7e133abcb6afcbd9ad9f8b9a1e3cced6344849729f097cf305be74847edc172e", + "evidence_path": "evidence/20260721T143944Z-420533fd/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fbce5ef537f7d68b4b90540b6db56f94eab475a22d712bb7704c459ee2789914", + "evidence_path": "evidence/20260721T143944Z-420533fd/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f19307fbf19d2910f4424d9331a884f291f57f9c77a8b51dd9bb74f1e2caee37", + "evidence_path": "evidence/20260721T143944Z-420533fd/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0295e5a8809408859a0e705eb1da66f3512add625917108394a98784a473b0d7", + "evidence_path": "evidence/20260721T143944Z-420533fd/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9cead80acf215132dfd3affac9a32f09074f6d2ca134ec9754e714cec9c80e53", + "evidence_path": "evidence/20260721T143944Z-420533fd/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2878a9796e7badf11433555ca515dbb425db0ef143cbbe501ca914a48136d127", + "evidence_path": "evidence/20260721T143944Z-420533fd/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0d655442a27710f225d4df1291b157549f7150cf293980f26b2c2c2875f7fc1b", + "evidence_path": "evidence/20260721T143944Z-420533fd/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c7f9814c55e33ac73fad61e9584b42a380cb6811617b1a592dd3b97fee1f42aa", + "evidence_path": "evidence/20260721T143944Z-420533fd/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb6a7a9c12e765b56139e330018c768c50e876fd426cdc1d9026962ec59cd2a5", + "evidence_path": "evidence/20260721T143944Z-420533fd/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e5be11a3862da00e935ab646725bd0f7ba0196a0d8f4c7308d8c83675e62948f", + "evidence_path": "evidence/20260721T143944Z-420533fd/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ebbf3770ad3994002e171a36d61a618f272b678fe365c8fd70620765b80ba11b", + "evidence_path": "evidence/20260721T143944Z-420533fd/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f9b55743533dd94d9431249f861fa334558ada4c9bd76b36f230b3384d441d42", + "evidence_path": "evidence/20260721T143944Z-420533fd/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4f23002b90e2eaf46d38ecbd649c3dd5cf4913bf029f4de02b15dab5b82776b9", + "evidence_path": "evidence/20260721T143944Z-420533fd/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:069375d5cc5ae07fc783187bcc8271bb277307582827460c39598832aaf87e51", + "evidence_path": "evidence/20260721T143944Z-420533fd/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ab54d6602c29b289d9c9aeb5b7a61b63b7bea098308c11462f5053beb4e309aa", + "evidence_path": "evidence/20260721T143944Z-420533fd/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ffd0aee54d484583f125c2cf352407636dcfebe8a9bd80e758ff0eccc6a03b05", + "evidence_path": "evidence/20260721T143944Z-420533fd/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 48, + "failed": 2, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run1.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run1.json new file mode 100644 index 0000000..c2e5cf3 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run1.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T143355Z-117895fa", + "timestamp": "2026-07-21T14:33:55Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:3319687c0c45f161518e71ca34bf040d015648650e03d378a4db54fae5bf67c1", + "evidence_path": "evidence/20260721T143355Z-117895fa/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4d2ad0df7c4d2fc06a91c74feb2b062eed5a549677751815974a27a80b806cc4", + "evidence_path": "evidence/20260721T143355Z-117895fa/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:0416aed9d3b4bac074301774fe9176e05562f8add65a92491f74bf587d615c15", + "evidence_path": "evidence/20260721T143355Z-117895fa/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:a39a72fe48aefe23ef768950719c9254047eb1e06bd1c9665ea8ec0279f25b09", + "evidence_path": "evidence/20260721T143355Z-117895fa/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7217fcdece810959744694571c1b27b911d5767528bcd0b77a3c5dccdf3f2367", + "evidence_path": "evidence/20260721T143355Z-117895fa/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:fee4deed4576e03c43192aea5ec5a110cc87b8c3c38c4316d66fef4cc66985eb", + "evidence_path": "evidence/20260721T143355Z-117895fa/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:fd268a22576321f76b8cd9b6342dbee8603b3621e6b74fd1b3ae999b4a0a8cbb", + "evidence_path": "evidence/20260721T143355Z-117895fa/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1ac911b5c8e32a8cab5603e382d3f09e6432aad659e6a4fe6b5b83021e7e11ee", + "evidence_path": "evidence/20260721T143355Z-117895fa/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6d5d8cd8841890d8a6cb17120ab23d0c95f13ce1e312a9fe46e9f31240d92f6", + "evidence_path": "evidence/20260721T143355Z-117895fa/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:712b455c981c21bfb3e50b8d9d105a749ab0244a5e82d203b782db27fdae6af7", + "evidence_path": "evidence/20260721T143355Z-117895fa/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:abc959683fa4c9c90f923322dd851defac083bb9a8ed644b018e9267e647cc29", + "evidence_path": "evidence/20260721T143355Z-117895fa/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8ef1583fa4c37449d73d1b9cd022cc663bda2289724082b282ee7751ce8c882e", + "evidence_path": "evidence/20260721T143355Z-117895fa/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c425e3945c27297a302ce6a9e92c030258b11cb4e910ca8f62ee8816699740da", + "evidence_path": "evidence/20260721T143355Z-117895fa/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:094f86423c1ddb37f3cc781b05c6e61af39eb7578576dc919218425f4a8d4be3", + "evidence_path": "evidence/20260721T143355Z-117895fa/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:73edf5b29699d4c7788254f8b82b7cd376d3d7bc976dea64ff3d71aab6132063", + "evidence_path": "evidence/20260721T143355Z-117895fa/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:17fdbb8312706095cc7fba0e44c235a1dde8face7496b73ff6f6c7fc9fa0f749", + "evidence_path": "evidence/20260721T143355Z-117895fa/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e8cad802afce1a8cdb7467e18d0a697a8ce4834d334db2435c6827e762bb8aa5", + "evidence_path": "evidence/20260721T143355Z-117895fa/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:92a37f0ac6763720921a0b3d647656bf2fa44478fa75512a445efc9531cd9b64", + "evidence_path": "evidence/20260721T143355Z-117895fa/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7729132e6de9ea5b2f515596309f2b0d01f275bba0444b27d4f7070ea5aacfd9", + "evidence_path": "evidence/20260721T143355Z-117895fa/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aacc5157f8ef5920e629c28afe064a403dc68e52f41d4a115269b66a7e5bf15e", + "evidence_path": "evidence/20260721T143355Z-117895fa/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9be8fe70b27f0f22913fd0a2cf783837e1f2f778711572eb0b54d968cd884dec", + "evidence_path": "evidence/20260721T143355Z-117895fa/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6bbe0720d60ae4332fa207a8f98ee6482d9cd7c0482b5452ccfedccf40c7e55c", + "evidence_path": "evidence/20260721T143355Z-117895fa/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e4c7fe396248ae9a3437f6d1770201179f2226c60bb32a0a21f47e7b234416c9", + "evidence_path": "evidence/20260721T143355Z-117895fa/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9022b2558b3d61e6dbcf73b1ead540a9a390555d7bf1a26cfbcb1c91dac8b72b", + "evidence_path": "evidence/20260721T143355Z-117895fa/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:715b9a6216b24c3071c2c6352daca1b9f8fadf54747a2f26b2700d6285a24f90", + "evidence_path": "evidence/20260721T143355Z-117895fa/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:32fbd77bd8339db2646ee997f82c997ebf4083e8c4d2b2e562582fc7499ce289", + "evidence_path": "evidence/20260721T143355Z-117895fa/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:28075181ca404bdad9d1f886e3c5c7c15dfeda6153adaee4b73d5ff06631c124", + "evidence_path": "evidence/20260721T143355Z-117895fa/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6465917d5beb007e13068496c4df089fd418cd44b7d8f4a9f50ab1d097c203d9", + "evidence_path": "evidence/20260721T143355Z-117895fa/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4ca1920c70f13147e844ec821b0823b702f9621f739a2d4afe7ae8e30d6fe134", + "evidence_path": "evidence/20260721T143355Z-117895fa/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:53a21be6845078847986c4dc9eeb4c0f90200018e3b2df678e5e48119e8e7cf2", + "evidence_path": "evidence/20260721T143355Z-117895fa/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6990cc68f14fe5b5a78e897731af9a6eab81a3d77ed64ac8a07aa66872ccbb88", + "evidence_path": "evidence/20260721T143355Z-117895fa/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d8906479e2312c3e2ac301dfc234f64a50ff0a0cbd36fb35f44ce2d0df60769f", + "evidence_path": "evidence/20260721T143355Z-117895fa/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8d0dbfa39350f2f4331b4ba5b18f6cec30c9a8a50ad478cedac26ccfce5af11b", + "evidence_path": "evidence/20260721T143355Z-117895fa/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f3caf89e3fa6bacf5d0710e302b65a49c7cb6ee6212906fc197a773db03f8ba4", + "evidence_path": "evidence/20260721T143355Z-117895fa/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8bb895a39439cda271ebd1dd86894e684c08fc9b8ce07be4e4a92a9b6b7f002b", + "evidence_path": "evidence/20260721T143355Z-117895fa/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2eb91f141a9383e897b42f09c7e105481963ae4e521a24f246f3f6f3b1bce461", + "evidence_path": "evidence/20260721T143355Z-117895fa/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:497b67c275bb91d0d026e72a3da598beb78175aff77fc0e7f789e902d619a23f", + "evidence_path": "evidence/20260721T143355Z-117895fa/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:65ce000f7a6ce7464e83e578d0bcfd591a75ae83db282ade61c9af92ad24e0ad", + "evidence_path": "evidence/20260721T143355Z-117895fa/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0905f26f831833481bedbae485a6192cc490aa1020daf63ffdcaacebf6e0a411", + "evidence_path": "evidence/20260721T143355Z-117895fa/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3c6e689abffaa715619015e5ecca27b8798c02c0f431ea87deb06228eb4e2385", + "evidence_path": "evidence/20260721T143355Z-117895fa/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:26ec2a8d7b934b1c078b6cc0459ea31642ae9c8a635fd4b53c9a72058bcb6d30", + "evidence_path": "evidence/20260721T143355Z-117895fa/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:702fdce6a2bf8b852784402ae5ceef418005ca1c0a237ce72deb12d07e4e4d78", + "evidence_path": "evidence/20260721T143355Z-117895fa/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6c65fd66751fecf1018c88bc2b534892c2f15be2fccce208b052a4a0089d3237", + "evidence_path": "evidence/20260721T143355Z-117895fa/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:28ae616f46963d70918ff1bd6fe077415502128d6f099ca247cf3da0c46b9bf7", + "evidence_path": "evidence/20260721T143355Z-117895fa/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1cd3f5041f9c9d83b570503af6623790c3b67b4b3771c1a37e4749313d1d4b6a", + "evidence_path": "evidence/20260721T143355Z-117895fa/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3508577bbae21cac8224c50baee51019ea70b45696552e151334e2fe465bd358", + "evidence_path": "evidence/20260721T143355Z-117895fa/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c388c9c58eddb75d92e9928a32ea509e01be3afa7d85e701bfa2084b2c06f3e2", + "evidence_path": "evidence/20260721T143355Z-117895fa/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f5f43590e5c02d3b1a26161e0173a803d67aef86528c2e151504d47f9d831edf", + "evidence_path": "evidence/20260721T143355Z-117895fa/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e26ef7a0fb1376a02be34a0c2b9fe31b252d33817a503235b8cc344e2f2f3ca3", + "evidence_path": "evidence/20260721T143355Z-117895fa/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6f0f19010730afc785cb82b229e23e764f7fc27b9616904a2fce7fe89ee8dd3f", + "evidence_path": "evidence/20260721T143355Z-117895fa/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run2.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run2.json new file mode 100644 index 0000000..ea99b3b --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run2.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T143431Z-1c9b0c73", + "timestamp": "2026-07-21T14:34:31Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:8af3478c763ec30c23bfedc5adb6a43a52ab3eddcfa3353b911962eb604968d6", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7cb327ea41971556b187e441de98a49908063bd2917c12561f347ed30c19e488", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:8f1d120d065b3bee1ce38546501f23c55039592960b83ef7e64dc891c0bd3c2a", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:8e85ba114923e45135c75b7a475aba5bd5f7343c00244095cedcf440c63f7267", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:271dd4405c160d7590455574b98729255708b500e5cb6520ba9e41e0232f5c26", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:f78c322398c042a373d1d457303fd8a939d8e2665c0d457e21ee17297f0e8f4e", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:ab0862f5c1503449ad16b9b89105ac00c4c58ad1b0dd1d8151a3e45e0a43e07a", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:62475c35fbfafe2b3d8304bff48d5252b80e9ae94f151bd051d9960216f23387", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:da2f196580b51200464bd3656400da8c50f21a84fae65fd015f182089973447c", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:92707a9c20a4a47606bd45cdb56dccee6b2fabaf6166b464006d1eab7f3f93f4", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:11186c67b905464c2dd77d4bc6b4249479ff3829c0c6eed0900a3b72bfc2df86", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:34dd489e746b37484606d9101696150b5a00da5c965ad812e7a383cdc68659b6", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4de28bbf6107606e5c25571da24cc058edbba0904886c5e8760df66b2da13bf8", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b966e8290aeb97c6094d282d05ec6cb5136c5e1c1248644eed1a25c20c291c76", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ecef50c54684528250b21ff37428a13fc1fb4a70a48728de49daf07b8507c6f6", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:71e35062e3f23f3b21b68c63dd41461233732c3b840794be9697f41900dbeccf", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7e4ef8a1fbbb33a1d27746bdf919a713f18cd22d60b1446c1d5fca2ccc458c41", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:377f950d63c96cc413076c07c1e2a495392f5ce0f3a790406e5e4aff0409e2fb", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:56f59b5c7192aee4ef4408f4f5c66472134cddbaadc87b5dc528c269c97dc33a", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:09dc14d3fb0495c08d466d4dd1f3904140d5c38ca30d81951e5a9e2b07b6666c", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:74c42ef852d5507c4e2c054b68bff02f33194db2bf6bb8ecb3b0be27a010a353", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:58ca38496ad4e0445fcbf2d63b3769e3980dbf43765242c3a7d831016d45f32a", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0bfe8a6ad94e7193c49579a904e8862fd4f19c9bedd7806fcfac159db0600183", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:db1cc9580ed2134a8fef15f96e98650360ebb1c9171d827f1eec72c670c1f527", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de60ea0a7d1dad801b6283c95be585e07c35cec28e7921ed875ea093be0b46fa", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b67e4977d25ae511ee78fafaf43b5a8a8b615782726a1fd1c114a8e9abb81993", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:69f1e02b1860908a7ca87205ddd4d24d9da251a88553cbe996ad4d82c5a1fa04", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9d4b939f743f526bc4130cb7fa3acfd37da96a9a2ef724ba1d67f21e32b763a5", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:104edeb41ccf4e98c8357e49c331e7e3a43424fec58e8be7d13ecbc20a4a9142", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1a9427611e6704e73dd44a89d17f1335c0449b40824da74874a5c5674d57fdab", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:45dde801d23971b2aafb6cca4898d03bc62015175523615aba3d1badef35b874", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:985e9422dbb80ab53ddadeff29ca3badced67be87567d61eaccd2cbab562a33d", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cf80eb010c8ad740f7913031606135cd5719075a08eccd358adcf46f872ed0dc", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0835c4b1dacc8987120f93b896173e79e6df08456f149e255794e12897254164", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4937f98465240b9d96855005ce1e2b8cea8d21607c940542d63cd5e0d084f0d5", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5d71f1f8c9889bd95c791efe7411a42ebd3b5b8b99802d8c24b7ecc3ba392b95", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9eba1b5cb342b4fa74d1f8663ed6288ccce897df8a0ea89f5faff60f48e2411e", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e45507b08f70bc72cd3c77776bb94fd171f54a022eea2e325cf6ffb6ceaadcb4", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ac4f8aa102e22f980cc8968d3f6c548572c947152d5a0c4ed96628bd116093cf", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dd06ddff79e432d8a9b29aa0090126876db544c20d8ccb8ed9a5d6c6805109f5", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dfadb4a3d67ccceb37b92b30c80819ddc24ffd1ebd12901a3d9f7d2f69e9a9fc", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:044785a965388486d99805c11a65cb430fb2c520635986b8d90ecae8afbf7a26", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:983186eee35f28876b95f6c560ba8dcd4f0272cc2a5d3cf481e0ad51e278d567", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2bbee4417168f485bf866b3bd9d4258b0b4030ccde0b50f164fb656f0b0d95cb", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4585f7c332502168cf598b026142e2dd8b7d7690cd79b46433c381036b4e49ce", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c257beadfdce1dbfdefbf9a5f77ad53fa8d3b9fda462e335687ab87537d966e6", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2299017a0c869c1370baf292a222976b602d205339c26604dbc9fb5392b911c9", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0f5a8d0d54771ef195ec256c52841e12b4f31e0bac20c8b43c2748300d76339e", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4d8e4603600b68a72e3e12c949e8bff73a0b91ec422470512993e9ade1bae7cd", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:53e56a940a7cb7c9c274fa672d02214c38ea2fd53e91a8f077494ddf0806f1cf", + "evidence_path": "evidence/20260721T143431Z-1c9b0c73/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run3.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run3.json new file mode 100644 index 0000000..0c3a178 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run3.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T143503Z-4b47230a", + "timestamp": "2026-07-21T14:35:03Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:1c7208e949fb4cf4c60709c1980f5f83f3d6bb8b1e0aadc58c7fb3616e6d2b09", + "evidence_path": "evidence/20260721T143503Z-4b47230a/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:961571b734b7b80246ac8e3ada36d10f45e3fd31de715a43de97f89415e1fba0", + "evidence_path": "evidence/20260721T143503Z-4b47230a/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:c0469840551975bb91be3b230a574606333239faf7eade51873d489013c23591", + "evidence_path": "evidence/20260721T143503Z-4b47230a/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:f16b6e52ce92382cb4c27c6df9d6c66de0aecd34158f04fc53855e14b323ca2e", + "evidence_path": "evidence/20260721T143503Z-4b47230a/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a730e88e8a010d4802087107eb16c268c46b58638ffb57874614ab49c5045227", + "evidence_path": "evidence/20260721T143503Z-4b47230a/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:457bcc739d0e0493c687a591cd6a6a83e6da51d3f720029fe94c0111b6d0b0e4", + "evidence_path": "evidence/20260721T143503Z-4b47230a/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:75ba81b963060a66f110b6e9fe9abde7ec3c72adea077c7c26d76056cfed41f2", + "evidence_path": "evidence/20260721T143503Z-4b47230a/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:33be6cd0d898171c5e0373ffc75c24e534342424beb828d3305cf4db09868599", + "evidence_path": "evidence/20260721T143503Z-4b47230a/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:53795d01e6222b24cd71ebbbf161a809361bf52ad4f7b8a13a033cb1a4d1ec77", + "evidence_path": "evidence/20260721T143503Z-4b47230a/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:060b9816b64055822a0f01fbfb4ea6e388c2c82ccbbdcbd5405640017f049895", + "evidence_path": "evidence/20260721T143503Z-4b47230a/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:78597b13b551e1f8ae4abb0a1d68b9d71b708c04e9a1493f46fe32521c436c01", + "evidence_path": "evidence/20260721T143503Z-4b47230a/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ba61d26259ac4f74a43e6acf319f27073b306b02b4b5977ad23490c9a7fca399", + "evidence_path": "evidence/20260721T143503Z-4b47230a/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3592cb827730fb31f1ac11c9a60f99cd69a4e176fdfa835937740b50ad0145c5", + "evidence_path": "evidence/20260721T143503Z-4b47230a/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:df06abf37ea168bae26e04f23992a42ff8fa7c5eea95c4e4deec2d9d047be73f", + "evidence_path": "evidence/20260721T143503Z-4b47230a/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:482a268020e07271243b0ddcc89105de10e826ad9cf1c7425bf76aa78fb138f0", + "evidence_path": "evidence/20260721T143503Z-4b47230a/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:88c3182df3ff2b9fe3066bdbc31f0015c40037fb34ccbda0cd3fe89b48801c49", + "evidence_path": "evidence/20260721T143503Z-4b47230a/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d4ca6c75995a04a787b84880a60de2600b855b5149ff37b5196d43b7b3026ed6", + "evidence_path": "evidence/20260721T143503Z-4b47230a/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:86bc24fa8965d754fc86108a2fcba0835b790eda1de4d6cb999822e0529de41e", + "evidence_path": "evidence/20260721T143503Z-4b47230a/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ef1636936f8224993a8baff50332eecc3c8abb6726e098f98c7fcbaad9d6a084", + "evidence_path": "evidence/20260721T143503Z-4b47230a/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:93d4541286c2ce358ef2faad795cd6dcec933af359ebe09624badb131085d8d9", + "evidence_path": "evidence/20260721T143503Z-4b47230a/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d51c4f0af6d8a8a12727cee28541321ecbcfcdbd96c447413d4551a4ecf71418", + "evidence_path": "evidence/20260721T143503Z-4b47230a/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c0786c04a1f13515ea22169abf07b94a242fa66e9c1192bdd64a4ea93047f62a", + "evidence_path": "evidence/20260721T143503Z-4b47230a/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a9990608960f44da046fe4d18229999ba2bdefeecd99c0ba0f2633d54d883c44", + "evidence_path": "evidence/20260721T143503Z-4b47230a/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c55673292fe200ca1d0f36610870eef671859dfa408965c68665edd635c90687", + "evidence_path": "evidence/20260721T143503Z-4b47230a/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:83e14a238a81499c311ba1dd0de6576f0bbae602d296bc061301fc250af138e5", + "evidence_path": "evidence/20260721T143503Z-4b47230a/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:42ccce6094c0fd967065ffd01a442ec26c074320ff5077da163bddd6a5c10fe6", + "evidence_path": "evidence/20260721T143503Z-4b47230a/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:951a4ef3bdb2588bd2ba3323e04cb0a5e15c01538f7d3307a44e6af6c4c46e60", + "evidence_path": "evidence/20260721T143503Z-4b47230a/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b02afed7ac0fd5d9565d32306602ca33f5a777ea7a4e7a214f90a173dac70dac", + "evidence_path": "evidence/20260721T143503Z-4b47230a/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3c046d30eacbc77c2cc801f45aa7b4f316cb9da4e678b875ae85d146c9df8054", + "evidence_path": "evidence/20260721T143503Z-4b47230a/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a3607b7b46248461e04fbf42e3750d5ee9176d35fe7e32bdf7bd3c8cfb5c19f2", + "evidence_path": "evidence/20260721T143503Z-4b47230a/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f85e28b2d55e65d098d0de8e277ef8aaf86569449c06979acc351fcc8742576e", + "evidence_path": "evidence/20260721T143503Z-4b47230a/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d281aa9509d51a59e4926b440d3e80a4de5aba3b4dd48ab7ce0931e5849d027b", + "evidence_path": "evidence/20260721T143503Z-4b47230a/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3ec886c8f5e1bc0521461b360037e225388402e98836f8c8389a291d0de2350f", + "evidence_path": "evidence/20260721T143503Z-4b47230a/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c52fbece81ab8e52d58de0f5c49b4697837d67411ae523755ad81d1e5445fe15", + "evidence_path": "evidence/20260721T143503Z-4b47230a/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:018e2e3f3c29d311ee9bda8d03d0486bc973a70ca6926920ae6b9e3e69a5f572", + "evidence_path": "evidence/20260721T143503Z-4b47230a/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f9ec179f0e4706f5f14f9994457abf606e94c3a34b88a23215af6ec43b3e9253", + "evidence_path": "evidence/20260721T143503Z-4b47230a/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9aa74defb8208a7ecfb80e0c851e4dba6856bf8ac5db531ef9f1723071486c69", + "evidence_path": "evidence/20260721T143503Z-4b47230a/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:919db8e5914321f24417f14ba64fe6e1c4b5de2c6986182ac53cdd8ea7da91fc", + "evidence_path": "evidence/20260721T143503Z-4b47230a/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1f241ff0f44b3feac655173a8fa5eaa54c2787f0e7a16c85d3373177363c1734", + "evidence_path": "evidence/20260721T143503Z-4b47230a/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5b161a4286cb0f7ef2c6180a136aacf2d332b17ea04b6319d952e32bade6c373", + "evidence_path": "evidence/20260721T143503Z-4b47230a/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:278e1c888d7a372c2f8e4569f053495156b475884f6cb1c619fe8ed01caa3d3d", + "evidence_path": "evidence/20260721T143503Z-4b47230a/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f0f23136112e09ce51fe145cff1286c61ebe2dc86ca4432c6fa5c900111efb19", + "evidence_path": "evidence/20260721T143503Z-4b47230a/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b0727e9ff2ffdcc492656215219ed09d3ef3b36ee9a81f92796135a842b99a14", + "evidence_path": "evidence/20260721T143503Z-4b47230a/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f0ad7530c7dea41b577365d665de696b06147c01d54ca2c7af094814ff3e3b07", + "evidence_path": "evidence/20260721T143503Z-4b47230a/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:43f549f4d362f8082cfb579ed56a07321e234858400f001d8878c4e544f3a81b", + "evidence_path": "evidence/20260721T143503Z-4b47230a/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:94c91f07bc8daf39dacbee5eeca93e8b4841f8016626fdd480d285f459d10ac9", + "evidence_path": "evidence/20260721T143503Z-4b47230a/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a2f3ed0790ce3dd711101b130e886058d76f48ff34ffe1fae2afc2869486e9a7", + "evidence_path": "evidence/20260721T143503Z-4b47230a/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8f3ebf66fccf74c05e675c2e6b1f34d660db2519721c51ea162b5b9e9bd40f54", + "evidence_path": "evidence/20260721T143503Z-4b47230a/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:565e4d2acdc0731702544b5ffde474db79fac46a46c8ec5167782a16761cc63a", + "evidence_path": "evidence/20260721T143503Z-4b47230a/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6269024fecb93c33db081d5129cda8cb64f4546b1f64fe7f21973a78f9a23ae6", + "evidence_path": "evidence/20260721T143503Z-4b47230a/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run4.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run4.json new file mode 100644 index 0000000..ab2c0ed --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run4.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T143534Z-ecb467c5", + "timestamp": "2026-07-21T14:35:34Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:4b5c7273ad9f8f3c583740f4784ef0aa73db94e00e42656505b9e0dbefefe96d", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b6565f89b49d2b28cd7c2c0bbc12f694513722cadc253e96f9d46472909511ef", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:1f28e8732abb4b94df232c6caac6a3df6edcad5edc998036b015e33876fed4a9", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:4fa6306e5feb1420cd3adb871e60fc18dfa19e66998e18851ac7e07d98676756", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bce217a985ebbea78374d68b827a4bfca223f15d577981804690d352b11f042c", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:d7533422c0e84d5252392b1335976769055ab1c71b2061cec647ff313b21e59a", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:9fdc3751c054970723d81b8b3c52151bd8db25eedd8dcf1147cd56a4499d4bcf", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:865293b8b48005fe8fc3265cc147893d8db415c0766fe0b5aa061352b00ddf6c", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ba8dc4069cc1451ce7ea29ec7548044e5ce0f3f7a6b0f99d2099fdf94d8e4bd6", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce7b94c257064211133dac772754a9d9af34effb2b1d2ae5868596e7421a62a3", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dd16d89d73d50d565b965dac1d9c24b2b643fd0daf3e02dd011aa2f4782fd8e5", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8920d68e1d049dc5ef5fd2b41471b136cfd2f016b9b968e61bbb54332169b47d", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a57480e02f61f1c882ac21909cd0bfb5a9a3e96672bd5b45312d379ec0911a13", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ec1071079c461a0e3c487065ff6926be42494f421ad02b920a3ecee98e77caea", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8d02f46d7f08413a91474b20b616dc611e60cbc32079c44584be997856ee72f4", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de352744b155a5b05ea421c3bd0e5472c6cd12af6f7661c44e499a65063110bd", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b2f0ebf7dc89f1a9af05f471aaf027c2e2db22fae4f6178f12b49081971187ad", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:773aa8c1156b7dae996ffa712034197e36d453e9ababa0a020d4da5cc517c7d7", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eef2e273ee551a74bef8e03073f3a94adfd98370278f45194d8026d98c1eedd1", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c3e658fb92c630ac31da1b50e234ed4a4e0244c0fe3224fad0c6b8acad3c0cf6", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6ea31befb33800719a22bcc5ae892e48e32dc0c92aca673de09b93158223a69d", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:907555f575fb4b3e1e03b8e3879ad1d859a7f92ab6934cc66af214b02699cf94", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e1f4fbfe522edc80eaa8f46267574b263d6967e04e4a62e57ab094073795545e", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:59db1d0cc95df37305aa0b996da0c78e3d789ae4616d432da78dc0a3d9fd172a", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:16feb61a582187f3880e2c593cf74687f8798d1648af31ee7530f2a371719864", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:281ecdcbbb4c5550ad540deb406e5af680f18dfb54cf0be8908451520b663273", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:75b3046fdf54b75ebb646c7815576b76639678f61a37f451383734f1e24dac90", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:49790eae6c4027d9fb82d7f292d37258785fcf0776193b92c544886950eed199", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b665aa33c0364596320f9de277437570d4705457e4b9359abfae769e45ee8e8e", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95f7f87d5b2794b46d341691551dfc29f3c7c37da32270d7d7a5eb3434499241", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ec24ec0906e0605a032022c7197e38828bb2130419760dc0624eaa81737c2d3c", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a8565585858e1a4c669cfe5bf940ebf1a5b03a9bda6936cce886b9e6235d3eb2", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e416654170a01b177c5dca0779332c2468ce9aa17b4133676efcc7b9bb729ac", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2bab883b214dc5605de5fc92523a14bda349685530d540cac3682d70d4f474a6", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:29c1fe7d90e958cbce59db510ecf147e1f96a52480a8380d7343729e0ec23ce8", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d19d74c87a519d0abb0152096e0a6f37b6b5165625ea92dab047094e2edbb7af", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cce52caf5e30c7615b6308efd82930a0051e7713f0e69b415932dffd25460328", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:44fcae3dc827b51ed5a4006e68ce61fe8697d2f495c728966023058f1995dd66", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:755949c1c124602c6ca2dc32b8def11f53ced8a551f8a6a7d1f4d93cfd16558b", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b85899843a25e64ca533797be9ca289db55c272c0529eeaa731722025cfd3aad", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6bba6ed85ff82618d0290fc3fa24b0dca4483ca93dbcb1765af20fc50fa9c5d2", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:24d4a546043c90d6773eb4a95393007f0793b1ea3b9a2f99f51587d15d1f4caa", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c1833777cbaf6279c040eedb2f84a50657637084517fc37ffb3f2cffdfc4bd14", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1e2a22335d11d8d635a083335b3400f0267b7b99605882e4f68c9fac1225ada1", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4329c5754edd7ba6a8191e2075bc514e1983e2225d52d6c68051020b0fac0e9c", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2ea1bff946f7dd3e2b08e6dca7cddeabe28f770c2919ef627884aded51f0b327", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5bfae0a52a79d22437af3e9d800e8c32f50f965afe2db6b1d5445e19fc830ff3", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fab5a12991389a8eb30e4f57a4ad31ef01fcba53e4defd6ced23651ce9ecff12", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cc1af32d6188e5ba1170702a7bf2ee24064a746aa16577df5a035802a7d287db", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:79d1e7dab0e6c9eb5e29120575871970bb685a92b9782d63c4a48e0ae9d5b4dc", + "evidence_path": "evidence/20260721T143534Z-ecb467c5/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run5.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run5.json new file mode 100644 index 0000000..cf8d851 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q4_k_m-run5.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T143603Z-c741b00b", + "timestamp": "2026-07-21T14:36:03Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:940acd40873ab9e56e6f6fadcdd7b9ad54b2283151c4378012ed78647a871af2", + "evidence_path": "evidence/20260721T143603Z-c741b00b/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:35c07b22d7cd0cc835ea1d26ba251da7c2b8df45e0174a2a0928def0ede1e193", + "evidence_path": "evidence/20260721T143603Z-c741b00b/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:a58ce7b02c4c2fca4e447ca2f1f876b36ab03d6e5943a6d45fbbf4cda7801d2c", + "evidence_path": "evidence/20260721T143603Z-c741b00b/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:6ee7191060fcd46f51336be349062f3fa1490d5f8d0a0314f0cca0c4f41a2d69", + "evidence_path": "evidence/20260721T143603Z-c741b00b/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:32da88d3cbe25b3347984a5e3a65b859e9255faa78143a883439f3116e1e86d2", + "evidence_path": "evidence/20260721T143603Z-c741b00b/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:de595cf84adb5bc265254fce9e5a0192d0732e702c99d744e18e7dc3f6cfae50", + "evidence_path": "evidence/20260721T143603Z-c741b00b/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:2edd8036a5872bbf27a7042f06942e395e560c61ecf1863c5c3cdfabccb45124", + "evidence_path": "evidence/20260721T143603Z-c741b00b/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1529a32967181f9249521504ddeb97b0c25b89695d470bcebd6ca617d93813cc", + "evidence_path": "evidence/20260721T143603Z-c741b00b/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0fa1fe7096a78c316e44ae231bfe567a530af7b69646bee2b05b4f0bb0a90da3", + "evidence_path": "evidence/20260721T143603Z-c741b00b/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:33c2c1e41aabe0f59c3ce7f42a4c7b73c2aaa83d77a9b50fa89fe6232c629cd0", + "evidence_path": "evidence/20260721T143603Z-c741b00b/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:38724ee957588fb620b4e27b0d6908d0b4709c51784b4b79223c3adf8236cd2f", + "evidence_path": "evidence/20260721T143603Z-c741b00b/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5ff5c8288e5053e38418421bea25b5aa66751ea4e0a5b0c828b2b437c12f336d", + "evidence_path": "evidence/20260721T143603Z-c741b00b/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1d5d95bbf4e86452d71c87a1b404991cfa6b58ad0da826e47a09d119f5fdde6d", + "evidence_path": "evidence/20260721T143603Z-c741b00b/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:373cfbdb974bc8c54a2421c3beae1660b5908e91e3b6fc76b2a38ea1ad36c113", + "evidence_path": "evidence/20260721T143603Z-c741b00b/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f1c37bc2d22458b39ec36f16e335023dc8512e5438817dbdd45d3eaba1ec8f7b", + "evidence_path": "evidence/20260721T143603Z-c741b00b/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f0b75dfe23b27c56222bc50eec24471e4cba327dfc875a50f2a3146bcff7de7d", + "evidence_path": "evidence/20260721T143603Z-c741b00b/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:90548e1ca0ef0e7f4db415f1c554f795da9f3e247c0c3476890fc13879cd9376", + "evidence_path": "evidence/20260721T143603Z-c741b00b/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ab17239410af44d2a30b21411d0a8125258fb9c7847e1a9ace361a8939130c98", + "evidence_path": "evidence/20260721T143603Z-c741b00b/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b73a1c8896fc03f719ee73981bbab1c56882f1f64e13a393dc905fa47e72aaa6", + "evidence_path": "evidence/20260721T143603Z-c741b00b/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:678dbd6a386c68994567838009b58bd93d79ae101200a8732b12cbb900b7b469", + "evidence_path": "evidence/20260721T143603Z-c741b00b/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f5503710eca5a8fb016c96ef19e34d0a50e50f34c49e723619b810713779274f", + "evidence_path": "evidence/20260721T143603Z-c741b00b/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:89c5b53b904ea59466fcf98bf07237d341d7a0eae13a3f8a68f7a029401d5825", + "evidence_path": "evidence/20260721T143603Z-c741b00b/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c422e5ac7d764cc535fc2bc1aaa761601274ee106c8d39b80934e4e278638f9c", + "evidence_path": "evidence/20260721T143603Z-c741b00b/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:273624b289d4075ad849d54a6de7493c42ed58f6d0cf42b9a46f0598e5d9209f", + "evidence_path": "evidence/20260721T143603Z-c741b00b/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:49e06395b7d28080443744db57e9af6f475b7a665c8468ea2f2d73aaac599807", + "evidence_path": "evidence/20260721T143603Z-c741b00b/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:00b39d4dad2360fe6f5f92bd3cd1d4132cf0f525ea63718f30c68482b6022b75", + "evidence_path": "evidence/20260721T143603Z-c741b00b/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2bd3204f184c7536b0e2e7017c274a3a2c91c5ac11ad3ac33f260061c9c03e07", + "evidence_path": "evidence/20260721T143603Z-c741b00b/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4d875f825b7bd2c3eb87b4c3143bf42d61b120bb0d9b86b660e9bf0cd0b4bde0", + "evidence_path": "evidence/20260721T143603Z-c741b00b/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:23735d0a1c27ca863e52fb85e702b329fe8cd1d0573019b037da42327d34815e", + "evidence_path": "evidence/20260721T143603Z-c741b00b/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:45d6b81bfd1484fd111db391032209c7814a1682dcfebc8059166fee09fbd836", + "evidence_path": "evidence/20260721T143603Z-c741b00b/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f37f1d2591ffe7dc06c147582756f2d44b725838df85c6b88aba309907ece908", + "evidence_path": "evidence/20260721T143603Z-c741b00b/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e4ac71810203aafddb97dbd9a587427d94a10cf5fcdab021d22dd89b177b5773", + "evidence_path": "evidence/20260721T143603Z-c741b00b/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce1f0fe4c6c76a2186f46b010191c55811fbefd825918fa3cbef26be76b32849", + "evidence_path": "evidence/20260721T143603Z-c741b00b/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:678667612abbb3f759a2c2922404bc0caaa16247876fa9f1ba1699e3b498e11f", + "evidence_path": "evidence/20260721T143603Z-c741b00b/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d477a8b520bcfcb266ddcf0e37433f995600bc35551c1917c159706e125cc911", + "evidence_path": "evidence/20260721T143603Z-c741b00b/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:08378ea0082392adb8f2030c5e7b07a2ee0ad96c154867c38d877fdcc20699ce", + "evidence_path": "evidence/20260721T143603Z-c741b00b/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:505ffc84803d7f935c34480804b46f4b9e0df885f079689b194ba2ccada70e1d", + "evidence_path": "evidence/20260721T143603Z-c741b00b/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9aa447d234ebdee4ea4cf989844937745ce978d2fde26db5b7bce0f599945b0d", + "evidence_path": "evidence/20260721T143603Z-c741b00b/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b4bff59dfbc2fdad5c69e1221c50059a335a579e9cc45426f73376b5fc6712f5", + "evidence_path": "evidence/20260721T143603Z-c741b00b/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1ea09dccdbc8611fc83cbf8042ea478a5c9e9aaa70d6f721fe5db507dff62495", + "evidence_path": "evidence/20260721T143603Z-c741b00b/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1f47fa53a8d5b027ce591f8156af1c21b16eddb8746f54ec32debd3ebc36f735", + "evidence_path": "evidence/20260721T143603Z-c741b00b/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1ba164b1bdc7140a17fde4508ffb151d32fc0d980a6b7f95e6c74d2f0af6fa55", + "evidence_path": "evidence/20260721T143603Z-c741b00b/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:542f1e78e753a945c1e50008f246e60ced65b04f9e096b83a178b99305e09af3", + "evidence_path": "evidence/20260721T143603Z-c741b00b/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:374f01014cf87fd55f79ff5f0dbd0534b368a8f5eb8009bc3614dddd9941ffb1", + "evidence_path": "evidence/20260721T143603Z-c741b00b/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1de5aca8ff19884d1184ae3f8c9931b375455d8b0ae75752f3073af361074652", + "evidence_path": "evidence/20260721T143603Z-c741b00b/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:428767039fba6bb9639315aabdfd2b0f3aa53bd0b23e9c1dcc2b809e45025335", + "evidence_path": "evidence/20260721T143603Z-c741b00b/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dc1bb4534250418e14d5cb94118c53d62550d11dff9b88157199b109b9428e5e", + "evidence_path": "evidence/20260721T143603Z-c741b00b/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ed5505c601c04197590395b7ca63c8499b145d7296b637179d8dbebc479efbe4", + "evidence_path": "evidence/20260721T143603Z-c741b00b/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:202a38f91cbe785873edb2139a9af1cf59f99987e4a403d066673586b6968ca7", + "evidence_path": "evidence/20260721T143603Z-c741b00b/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:54ce51052df260234fdc590cfe2c0f63b525a6cad318f21eab0aa3014f5d08b9", + "evidence_path": "evidence/20260721T143603Z-c741b00b/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run1.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run1.json new file mode 100644 index 0000000..316c3ec --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run1.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T142948Z-751f1c4a", + "timestamp": "2026-07-21T14:29:48Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:d834486a060275505988e42ce1c047fdec6eb1bf75cc1e094b42324b05958f07", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d047710416a7a7334c409fe78f46e1c955272dfa871ddbfbbca5dc05963dc839", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6b9cf24e4ef3dbb6428fffafb4110fa7f6060aa15d87c49c9d35a9952f659251", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:e907f74f0b2762edf753465b972f0518c290800c186e70e12c3c3c2befa524da", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:853240226ac17fd9759a12a4230a2a580750a383dee7d7333adbed73b3429534", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:42ab648e3108052a25586651d051b8c2c5ca2ec0eb83dd08795137a9ed7903e4", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9ea84fa2a1f6c0d8e48fc543405fc6e6eb784bf77e9f4f46671de447fa798e2d", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c1bf9e72a9716b21e9b115790a655dad1d0047c716e724fd7b128c05eb505ea0", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:22d47cf8ed1b79c961d4ed4782e6455e7da3f916297774a8fcf7175326ef11c3", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3ea3c69091964645150600db5da5249b397dedce96befd00781b5fb7760d15ce", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d83978703dadd74053cc6e0610ad8950e4df0f10952a3c467da9a3baa22baf1f", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7ca288dba50d4b17b06b135b0b5391fa3acc17106405f941bb22e539e9217bc4", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:31da5c78f3febc897130220163ca7cee0ae4df361efff280796925e5fce112c6", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9df81f230a22fe849b17e95b89095fa88941e99a77d410b9dfe3b9dbe2e94dd1", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:84c0b74b01421a669a96bf09dbf66bff5b4e2bd50caaaa75b0e419dc6f1b13af", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ef77545b2461018fef64db92dff96a8084cd89957f872ba69286d2fc699d2936", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b569843415dd9f0aa433de24f36f8efd86e955cc304408b3c581cff26afce2e6", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:746610d84f3d70c7cc539b77141b052fa391efd2e24cb7b01e01d204cb5eaaf6", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fbc13877abccf7b7d5b81c545bb715f65f54302d5b86e8fe8eef7f310aeca05b", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:70ec7c27c91801c31a670df9bf9b3b557bafb7fb87e302c8568e500eeb06b436", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:86b045ff6bae7717bdc576971a1d054b845076f68d243151e44a06f8496d66ba", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b995f0810b15d45a83dd690a6b9d34904538cdff91c9f2ae0dd3e8c2587eb0c7", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c66d1c86aa1b507b9e8f0335dfd529141cfe7014b61656c69b593ae22142e542", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c8951ab52d39af5d153cdc075838e01ff19044d2959e5042d79c1c7d19dc2fd7", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b5d548bfeb1c80fb50457911e7b01d7d261f23c21f7622f5bb6b83dac1dcec21", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:14d5998d96583516224c213174ffaca63e60129b22adccb2e53182525eef39ab", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6706be66a868d61f7eeef7c0dd64afdf372f802c6adedcbd69efdb1d7e2df05b", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dea9f36bbb05bdc9da75097b790bb748fa718bd3c131c722c3d339847e39d3ee", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c73a8ea23fea392dfc627083e91f76be9e117684a6c1128c43bfa819cf40e2c3", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ee1fb3ba8e9c037902e2a0b2e686f977c9bd4bbb54fba5b5907a6fb4890bf03a", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c2d109ae986a96524317f5ca69f9ffa6968bd6f36e780ad91dc0d466b1179379", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7690a3a94fb0b6c075057fcf2c4ce7cecb44335a3465b4778bbf313690fd5487", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:907ab31646c5147d4f8e103af8811e588343ebcbf3e42d9b7b9c02b4143734b0", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb1f5adb6e41161034cdaadf0dfc855e50890efe9d009c561194870a676bc53b", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:49c518836ae2d0cab8875bc60c2d77d75320ff1e415e8f778476245b4d809dee", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9330c8f0e371ead1f7b6d4c48e0a7b167b136f55973f5a27c489ea356729b2b3", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f461e7f63790671f56541269d2940a87e3de8dd14de07ee40c05d5803b658f3a", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6fd2692075c4a4eb458f574fb7edd25d27e2ed8a8092ccdd83451c99bc7c6c32", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b197de64057737019ead32f8f210afc4683dd03eb8a9cb5544f4d6bf1d1b6f70", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:55fb7dbdf07d18900832380077bc9f96d37cf0b2a4f461b75b38f7e9ecdce31c", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:848f3bd1e3ac11e8dfed5c0584545901bb24a071aceedff165382afd3173f467", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:957dda1257869f8d428f57f07de117ea8283b2205eaf9d0711feee216ee5a065", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8d48c8eb2ef6e8554f009f23ba9facbb3eb6767d31abed26697125a852d28eb4", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:98d8cefa701f2c5db0e6a91a405617996acbfbb54a9e1e3100c8b2d731bbe1c4", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1145cd83749a8a60d59d6f2e224f9ef18f3984633bd93c4e00833f3e026c1c07", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:75a3bfd5b68b501d4bc3693354e9cd6a61f2523456ad944daf580cd34f3a4e3e", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c9c5c30ec118bfbcd5ee33877bd9de4adbbf3f5689e0ea10e29a821cb4c8ddb7", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:57db2d1d35cf37fc380f0f6fb0b6c34478ad12b4f3bfc30b9371d1a564bf67e5", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e26c697d5f4add951fb0703f55cae50c1e1fc94780f119a2334dd8cd1ec4a361", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3b4de3ae86c7dda4cb5ea61bb35915cf2846b731ce433bea9b4069e05ad13ea2", + "evidence_path": "evidence/20260721T142948Z-751f1c4a/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 47, + "failed": 3, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run2.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run2.json new file mode 100644 index 0000000..8c4d073 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run2.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T143037Z-d80f50bf", + "timestamp": "2026-07-21T14:30:37Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:a5905e155862cc506e45be2f1a34264cfa0326c5b350c9dd947eb0f0d24d505a", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5033c35fc6f23cb5e616446cdadf4be45fc1643265ee43ac641de5d4615789ff", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d9551b59283eaa3b34ce96de31ec8a54bded363e7d7c98a7eda8635ab8fea169", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:3bc5d2355aad7706bf28fe33357470824dc37202665e8b8d8277c540510a75dc", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:96c75a0895d5e61c88d65a1227cefa6c8fc7b6b073a4c2d13351472e26080fed", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:2842c624b83e0a6149a66f7673059a550ff096c6575bf1dc498013b3f1614ebf", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bbc5e30a86d1f2906eb38ca47f41ea1e29fa8328d314f0a307d1bffc8c922295", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:55c1095377c9429c1f5e891baf5b4aafe2e2baa0e966bb1bdf7ddb8bbfd2f8b9", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de5ff93c9e1ee24a9063fb5b753140aa8ed068c1bfa41182579c7a09ed982b68", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:74142878030b849892f978386fa7641b3afe3124069536f3f44fdc9ba206d370", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bfdd65e733b309321bca0c8766b8ca402c9d973b0cdaf7829dbfa1b3c9c08d6c", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c4cfd615f280f556bfcb5b2981987b51f8adc0d4fabae6d5cd376268f53db0ba", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:92fce535fe91f6add9a307826d133164d524a5be583072d3e7bf8b43c41036c1", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0941290dabde3cab60338264fa820b0515c00215e511dc656071268818402f7c", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:828e662790176bd88a72e91e52e31f0ff7cea8dd3450095725a54e4a32c431ea", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0a0c0a477948235e494530a16604ec23176ad07d2481dd5d29baf4da8f2d8a9e", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:65a78b3d619005b2673624b2f767a2053e221a9b02bbe0b99d4c2ba54d96a778", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a6524168b8c726fbf0a0bb31db5ca41e98957f51a34664a7648fd6e3bfdd5a05", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6ec2aeb3ec8183c02753402c0c48344a134d7cd7698e322876a443c4ad3cea01", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6e610e62187c3a8d16a676114d0e2519440f3876051226e07f240948c6fd4984", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2b82bdc4a132d0ee41bbdd9b1c64ef977bc8182fe52ee7c762aa54adf3a0d88d", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5123ce6267ab66010dcf42c0db7e8654d92bd663dfb3591205a7584159f9bfd1", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5b09c164f949b93fb8748047f5993c619ba92b36afdff47edb9f658f180c702e", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8788e3b58a878a84f69c31891a292c515ec71b757c35b2632d879f74d5f6a6e2", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:219525b3300bbf8cfb48e1b7033b26fea7953b16ed233ea84f607ba8fae2e7ad", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0001f1af5fbdd39faec3f3a02d83108bcca099208bcbaad0a4bcb4f711d96c01", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8ccd65595eb63de337ab07f2f92b76b8ab093dde72d47d625f9e8368a1376582", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d8bb42997e1810d61e6638103f2a692da47a88d616080a97d47e00d3844186fe", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bf11b4a81538c24fa04bd19b6db3b06debd20ad4ff18e12f3189f445e9502418", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:86f3a7be1814f041b4f510db87c14c23bbcfec2fe775638e4e93bdb93f6f379e", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ea628f0202ce6ffd4701821d9e826cdbd977ac66e1209b2b3aa1297d6c689373", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b1df75148353a7e4fb906d1fc7085237056fb2fa54019f67e36129cc6647a2fa", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cd06fd26e02c6b87ba3345b4fd8d2af0d4a3507f09cc16ead4c8011d01d58745", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e33ab03ec222eed629462fb3e03da7db2e5a5c556672bd55d647894e8ad3f7b3", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4a8d84b41eb89a075c11481ecd6b1653d98e8a89b83fdb2f40bef016b73bbd09", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:478b1289324172daec182ed4a4909affc7c894c3443d6cf4a93a91f91d824897", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dc3440b7c4fecfca0425b34cb0827af69f4284e19c0e54ffd37ad5bea3f6b4b8", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:67356b7f514bbab774c227203e8bb45b80b3bb87cfdc5d7be00b8f3660cf7691", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:568639d214420611304bf2d4379db4810664cb2cc3fda449e88d36ed53a50d3c", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b26c31af3b760ed42803e8a26dc32a42957d0259aea0b04236003dd24229cea2", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2c2e001494c619327e4131cb923ff8753c1b9da758790b071d019d0b50796cab", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a2a4678228eefe9ea9e76273d3852e81f80ec593736f42eda29e5f155db85430", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3158845503ec321ae6b91a74653e526cf65560cb6c7d4ce77d28f87bbf604339", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6418d3a0976a1e7146466d20bd017c24270c87f0312fbeeac7f41f3aa4fda357", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:27b400b07aa44395ada424ab90ecdf57ab1fd0f67efed40f153b8c42bd0ee9dd", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7a9a7c3e0ec20326c0b45e0d06d531fe530e3355576ca16ae0b8f28d6be4da6b", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cb5d8719c7571fee886f5c218bc3523b95115e149696180184d884a5ec509ea5", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:78f1fd945f8e56cb3e23388b5425a42da1464eec8bef8e1ea8623701dd131b27", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:10671db749d2b891a65ef4f8857682733b6d50b66779ce6e1ade7c100ed88900", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ffe63dd6f2e3e750c2d1daa4942de2d29358974028604c52d0bacaa81c92d718", + "evidence_path": "evidence/20260721T143037Z-d80f50bf/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 47, + "failed": 3, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run3.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run3.json new file mode 100644 index 0000000..fe798c8 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run3.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T143121Z-24253c84", + "timestamp": "2026-07-21T14:31:21Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:15ffaaf5c099ff32e547c1866246e6da99fd35088784a1c7d5cea2137f0d2da8", + "evidence_path": "evidence/20260721T143121Z-24253c84/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:88e1c5111e4137943037830d87a895ff6a4c3918a04017a2bfc6e51faebd2659", + "evidence_path": "evidence/20260721T143121Z-24253c84/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cce43b973bb17b40113026fadf1b486614bd3d275b1c131c347395041f90d35e", + "evidence_path": "evidence/20260721T143121Z-24253c84/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:9bd6c0a3c2efe1cb7fe12af3aaf2c7c8b685d6747d6008c97466efaad6f49b94", + "evidence_path": "evidence/20260721T143121Z-24253c84/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:653368f96b2ee13a3a96152320d52e6cddd64037f50f945c7614b9eacbf97dbf", + "evidence_path": "evidence/20260721T143121Z-24253c84/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:5033027a3f8594c8d1524527f958f9886167be7ee084ba1d18ba2e1447d6c44b", + "evidence_path": "evidence/20260721T143121Z-24253c84/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:036b59260badc1684b53507df6d46d0faabc34fcbf0871104e8aff08349a41b1", + "evidence_path": "evidence/20260721T143121Z-24253c84/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:975eafe1944104404efbf31a40334b72c69433f98f9252feb37ab0723e65798e", + "evidence_path": "evidence/20260721T143121Z-24253c84/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:94a2347973b2a1f70416960ef15f18a07e1914c059801759328530bb9626fe4a", + "evidence_path": "evidence/20260721T143121Z-24253c84/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:961997a819f16f83b1bc16f586ba69182fb43d782143ea87bc4ed83065efb663", + "evidence_path": "evidence/20260721T143121Z-24253c84/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cefbde36778fdd550250f489193936ff8ee1e36ff5d3b3bb564a75329154272f", + "evidence_path": "evidence/20260721T143121Z-24253c84/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:736f439713474f81d8c41ef1d42d96c03eae0013e663ace233c515a743a1454c", + "evidence_path": "evidence/20260721T143121Z-24253c84/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:537bcca5be8d38b900ca521193af6f64f79fff9691589cb1a6efeab6c3e3ff22", + "evidence_path": "evidence/20260721T143121Z-24253c84/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a6e2c21a99094d221fe4179b07549e8070470650166e41d82b1ad2bb4d4d2817", + "evidence_path": "evidence/20260721T143121Z-24253c84/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:994ba4199d518fdda3c5680f1bff1ed83c21725f6a48faeb7a60fd835d4c98de", + "evidence_path": "evidence/20260721T143121Z-24253c84/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f1ca463e641516f17fe7e76029299a28803d70a10fbfeb67762de3f981bd3408", + "evidence_path": "evidence/20260721T143121Z-24253c84/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9d1036dc24e22db6027b29da571b7eecf2c448d09747362de83dca0ec1e8f5a6", + "evidence_path": "evidence/20260721T143121Z-24253c84/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:09d257604059312210c035c0745801e84ba2a81d6c0c104a16741c4680998073", + "evidence_path": "evidence/20260721T143121Z-24253c84/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:46bd7a1df1cce90ba043ad120c7b0e5b7926761c3e741cc511ddfc7628d36f9f", + "evidence_path": "evidence/20260721T143121Z-24253c84/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cfa542cd77c7a29474516c02a6717da9a9994e889318530d70b1093b1b8fcdbc", + "evidence_path": "evidence/20260721T143121Z-24253c84/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:be1c9137e541170f33ee5f1d3952c59139b07fbb84068f4cfc8c126e36a4454f", + "evidence_path": "evidence/20260721T143121Z-24253c84/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:71ceff4690a4da6a4109e09bbeedd4db214dac2d728254591bc5925e618ba3a2", + "evidence_path": "evidence/20260721T143121Z-24253c84/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:50742bdfe2f1f256038ab9388a4b9966c664f1a3dca0d6e0150430311f6317a2", + "evidence_path": "evidence/20260721T143121Z-24253c84/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0f04094da2d94bfd111385593b779adca73f762db9110dd8ed3e4bbadb0bdd6f", + "evidence_path": "evidence/20260721T143121Z-24253c84/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fd0495cddbbf5a0ffe44c3d1fa00335ec8b9d0acaeab7aa4e717f20a201f6a34", + "evidence_path": "evidence/20260721T143121Z-24253c84/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7fe97b9be44eb41a225ba84bd705e031fc0c48f5916c23de73507589b05ed057", + "evidence_path": "evidence/20260721T143121Z-24253c84/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0766fd1f83e1a8fd9405d7516d560f3a28e2b4ff093a0d70fca0fc5106019f1f", + "evidence_path": "evidence/20260721T143121Z-24253c84/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0f246071a4067da6a9d4333cbb66502d762bad90751046d09e285dbaaad4321d", + "evidence_path": "evidence/20260721T143121Z-24253c84/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7e1f956b9febd1950f1969a215dd99a68d3f82620ae15cbefc402b2a582b837f", + "evidence_path": "evidence/20260721T143121Z-24253c84/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:31a6a9094ed142ba0f8886ef36c2271233a7bd9b996c32c8f42fe854c751a5d8", + "evidence_path": "evidence/20260721T143121Z-24253c84/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e657db3c199a4a412ad1282facaa9e3fe4cfb17932a51161de7a9ae670009fb6", + "evidence_path": "evidence/20260721T143121Z-24253c84/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9c0ec7fca32c72983f6db113e426c681e2af6fa1424722326fd53e16cf2fab54", + "evidence_path": "evidence/20260721T143121Z-24253c84/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7968e70f07c59a1db86b79c0b83c14555860b556ccb9849ab1d2bc604a83a234", + "evidence_path": "evidence/20260721T143121Z-24253c84/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0f2e42f86c668ebfa9b3da261509beeac4d045ed41f47544e258d2e66959fecc", + "evidence_path": "evidence/20260721T143121Z-24253c84/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:048c9efb37659ee792bc912388533b552b431109f45509b9835d0c7bb4d72342", + "evidence_path": "evidence/20260721T143121Z-24253c84/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9871603e2c257828602ce228e64d0076eb058576abf4d371396285da3044562d", + "evidence_path": "evidence/20260721T143121Z-24253c84/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6fb39690d7a3679507555d5a8e94c1cbb946a77ddb4a9aedbb417e13721cb236", + "evidence_path": "evidence/20260721T143121Z-24253c84/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:42d4fdfc19828d4a7f1ab3a7b1a877e25b322b43673e2b9d6c5d7cca4d0e3d3c", + "evidence_path": "evidence/20260721T143121Z-24253c84/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eae0def652fe9e5083f359d0a195cccb1c2ed9e5b618c3fefa2c8737ee75c15c", + "evidence_path": "evidence/20260721T143121Z-24253c84/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e5d1577ecf4b45f613de748e35bbfa49179a6dc395d2ec61c0c15e72c2e79251", + "evidence_path": "evidence/20260721T143121Z-24253c84/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e044d715f1008bf8c43fdd54b89cda2b6d7c6336c4bd14cb9090635152be03b1", + "evidence_path": "evidence/20260721T143121Z-24253c84/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:facf4382220e9969e9aa305709a2ad5293014b948487f3d7e8f8e9fad59ea69b", + "evidence_path": "evidence/20260721T143121Z-24253c84/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3d3991766532bd77c160be38df7664a1e29ffac410f4f6653a1890c78dc3de05", + "evidence_path": "evidence/20260721T143121Z-24253c84/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5c8e5308b03650f69a06892233213741dfa4fc02935fa3f181de22253a547c54", + "evidence_path": "evidence/20260721T143121Z-24253c84/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2a0f3386bb1a4faeedceff2998272324b15b43ddb209633ac9a6cf15daf4d935", + "evidence_path": "evidence/20260721T143121Z-24253c84/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c1f2dda4fcca04314d511939f051cf648d4537d3e939e869143f230cd80301ae", + "evidence_path": "evidence/20260721T143121Z-24253c84/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:92b2cabbc38f4d52e8abae0468f8b576667420ad64ba693a3bd4ad84d8583b81", + "evidence_path": "evidence/20260721T143121Z-24253c84/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:97407b48eeed7209839db80d4bf5a788d6b29fe839b415139a03c3953f800734", + "evidence_path": "evidence/20260721T143121Z-24253c84/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f5f4fe96d33341c8a75f21bc688e42d0a10dcdc942bf856c384ae360afbc2ce3", + "evidence_path": "evidence/20260721T143121Z-24253c84/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2bc16d122549530d2f16c68fc6a3c7320006c621cedfbc1628425ded532dcdf9", + "evidence_path": "evidence/20260721T143121Z-24253c84/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 47, + "failed": 3, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run4.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run4.json new file mode 100644 index 0000000..08eace0 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run4.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T143204Z-0d19e990", + "timestamp": "2026-07-21T14:32:04Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:e6c6ddbb90693e7f6d25bdb5e438e124a8d822ce70b732b43cbfb933dbef628d", + "evidence_path": "evidence/20260721T143204Z-0d19e990/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1c452c856f686a4e48ff89e312d1c7c3dad81fa1c9e85023d6734608ed5a37e1", + "evidence_path": "evidence/20260721T143204Z-0d19e990/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a68242566d47d7c48afba99966236048a0e52231fc2c0e9f7a433af088351ae4", + "evidence_path": "evidence/20260721T143204Z-0d19e990/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:b875e4acd4e71d8bcf3b29c3889bd1244784a0183e7a8e6d3fc7db9a9c3b3bb5", + "evidence_path": "evidence/20260721T143204Z-0d19e990/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:09c34c9af26bdb97b36fe28092586a9e237ce1d211a3de953067ec83a6a4e20a", + "evidence_path": "evidence/20260721T143204Z-0d19e990/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:a71451e3bf4929213244c848d4b1736e9f40c8337321c4c7c20341932b90a057", + "evidence_path": "evidence/20260721T143204Z-0d19e990/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b481a56c885d6d45ca0767cdf1bb42f32c413f72d5dee5f80b63233d98327fc2", + "evidence_path": "evidence/20260721T143204Z-0d19e990/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e9639cde011eff03d09ab570ecbc629bdceedb56231ebec70793c5ef99454014", + "evidence_path": "evidence/20260721T143204Z-0d19e990/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2b3b2b4d0d113f273902287d2f844ab704d266841faafaa7d1f339e9db611151", + "evidence_path": "evidence/20260721T143204Z-0d19e990/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3605cf4ba095170e1fc1de73948898c57b529f4e76ac5288722fc855ccd3760a", + "evidence_path": "evidence/20260721T143204Z-0d19e990/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f503c254623108955f8d9d230ff3fe9e46edc1fb5ee9294f1a446e87021188b1", + "evidence_path": "evidence/20260721T143204Z-0d19e990/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cf86ce9ce902b9391c4c92ed7f4cfdf15b76c9285c07b005163476449de73e62", + "evidence_path": "evidence/20260721T143204Z-0d19e990/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e0f848454cbac6d42565102785aa79dff26c33d7178385ca1a5d90e8cefd2523", + "evidence_path": "evidence/20260721T143204Z-0d19e990/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eab7a5fd9ab944a539ec075b193efa842cbc0fc78e6cfac0a5c48b84bf66d75d", + "evidence_path": "evidence/20260721T143204Z-0d19e990/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bdf13e46b9768247f4cbed07d377f12134831ad36e004c8abe82c765f4a79e07", + "evidence_path": "evidence/20260721T143204Z-0d19e990/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ea67cc09b83819bfdd9fa0ea868618f1272deeee42c1d23be0a2cd72be1de4d3", + "evidence_path": "evidence/20260721T143204Z-0d19e990/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4814edad465545ba5c0a26ff6c09b995c087e0d2fdac5ac14f911f4faf282368", + "evidence_path": "evidence/20260721T143204Z-0d19e990/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:46435d38a79b4cddd6f566e7af67b94c4f7832650558039661f1a5a8f5aeb4ca", + "evidence_path": "evidence/20260721T143204Z-0d19e990/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c84c0f476b8ddade5692fed5ef18a70cb4b9281ecbfe56184ae22089f3731d37", + "evidence_path": "evidence/20260721T143204Z-0d19e990/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7b4d202e0efcd32e3c8e7d60cf4fc0e6cfa718a79e0e1e120e867198ff63cb9c", + "evidence_path": "evidence/20260721T143204Z-0d19e990/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a0a9a874b4e585d12aca12a8850d7bf14f9678b7cf3c06e205e55837a8512746", + "evidence_path": "evidence/20260721T143204Z-0d19e990/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:04448c6a536833a318a4ffd76cdd1dcf3c377442d4093cbc937ea15b190ca91b", + "evidence_path": "evidence/20260721T143204Z-0d19e990/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce4f14044d59c124362573060dc7251fe6a7c9324b778516a78d4e3899b613d0", + "evidence_path": "evidence/20260721T143204Z-0d19e990/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d58aae6122d96d725c252201e00227eedc43c2ed646d6f6498bda1d383e3a5f7", + "evidence_path": "evidence/20260721T143204Z-0d19e990/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f4845156b20cde0cd13dfe0c4fe8363c213b3a863906619002a6c4380412847d", + "evidence_path": "evidence/20260721T143204Z-0d19e990/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ed82e2b7abba0ef7de5e5f9ee08c01c61ca655487c67b43d387f660a5e1c40a2", + "evidence_path": "evidence/20260721T143204Z-0d19e990/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:db6967e500213f8fa95e951cceeb563f33d1e098a72d56a1a33f84b6eb0ab4a6", + "evidence_path": "evidence/20260721T143204Z-0d19e990/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1ba1842090ff2aa12c8bda68caba8f406a4f32910ef67e4a07ca7b86e261b39b", + "evidence_path": "evidence/20260721T143204Z-0d19e990/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9853b8992ea6ff5b5d7ed9c8e7f5eb1fb6e4ab3b49a4c86b196ba6291d104d26", + "evidence_path": "evidence/20260721T143204Z-0d19e990/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:11ad18dd01f284fce589fa1f9967bb1e150f92803fc0d4f968737a84d93a1f6d", + "evidence_path": "evidence/20260721T143204Z-0d19e990/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:245963758490ba8a1164498cf33629ac239c64972d9752725c7c0f2de86d5ccb", + "evidence_path": "evidence/20260721T143204Z-0d19e990/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f1114d165f05fa0e328bdaa1f8b3683ff5e55782a57582905481c9b4a67e7153", + "evidence_path": "evidence/20260721T143204Z-0d19e990/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:154004d1b74bdfa8f26187bff0633b756dedb95188a427d4f0e3b10927faf494", + "evidence_path": "evidence/20260721T143204Z-0d19e990/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ae62d548eb5996d59af208a795abd698bc419ccbd6e427de83e247333f5cf6ae", + "evidence_path": "evidence/20260721T143204Z-0d19e990/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5a128664396e132ae8d9afb30798b5309cc96aafe55a3e0f993ff39de92287a8", + "evidence_path": "evidence/20260721T143204Z-0d19e990/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fc88495341789c73f2cdcde9f478b80223bc4dbc35ecafeb48501d6f067517fa", + "evidence_path": "evidence/20260721T143204Z-0d19e990/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:98b88a42390063c5f8cc09a4bba56c8623082f8749a4bb8b318e730837ff4496", + "evidence_path": "evidence/20260721T143204Z-0d19e990/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:40ea90464c1c3a851b9e07f3c178f97ae2862ff5c14eacb39a1d149e93aa9952", + "evidence_path": "evidence/20260721T143204Z-0d19e990/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aecdc3f3ad89292c8aef9d02a3c43e24ffe8889148b3e29a25c6a92ee39d8bff", + "evidence_path": "evidence/20260721T143204Z-0d19e990/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:21b0e092db902f1a82eef4a4f4645770a2d5c14c0b585a21d8fc2e079ac5302b", + "evidence_path": "evidence/20260721T143204Z-0d19e990/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3a946e378cecd0e2744bcff094a94941333130f13f438f326475738e73589f97", + "evidence_path": "evidence/20260721T143204Z-0d19e990/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1f87c80313b148d6551bcfea0348bfe011fae700dcb3030d0a954ee7099dfc03", + "evidence_path": "evidence/20260721T143204Z-0d19e990/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a40d17ec6d950241b449b739e2d6028097bd8b1fd67d6ab2ac07e1cb71feb9c9", + "evidence_path": "evidence/20260721T143204Z-0d19e990/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c29c22f44f72184671986a98094a38b296c0ca32ad63cd4c153c5c4068f1b11d", + "evidence_path": "evidence/20260721T143204Z-0d19e990/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:168be20a4b54563c90e7097efb34971149d2cc34146b1c140d814121b43f0bfd", + "evidence_path": "evidence/20260721T143204Z-0d19e990/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aec3e5f5260769f32f41ce0e7eb2bb474f18bd158564793dce4e8536ffc7efab", + "evidence_path": "evidence/20260721T143204Z-0d19e990/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6db80b51986aad2c3d32eb52beb55325cb5e79e532b630d5fb72736fc109bb58", + "evidence_path": "evidence/20260721T143204Z-0d19e990/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9ebd8525d57fb2a69aefbf79bcc40d06dabe2cdd72015d13e32d6928319966e8", + "evidence_path": "evidence/20260721T143204Z-0d19e990/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:65b04bf3cf298b6e9787c4d9d8fc813f7bcf0e8418e0d838d0ef5f464eaca444", + "evidence_path": "evidence/20260721T143204Z-0d19e990/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d8860e34728c84ce74b27d937df10b857f268a43aa98c3ab461aa4da79b2085e", + "evidence_path": "evidence/20260721T143204Z-0d19e990/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 47, + "failed": 3, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run5.json b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run5.json new file mode 100644 index 0000000..7c89543 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-qwen2.5-7b-instruct-q8_0-run5.json @@ -0,0 +1,485 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T143247Z-ae7ecb63", + "timestamp": "2026-07-21T14:32:47Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:daed0c16393ea47632441b456bffcb6976d262e330d4823b60fc4510e66fa484", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5663dbdcab2ff09d99d836c1f52e828c1237172e893769c4dd77ad66a31665cf", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:996aadecc866b2debcd877171d741a1c26567e98c3bba9cb5b0e79e6de033012", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:3d63422e1b5e1225da9acc653f6a1f27f576c211b51aa5b965746f50b62b8bcf", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1895ea7431f1e3b836ac7eab6951040a0bfa71794bd37d709d17946a1f1b489e", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:e526d2fe03d2eac85a8a4c66500ce5a7fbf521c94b4bcd616f2ec0b14c2d5901", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d05a95b5ed82139964f5fc69580939cdad935edfa50fcabb2d4bbe44eba200c2", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f5dd023680a1f18072e74b141ba0f39d85bf6ed76f8331f4d7017901aa41b467", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a6901ba3a348ebc345f5c4434ba16be6e0561c16aa3092eb9f0a7864d9e60189", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a3258375643b0291112608ed0a4298aa8da7121547dd2df516ab569e2352e6cf", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1d44373a988af6a4bb88384a0ed139cdb67c1c968584febb3519de353cf3c366", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bd50b8b1bd0cf04fe3e732262fafd08b9890ccf2a636f3c832745813424ad087", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b0897e96cdf962f4e8331d69ba577b19cc8e5168f447d4c477ee835fd1a7a292", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:029bf619f82d2edbedfbc9c985a5726db7a3d46bb8a66a59dd60bd16cf735925", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9bf77a1ddea25626bc479dfad9072b6321dc35337488a87a46aa947560c07dc0", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:24d7f4ca51f796bdb8d252e789328a0b8c2d5440f4d2a238909754a313434912", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ea3bd2b2e0b9e9fbe491414ef5f7e29c86e78de818d7da51c0deb67bbf19f098", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0b71de4b6a0dd09bdcf9d5502c3acbedcdbe8eb0dffd79b45a2cc7dae7c2a6bf", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:97ba210ebdf774e88cbcd1426322f490f4a335f9312661b189ba8de3f9d01d81", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce27fdd9d12706da5d098e766b3daf4fdcfd30fff32ebc8fc0ca1e0222d043ad", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fb827b74bdc6d40bac0d6110b225d20e02b585811dcdfed7f37673478701d351", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8a0c94b94406832511f4c711aec69feb3dce58347f30475f3a1c82c002e06766", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:858da340564f27defef95ffa0531b64b7dbeb74243ed8cb0224bc292c2ca435a", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:07148e7aff98631f76cf5032dc41a07c2fc847e469857bc5ba79b1498c500eb4", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ff8a27f07ce1197733fd2f95a0d392150890c47f42af713b5e6dc0de5bd1296a", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6978b2ece816a90cdaee7049d33b6f2d3cdfb7efb9d3795727c9a75bc3ad2aeb", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95e96e0d7c564792d7fbb0406d11b60c3789770bfd88ee30801e4b1237765fca", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:97e45b5771bfef15958c41fba5fc5b1a29fa079a8c69dc86b6585a31f6692f16", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8a335d502a65768943d0723edcc4d922fe2e610af62f164b86ce33d8e1aa424a", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:041e506f9f09e759c7717ac64c4c54ba9867c7b06d1d362dee5b794d36b0681a", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3888115b390aaf09e2a31b7a67edd58536716b9474d4ee00d3b5e75989bc08cf", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f5ce907074663116b1a21c7519b39689122b551552c014cdaa03cb21847e706f", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2136d16e4b1ca082668b93ec9ab07c79f8dc934e55834a430e880f3a118eaf9b", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a48cb2bb1178118796c68e223079841d5fc403abaa84634d7514007d74c2a864", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f1bf1d316cd6dafc774660bd582e2c9de2cbc93b15b7268ab6ec9b395e0ad521", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0ac7c8164c5ba5f7ff53b328fe3acd676de35b03b14e58b51ae281c91dcb6d2e", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f8d5f81f517eade055c8e7fce3778fc12347d6c482f52173e3da0417b8507edc", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0872532b31fcd3d926128a4d192424ec32372fda4e0dd7ebdf647b590f73f797", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9bd790f87edd34e0e9c536d7f13673e843e75f25ab19dc2dc266669046d8d65f", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d5cf2cb1442e0eb262a763203513a08d9a7a6944a264f2d26b71d40a6c40de84", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3ce81c80006bc446cd17d543a35fe4ec6202e0f15aa05a8fc0c999e4b1e7dc81", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0988092f4ad6be316a7eaa24ef9953e167099bf94cad9640b00ce4fae07915b1", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:64390b7402691788ba63efcc2a24775ac5bd7608cba0227a73c49b33f60a0c44", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5a5a599507b57c85d504a0d1c02c275af472961b795bbc796c64e1d001d445a0", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:243d8cf86d5b4bfcbb8da2ddb160830a407ead3e08700788b2121ac3d5b01495", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f40cda40202647139202570c8e6b14c9d7d36caf9b9f1ff0846f5eabc0a81b9d", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:50ce7c3772732a2258f7325a69debf3dffb59d8e267edb146cffc14c09ef2c35", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d3e4600b4a9239e7c588abcaec6f55827517bd0306e43673f19d176c01e7c7b3", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7bf0b588d1365ebcb3549f95ce85f83be87cae76175f3f203c6a50034b636444", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e926d1bb540d641bb01aed9859f26c76228d73fef63d5e6ca8bd854996ae8414", + "evidence_path": "evidence/20260721T143247Z-ae7ecb63/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 47, + "failed": 3, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armA/llamacpp-version.txt b/docs/case-studies/evidence/replication/m6-armA/llamacpp-version.txt new file mode 100644 index 0000000..9a47462 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armA/llamacpp-version.txt @@ -0,0 +1,2 @@ +version: 10050 (b15ca938a) +built with AppleClang 21.0.0.21000099 for Darwin arm64 diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run1.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run1.json new file mode 100644 index 0000000..df1180d --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run1.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T153648Z-bb60d3e3", + "timestamp": "2026-07-21T15:36:48Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:aef4b3605f13ee7cf2f5b638c94da87cb76c134f868649c16339205f4d91b010", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:974bb5e02f51376fe15e145ae0f77b2d25fce1a5b045457ac174cf09c777bb6c", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5a7535cfff31986cb638ddb5c96a3e1ad1402eaba4f27ecf95c4753db0e90135", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'first_value', got string", + "evidence_hash": "sha256:cb1926eb5d2d54867776f4814de03ed5b99757c50482b87e8ba5a52bc09e094e", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:0afa5b56748c11b8b8110aa1e3fd6ce78697db479aa4010eda63f510579124e0", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:fa30db4c6c5f990255100c769983e152a87cf176727b4ca80a2f760882b9d8e1", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a9067658ba8e3c5b1420fc8a55143f144daf5083987dd6b2ce20fd1a8d6e2bf7", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:faba6d9dcbf606c46d9cf3e928539c91474c9b4cda403674c11e142b65804d45", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:625ebc4c92dab42f73bf9bb6713400b221d2850776824f4ce589d44cc7d0325b", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c0e8878fc7d88659fb0d207fb48a3f00354a87ce1c51e4df61453fdf50e48d01", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a7a87cf51f068cc45b508acf626ec9d59f40516adea5a2807a8cd207d19feceb", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/negative-invalid-schema.json", + "retried": true + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f1015b33f394ba809a3a5637959f9ebe38eb5f131a7ac1cf93706a03a44ba4bc", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:156de46037fff0b8f9219a5ab09c47f39eafe5ffacec13b1a3e732941ffa2648", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e46bb2f284b73cf8366d2aff6755a6f2d7ab750d931a3669a583b1c72c53187a", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:42ea16bfa3c7606d5787bc877321f7725f9942deba2468608e4ba636950cf8ba", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:7f1030a0a096c478613b975a40e4e8cd088a21d27fa58226aada1885ad94a236", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:afc02f373e6c47ef5348aae841c509d01abda9fa0894968cb180051db0db6403", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:00d8e221769d83eabfa2739f60e4a864dcec4870019c8b808df75789f7b302bc", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:f6470b9435184845fd635ffff84445faf72474827ffd3490408476f5009edf6b", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:31de61c325df690217e87dd64a77932258ebb750922531fd5584e6428fad7b45", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:d758df90bd1dc04f144c57560fa160ab265d344a9437d0c610a1ffea5ea7b403", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:9ac76f060a4888ab8ff0cb568872d4194d8f6b930aea6a51c0b3486ea20bb162", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:487d0c5ba95f62da0d82482f1712bdc751a719923407e935acdab5a1c123befa", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:62bc06b3e12ccd7d4aa5108e5becc53b38758835eb98ef5b099ad62147ca3c07", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/single-boolean-flag.json", + "retried": true + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:908c1df53bfcca8ad28856a7625ea03560df3e83ddbecd7daf1191b9a2759ae6", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e474f9ac276ec668d59cca6e68cad8c8b242abd3e88ffbde880addcbd9267974", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6d4432e2d4bc5f3eef09d8f87553c9c8c2cb9c72006fde8964adb174f5157067", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:551f8c0566804251c37624ac476a9d31ca1b688cdda6c35496487b4e839b8be1", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb299f640dcdced64aac780170c713e148ce70f24564a0d74d88191e2af05c64", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9520511448397a28a6ee6d10cbdec527b2c4a659aabedd49454a305092cda8fb", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got null", + "evidence_hash": "sha256:d6ed62998d05ad18af769507d4dc12563809e32e50847b32fe5d3ccbed6b17b3", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fb538e4bbeecb8d52a722fb55be4ceb01e3df84742673fdcaa2892a58f422d2e", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:80193f0ef0cd0860b8b4210642b696bb777140d722490832d9ef519ad205a833", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:44699e426024b21e65abf4f789653f08c6fa48869fee03fe3add34b1e8a43d53", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0727d6ee871b8b821d836ed73220145b768acee467732fc5c47befc3995d99fb", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cf675b4f9caee72dfe688a369672f221a0c5f1895363eb5eba69321dcb3e5c66", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9c085b13acaf4810be4a77600bea35277430543d065163e76c6c1a47ad87773c", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9ed39164cc303fd85a6414795908fe9d8bb81738afc037ddec926821a17bf374", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ebee2a99f21c563315399312bffff57db4bf3ed30b0128680ee2219eeddecf7e", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b271b723cd2b3f938036818346f8d126d7c91ac344882ce19c43792a1802aebb", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:6e0ab18272c70f6bf47eeb5771160c1683b4152894cfe69f89760a31832624aa", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:ed105d6b3fca6c26f3adf01ab2605291743e2444dd29e93bab3709a3ab65c5a6", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c3b97ea436c7d1f727e56cf11e5b0098d03d1e216870ff3db29384a80b8b1ec5", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1f5c7157656895f5a2550f9422dbf4ab154414e576218c1dc7ecebef052fd825", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:3a43395117e62ba3ed8909b6ba5cba4fbe4c3d4226c75d3a28d464892294dd48", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:47b33162dc7b3ad0e3853ce04e87ed1b667f3524776cf6267a6bff6168d43ac1", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9d741e38866bac4c4e13600bfbfc6e6203123ea6bcd2ff57be5c438f28e98b27", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d80b3e7016dc9c0377dc99534a7350d9ed23bcc67178394f57f4e6a228a290f2", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dc7347def7f3e47426ed12edcd24edb5e63655aac597b72f4de81e98882654ea", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:037914b96842fbb9334cc8b69496c81f66af95e75966a9b70dd63714b578b128", + "evidence_path": "evidence/20260721T153648Z-bb60d3e3/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 25, + "failed": 18, + "errors": 7, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run2.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run2.json new file mode 100644 index 0000000..0c8dac0 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run2.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T153728Z-7ebebf64", + "timestamp": "2026-07-21T15:37:28Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:daa922bfcf76b07c2851d4990b8f8a1d7b40e90db454c0ddc662f043c8502b65", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:591f4b888dbf82e144575adcde62737f619394446ef66e529beb6e32aaf0b652", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:b46e770e6e15e15d829749affbb47369c77de47cfe6b09283d09c6651c7ae946", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'first_value', got string", + "evidence_hash": "sha256:d04c035a0c1254a03ddc062b4f2fb6637542b21f05acec9b49544544441a85aa", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:2ee6363b6ef359d9a1dde2066bcaf0c9a113378fae63aff131e95d92654e7c3a", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:f89dd419a0360c8b661551d04188e242e66ccd92fe96f0e750ccc26f1b7119b6", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:85d755d3d8b8e4e6711a91a5a0a5729aa1ce522cb9d027bd552d5e3c6da6a14d", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:2625807c49fcec5398cf8ce02aa775b3a87c6f76f27185892d80bfc19d2a36ed", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:3c94a3ff581d591fdd60a4ef31b7b0628b277c48ba928a1e469afbcbdae3549f", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:46cc5c8d19259e8ff9f3cab0a5f2e115363631821871be74e459f65493c939fc", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:27a07639a81d13038a1a5ae0461fd7367076b17c15d0b8780a25dc88c7f3a904", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/negative-invalid-schema.json", + "retried": true + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:594534c8414b18addf8d52d28c2ec001d81a92bf266e9124e42c5d5e52d75ea6", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:4ba491322f08d85108b54c76afedeae78de71cf100ae46adbedff286dd679719", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fac16d24b77a1a32f5a7844cecdd069cbee1a6149d9eed0210be5de809dece58", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:fc853e1585662e432314a6585bfeb2d4a86648b1b63d64199f4181752b6f9868", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:6b5764acf9059826e759b7103350427d6ffb0eee90520a97f9c6b9c21841d627", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:fe2b3e945e08cf90ceea3554c6d84a7b66dce75b50a7311ddb3c95ecefdb76b9", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:7ea119b3a5ff64156e68f53ac8d68ef35473fd0550705deb3fd081de960234b9", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:20f60e99ea31cc99047a4da13bc59adc73f5954ab7a8ac59c537f5956f233280", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:1962df23a543929fef0a1875b91378a822d72e2364865a2446a53b59b8dbe777", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:9f43929c064b17bd907d7e8f201b4624193a3adf7952a7b997300a46876c14a9", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:cbf4fc62af2a9b1169eb02a9e3b53a3a106e66b4e98332519a42d56b58813625", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:10eed7e448a2dcae40ad03bd09bed0ecfd0c0fcd36596a9ef9c33fff4455100f", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:485d035090d080edf829c1346945a1c69894d007026cd8de86b614d9ea916e0e", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/single-boolean-flag.json", + "retried": true + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ca83635598c6077629b7fd9ef46b542323564e8498fe161bffee01a7921c574a", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:46d4c5880153a8adcbfcb820d3bc8da70f00a30e28a7eb5fad28db86bf86527f", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9338ba14a5ff27f68793f459c9b0f8bc956982ea4b961dbde09e4babe6c254a3", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1b25a1bbf8f852eee0caa702c5a79200ada465069509cc82ba882ed1f757d4c6", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:31bd59dbd6b107e13836d75ab6ea77c8991a91a34f14b8855669dd384fa1c5d7", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:121da14e0f8508c10774986ecc74491f16816c66209c5f93796ba7b7551d5f2c", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got null", + "evidence_hash": "sha256:f0d04c94f0e730ed5f2a63d5e82c851c772e8b320c8ff06d4b6708c0deaaf6ee", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4ee3116115cbc25219fcd6f0c5119ad9be94c32fee5f9b82e9bcd74458ef15b6", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b605d8e0de233825f07e15b6b8c11cd3abc06d2181d7d58469df70b10cdf7eb0", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6ca6a65702037b0d66950e7d0f94175e0f8ec0e154ec8aff032d2e72f4c7a539", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8c13ed886065e1855b03bea116555c6b361d352dcdb289eb186b63f85e5f7854", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d9829088ca892231893a1d30055044b2e0d2b1368e067d2722cb7bdafab100e8", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4d593710e66abc568810088904f46b7f3200e1903378d968aa276be0bcd68819", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3f896a96b909d11e8f0a05da40d1e7f500d5cd9c05f1c2483b3df01f0c8d54c2", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:04389aef757f9d62694abeaa20aab84ad611e456d8d95388f6256475610384c7", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d362de724b4550a81b35a65d34e3f9deba09a9d17d27b1cfc84cfcc29b53f71c", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:eebb7163b781730b2b0eff6177556f30e82d25ad6e1cbd4f8580b74fd77efbae", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:9ea1375ef1daaa5a25e7859f3061977e20187f142e9186ea77a038ae048f2769", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e12cdca3832b4aceed9cd4f6d200a5c2eefa7953e355bd42becfd002f1ec89e9", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4fd3911a6b142c95747f479a752ddfb9b6b4dd0ea28910ebfe3a6b695d0cadb2", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:6b0350a18f5b1b1e68305530cd805bc370011ba0041ebe8f93ee7b5d6fa7e7fd", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:45f3db7ba013a1a057952884b792ba07dc2aa6c61a0961c48f3be97b882d4a76", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4c7e6660e9d47d8d129db786c3f62309018f7ffc113da5205684b16109ad41b2", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4bc6f43fc9368b0e289d543a2f660a00a747987f7a952676415622f4a9e05a9e", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6e1bf87c3b81f218e77e6cd70541dd2463ee66fb269b08db90bb6be144ffecc0", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:257f4f903432aac07dc6cd5b9b5fc0aaefed2d3e52e4792495a9f82ca755fa6d", + "evidence_path": "evidence/20260721T153728Z-7ebebf64/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 25, + "failed": 18, + "errors": 7, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run3.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run3.json new file mode 100644 index 0000000..c989402 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run3.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T153805Z-7f4e78f9", + "timestamp": "2026-07-21T15:38:05Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:d9176cc615bb6d77cbc3d9dfe81e49eff0b385124d3f5f9d456d679338b0d678", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ae380fc4a31a6ae4f357a14d104c9214266087efc149da57fb879a0de47ee296", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:885ded590ddc1e5ab7ecb28e29403033d184d1c1599046e69e14dca4acc54604", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'first_value', got string", + "evidence_hash": "sha256:719261cfce1a17d7071ab7e4198ce39e32256114d2451af9e254d6b01c13e428", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:d5221b17494cbcdbb8e8f3ba247c0d9ec5cb66f78512ca53f226a0a4543c1a6f", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:f5cf6ea6cf9a01f98926652451b08eb47763ffc9a1741fc4b0e52ac104c296a4", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4feaad4b4dba65fb16a6762e1162a4a8f59ebc21fc6a733fcbbdde68bdf5d269", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:4c18ce9c55f64a124c7e09b80d38a22cfe23506a4914fb17432169bacd8ec04c", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:6f9bab7b66aa3e5c2b31a4f2bcb562eefe87c9342984cce80986a562a9c16678", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:edba1b9e816dd8fa13e215ebe0f2e2fb20fd2249d6df65004901709d431d1984", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:83480d0f2e3598c5489a1b8d23d1228888d3640c4b56ec5e57db0d1feb5cc8b8", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/negative-invalid-schema.json", + "retried": true + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d0ef7db3a0de6b0ed54c3663ed8f5812444ed23e909459b4f972bb011526a7b3", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:18b4443f41ca44f1a7c6c4b0a7bf0287ad43e29f17c40e17dcd844c6fbeba284", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:15a54a4dbb4d08c79d5ab96e9fe8a37d498bb4179bd3563d8d085baf78f946ef", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:7d51a297287e7a294c762230ea974f5d96b3bfb74f85be2fdd9e6ab24d51cadf", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:c6d327fa463f0a25eb3053b26ad20f51fa6f9fb7d562dda355f25990f166af13", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:08360612d1ffeb63b59e7022c6391497c4550b55021cba3bf04f24f9520e2aa0", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:9e118ae287c7ea50ab6064ca1b38b792ac1ade48cf18896c0632a2a735469246", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:005d32478d4109d185de9a892023746e1de4d4820b6704fb9bfb052c818b7fb6", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:6836238d526b1e79169617b51b0f2bf2ef6df851b49133729d1953d6142dc912", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:20c7aa73929f1cc44cd8d6d42606a7cd12b3e7c4bc1b31210cbbaee7e439c1dd", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:00f531b22c6614ca59b2b64b52510d22a881a620586327badb8bc9ed65b17050", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:98ebc1c7734af8b327e04fadd75f3da5ae54ef92175bac7317072560730c564b", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:59a74174c291eaab51bf84e006c54e2c1c1c8a5510f5eccbf0af1624bb48ef86", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/single-boolean-flag.json", + "retried": true + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a78d352807fe0cc1c8f3f6a21f6a8106c6a85b26c48ba2184900d741073ce404", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cb579355a3298a55fe5caefb00025ba49b1c4791b5791a1f7d083d9b16abe007", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce2cbf541586695e99da5668844f455e5a1481f6acc75902339cef5c1c992b2c", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4e243131d8a46957c923a7ccd7a1d748033119f237338ac87cf98fb8d17d1fb5", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9484c01ae09b4b172d4d153ede404b81c81697a291e201430b9da122801f93c4", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3cafeb39c5b22cdeff47a22bc815073ec5ba0ef19a6478c93df0bd9cfa2b6d60", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got null", + "evidence_hash": "sha256:99b3f2a17c7f1c97ff527ec6a16875c3259ccc56a7f0fd9a7d49f4db57742d4f", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:16662195eb8acf728ef91b8989364c937d955691e01c069f0402b7ed3f2e965a", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e46946ce0ae0ea1cc1af9167e5f5bbb15be4ea1a2dc7062a9d40e27a96983289", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6097853421655a43532a5f30f96a2ff0ca9cd8789ce302baf4a50ae0af1f1504", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:06858ad0ee740b6f83d1f2e8fd5aa03cabd8eb9fbcb1a1daa92e681642797e4b", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f6483e75ee95e16243a4ddd026514c798c6e153f026f5f903a66535236688719", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ef940c5615ac142e80f3d3f3906749ba21d6dc9cbe1fff460fe4fbc92ea933ce", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:91a1a5e6961ac1d31d127200965952bb15c898e231a313ec45a074a06673a320", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c96121c5e4b063c49e24766991462ea3594c3bea2c83038e675ad6dbfacdc979", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7ba56e7bcc264db8d42d0f69c71fc36899367f04f9058b8e0ed0b337b8f2abc6", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:77cb714143a845ce027ad2d3e5c6ad12967ca1953235b3b38a56036b53ca6bac", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:34c279ccbe6449d7dee8e66eedc02eb8f787a8822e40b39f2d61eba45c378a5e", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:70c7edacd3ab8693169aa55dc1b969d102666d35994c0d2b06fe90b97208817d", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2ac341cac0932b960096e59aaab7e588070f358fd9e0ce7fdc0874fe19536755", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:feb065faaa3ea324138def6399375eb2b0a9d48bebd3e1d321a863eab6b7fbce", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1b58e38b258f1cea71fe08f96eeb92b9607cd3f7430a478f224bf3c59920962e", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:127aad266db92ec131dabe68b9395a6f0c594074386ae84f2c61d2e86046e720", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cea0fc711a27a9ffdbdc911e1f2e540b37e3e76d1e48a5d4c32f27d3cff2346f", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c5ebf62367a9f6f7fbe09e740a4a5bd7730521f5ec7f55c00fcab870146f3b05", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:503db4a83cd4b4c24439204a229e2a71a9a2fb04b035f452c0ee470c9f07b7e4", + "evidence_path": "evidence/20260721T153805Z-7f4e78f9/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 25, + "failed": 18, + "errors": 7, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run4.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run4.json new file mode 100644 index 0000000..fa0b171 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run4.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T153841Z-eecf96e8", + "timestamp": "2026-07-21T15:38:41Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:284e66b3a7031577951f29b0fc4fb634262a924d51ea028e05099b05079ea679", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4746acccabd1574fafc9af879bbf1cc82feb103c3fdaea49194843a6a3bee167", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:950de404bbbc250d405a309fe271b4d35a48bc1d5e3edf07642f9c5efdd9fec9", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'first_value', got string", + "evidence_hash": "sha256:10217bfc5c8b725a04aa851444e732a2217a17a8c0a6466971d961871f472ffe", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:7ec13aae17e8f34012949329fab9933a86159dd868c669432fdcba54a3abcca9", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:e6dc2a43dfa09f4d5fcb8f9d52fac1f5c3d6cc58309336ca169aa8de526a33e7", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4fe5ba1af3476d4a18b2758dd5c1fa4493f0a0d0258b0b7fbbf2f1037e98e5ac", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:2359aaf042c1401009f5d142286c9d71c227f556eb85dead08e0ba3e8ac910f1", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:62bb7fa4241bb636a6b3edeedda0ff8c2148cdda5ea7678e9c42ae81603255aa", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6067c9374b1d028239a88ae47a4f9d56f95825d137639d703ae1deb97eba8028", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5b87a7aa87ac8f9a6c00f31b7d1e1423726c44389bf5a5954f9239edc30ca045", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/negative-invalid-schema.json", + "retried": true + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5972a485cab4f6bd9ee3c96c0f68a5ec89eb8d64ec3001585077d42cc5d4aa0b", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:273e78230c268d363517a2674a2b4fbe8d99179beaba9991de89caf136934a6c", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e6cccfc8dbcc642b841563d3e158bdfa6be39972f5217912a9c202f436a8dd4b", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:bea6b9778f89a429a0c8d35ad3c0c1f5a4755d0c7aa9ed16d7c8ed0eb7509ac0", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:36f45624ff94fec1d10b740308908e53e42fb7887db371e2d89432af7916ea9b", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:b3df3219c5761b8c72913f3b5ced065d2f54d8c6f463b3996c545b418cc1e834", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:259decc60883306536bcd039afad8feb2753bbc7bdfb85b52cf8538f1b0ff420", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:ba58922fc7b23b3017b6d194a0b8cf1b1329e0ace607e79f8d49a4bb8c3cac99", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:866ecde37bd18b5837a9a02d36da16b3c7b42b3de00df9dc3a68db8ac50b1647", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:9b51c2e86bf16924744f1f018943bb43739389d439d5d98676ae287747ea0511", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5a2fb371b70d74c602ef1bfce45c569069e2a9c450815138a9ff6f1b4c235498", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:1f18655e556c630a547321e4c4924b4a6b06fcfdfa4bcb8c7522d74b50060eec", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:b07d9e14ffd122c3112b2ac37dcf0e7d5753c104b1780fd7bdd911a70c4ca9d9", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/single-boolean-flag.json", + "retried": true + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:04ddcd2a71ae2bb1213d2f2f0b0612332a4d35794e166b7ed35b0d12889a7af2", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5e4b21da97c990b76921da0ac7c5b7d98c250fc96090924f68091538fba47f77", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8c7639690d77270747c0c5663ce6d4c32f974b7f47b81f2ee22e09ef860c1ec2", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:693fc6d73a760ddb31695de9ec58e01b446a832ee40735075148c132ace3d184", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:47f48fe582388869cb1c72167bb26c79052f6193d1fe5d053a07c197dc75768d", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b57210f964105953d8efdab9244ce0e25a5217f099ff69bafae28226a1be74f6", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got null", + "evidence_hash": "sha256:b677321b9611cb31c7e323b04248ba112ec5559afad530cd26fea0a4ad3046e7", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:38f864ec2d4e2369d8f04fda0e6e691e924ab36fe5b37bd81d4f4b3989f9172f", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0d3c983a58be01b1a72a86a5188a7232a327e4ff24911fbb729bf4b102931b50", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:154f302115608f47bc2a540f2c8c0bf5e4cd2ae321c6d453899d0fa2a0e937c9", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9a8d07e25ab17dbaf1d5f3e895a84434f8f72d32f2aa0c1f8584474bf629e2aa", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1aa277e6f394043ed9ad7b45fa756fadcb9864472642bfd9bc4a52ea7d8679fc", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b0338f7a20517e017e9b674e0913ca15ea12ff393e606dc97bab8ae5e362fe83", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dfd10482a95e077d27b97976cee8eb06e660daacdecdf4d5318a7a152f6d07b4", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5a5a6bc89e9f65b2ee45bb2dfc6f81fc5b13201004c0f814a6e82aef26b3b211", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0867f882ce025113613e24f1c713299ddebacb8b61658c87292827048758f797", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:e8128242cd2bf87da21819ea2a53a096a74d01fb0d81e90072ab50b3c58815ce", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:6b0f341806b7fc5e0612a113f136379c4a4ed225b604e7d83c76f9e42363a866", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cac277c53d482ae0e3fe1e8abc365a3fbb37ea3fdc5ad4ca10e5841a4d8fe936", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0cc8c40ca84fd162afb47b0c6ec65721f01cce1a0fa19d147b31bd5a9aa65898", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:5bc5ff96380bca61ff8164e17d24fedaf161f29cd4b88ab59945c5da32777394", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b4e4e30907dbc90ad53a1d4c968645a474b2492a9d96a9820666eaf87fd805de", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e2b4a2b40da64acf5d7f80d02859d93d7aa43e52f61ab1939ae7b6784e9afae4", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:af9f7cea79ecc6d5bf75f93af899d04b859e35af860f334e94fc320fdca91d99", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f550616ee15c3dbfac9a73b94d660c4d8589de823e3720fafaeef1d3ca635aa6", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cf0f0888d5e715c979873527c51122d724a2d63c5d232200572fa9c01efa3d17", + "evidence_path": "evidence/20260721T153841Z-eecf96e8/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 25, + "failed": 18, + "errors": 7, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run5.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run5.json new file mode 100644 index 0000000..ae0c15a --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-run5.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T153917Z-04ae704b", + "timestamp": "2026-07-21T15:39:17Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4ad8ef81996d94c8af727bd43a5940d270dc736d49c77b4a3f145722568e5fda", + "evidence_path": "evidence/20260721T153917Z-04ae704b/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:46c6dce26a8fa2c714ec415a3de29fe017610286d970b51a550225109824e518", + "evidence_path": "evidence/20260721T153917Z-04ae704b/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:add378a11bae3942379800c0c63b846be67571963cd6d2f7ceeb807647ae8616", + "evidence_path": "evidence/20260721T153917Z-04ae704b/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'first_value', got string", + "evidence_hash": "sha256:2239519d6f28bd00614ba773369e5b5b0d1b429aad6e4299e93550da69926bf5", + "evidence_path": "evidence/20260721T153917Z-04ae704b/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:e0ed0928c9f52619b2a39ccd06e4f7848d304d190e2c546e107e5caa85a5e797", + "evidence_path": "evidence/20260721T153917Z-04ae704b/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:7987051b2489ae8dbace68ad123b30b0dba4923c5d5da1e88dcf2db0fa08bf52", + "evidence_path": "evidence/20260721T153917Z-04ae704b/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f6fa6913d1322a829a09b21382c4b09d5d43a72d5833e6fdf5abe2b2df42551f", + "evidence_path": "evidence/20260721T153917Z-04ae704b/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:55ce83aeb6c0873eef27ce7a6fd1e8b4aca418724e7e24c8e4f7fc4ac8653129", + "evidence_path": "evidence/20260721T153917Z-04ae704b/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:bbfb46f54eb60743381248acd47a746cd3f5c7c6d65baf0a71ddd5762ffc417c", + "evidence_path": "evidence/20260721T153917Z-04ae704b/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a2ba5bbd64af71c760299440ec92a2b035471f07a203dafc7d141b6e59d4f3f9", + "evidence_path": "evidence/20260721T153917Z-04ae704b/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:2ad101c996224e95ae9924be517a09baaadcda51c5bb1f7954be988be2894549", + "evidence_path": "evidence/20260721T153917Z-04ae704b/negative-invalid-schema.json", + "retried": true + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:39cf08e667d244c33a7fe36583b659a70abc6ea325f7731d8c82f7cdc70aae82", + "evidence_path": "evidence/20260721T153917Z-04ae704b/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:cbcca1413aa573cc30be996ea41ed0de112dce2c1044b39539ea462e23f3e5da", + "evidence_path": "evidence/20260721T153917Z-04ae704b/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d625ac8c2d2d122e919ed85dffb73c67b855a83cc0b66e36927080b795d4058b", + "evidence_path": "evidence/20260721T153917Z-04ae704b/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:99a8bbaa4d63e3e6d46df34f2b0496571f131c9d43a83706931cff43aa71c226", + "evidence_path": "evidence/20260721T153917Z-04ae704b/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:09e3a29dc0005e8b7fa4afec4fd1b2a15506ac2f2df49967b414d3ff8785456b", + "evidence_path": "evidence/20260721T153917Z-04ae704b/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:c91e57fbad6eb38ee13603295251675d1bc8b12a48dfcf41dc17176475af57b8", + "evidence_path": "evidence/20260721T153917Z-04ae704b/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:a60eca05d06d6945eba4d75b2d2121ee6289f5956c46ae73c45f05ac97813075", + "evidence_path": "evidence/20260721T153917Z-04ae704b/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:a1ceff7745fd9e7f620965e45dc851b504d13f898f8bfd280a6b50521526ba03", + "evidence_path": "evidence/20260721T153917Z-04ae704b/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:28fccffa68fa83f9eeb32fc4975747ef450fcb68288531545d9e1877c02f1113", + "evidence_path": "evidence/20260721T153917Z-04ae704b/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:8d8ae2dd7bf23ce298acd6a31ff03191fb034a04a939d930a320c43b82958907", + "evidence_path": "evidence/20260721T153917Z-04ae704b/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:9783cd523b8e0a1c9b199234d55dc26f30a9543d061c4e2e2b5cd7590e21d616", + "evidence_path": "evidence/20260721T153917Z-04ae704b/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:9e6ceb8f93aad6f474a3dc3234c2c1dd8f483980a7221377b6cb13e890b67bab", + "evidence_path": "evidence/20260721T153917Z-04ae704b/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f546be3f9213ff5aae19b589c0f710a0a55e287167b37a1de87456866ddec854", + "evidence_path": "evidence/20260721T153917Z-04ae704b/single-boolean-flag.json", + "retried": true + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:84de9f2abb74822f66d84867adde8d142933d85850cae1bfc1fed7149a1304d1", + "evidence_path": "evidence/20260721T153917Z-04ae704b/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e3d41000cb9981e101003c42964c44663731cbc4d6a0fd1ebcb53d97767ead97", + "evidence_path": "evidence/20260721T153917Z-04ae704b/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ad3c5a671bbb90f4eeb81cdc198f29fc1205cbe17c730dd5bb92912c136d46db", + "evidence_path": "evidence/20260721T153917Z-04ae704b/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2fcbba910b1550c11dd4bb70b8459993a31afe4665aabce14485c98aad2e9e36", + "evidence_path": "evidence/20260721T153917Z-04ae704b/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:10f4a98f0d88a2f0467696291df2e0e11e904930fcdd5f197ceaef9c849e3146", + "evidence_path": "evidence/20260721T153917Z-04ae704b/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3be747620b60934a6ca0804c48871e42c7c8598ebe5613127083ffdae72f5094", + "evidence_path": "evidence/20260721T153917Z-04ae704b/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got null", + "evidence_hash": "sha256:7a20106ed512fa6e665ad7fdd7835b470811bd6e25290675947cc238adac9a04", + "evidence_path": "evidence/20260721T153917Z-04ae704b/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f7ba59840ff3496f366fd97ade6aab511cb94e0dba07ba49393d70c805230670", + "evidence_path": "evidence/20260721T153917Z-04ae704b/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:96f6171052e4538d6890166ae7aad1162a89a27c5e429b7a343ca6373ddf3df3", + "evidence_path": "evidence/20260721T153917Z-04ae704b/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:75513a20de5bfc7d0385db9ce0b058110e53cc291f7bfe6f1c957210d4c51cc0", + "evidence_path": "evidence/20260721T153917Z-04ae704b/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3d0fe9c040f26873db4136a6437d29a2c00921a5d230b93c718a88fb35ac1a7f", + "evidence_path": "evidence/20260721T153917Z-04ae704b/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b18c96f04c52cca316671ca8b43ba6fa0fa16d5f58ae8516c3a019e07d271731", + "evidence_path": "evidence/20260721T153917Z-04ae704b/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1c66484a8f8778c7409a98f21b6c3702e4c6505343ec9772ca3cd55f9c82215a", + "evidence_path": "evidence/20260721T153917Z-04ae704b/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5ec2af736dc27c16ce7159e207ec82262fee16336c3afd07e5898b40036058e1", + "evidence_path": "evidence/20260721T153917Z-04ae704b/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:af584c62dc94772d6321b0d829adbe6f38798caae79c21f430b42f4cf30529a3", + "evidence_path": "evidence/20260721T153917Z-04ae704b/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b8cb5995a511862a70056e011c8ae32450e5521019f3740ee2adf2dc45313d68", + "evidence_path": "evidence/20260721T153917Z-04ae704b/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:8133f1f28a92ce2b017c30959866b2cb3ff5388e951ce6ed6b9e66153fdcb15b", + "evidence_path": "evidence/20260721T153917Z-04ae704b/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:a73cba06e18b8eea9a267485d52ede03b4767ee839f857f649ec91b5d4c3015f", + "evidence_path": "evidence/20260721T153917Z-04ae704b/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cc9da9ff14b4356646f1b2d90fa5dd692f6b5a02f8e805f9923a8bb40c4b60a6", + "evidence_path": "evidence/20260721T153917Z-04ae704b/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a5dc8b6c2a59f59d528114b2f658e9a58f3538341fef46481bfd9857d9037f40", + "evidence_path": "evidence/20260721T153917Z-04ae704b/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:46b20fc508b9ce11d5059ad8bdc696346e0f3ea0fd149b976ac0ff45c57873ae", + "evidence_path": "evidence/20260721T153917Z-04ae704b/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:79e7204aaaad4059a7ce4fad49d66fd727867bb31d82444e7cfe55a05d1c8f60", + "evidence_path": "evidence/20260721T153917Z-04ae704b/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d965aa51b1f384b0b84072a1a903358cc87732b49c38f1dfc2c261364308734a", + "evidence_path": "evidence/20260721T153917Z-04ae704b/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fba13c5887c286436136979c5973b4283f7dbd9e6182e48ecdf1a0bac2456574", + "evidence_path": "evidence/20260721T153917Z-04ae704b/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:527f7252e7a10d685e810f02e26601896023134bcf40758307157458a20e9ece", + "evidence_path": "evidence/20260721T153917Z-04ae704b/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e94424ddb5e34671b12965080ecb7c798fcf73a5d3945ba3d540e9af3cfa470", + "evidence_path": "evidence/20260721T153917Z-04ae704b/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 25, + "failed": 18, + "errors": 7, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run1.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run1.json new file mode 100644 index 0000000..4205af6 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run1.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T153343Z-b9ca7df5", + "timestamp": "2026-07-21T15:33:43Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:cf2adba99879fad77220e2038f42600eb007475dd60affa314047a386018c416", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:3f25230d11352b3a605f3b219512b78eee699101c2306f42a8d55fd517a82e6a", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:b50b4aaf6a648b2fadb953872a04facc3628dfd65a9a185cc65b45a6f10a1152", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:fabfdfce84b466680dc9d0e93700be4aae3a876f6317abb2aff3011322975546", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:6ec66a6caab31e17801c0e9e623018683991058a172ba55d3f63078239e46ce9", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:b329f96b38c4df07a0b4882ba6b787faac46fd178e5d4326ef1d1c04dde9daf9", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:b7420795eebcfc0ced58198201a9e0bbb21d24b843adc03a2ae791540f4f89ce", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:0277ba23abeaf48fed74794e48eb1d63258075a0b1095cf135efcd3a5d29da43", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:6109a6738aaad087b41eacde0a6751953bdc0c21c032e92df5992ba38a971441", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:46b3830d4cede8bd2d2e11cf89da43984c5df9a79389fc48f3ba8d267de8be5f", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:dd6515976dca88ab2e66951f979c7bb235b187c62bf5446b7cc092dc3a85109a", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:38533af86af0be58bed93badac6bf6c623d433bca3daf53bdb76437ae43c4cda", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:2d3d87d8c36a8dae69e628d7fe425cce402bfedf806056ff3f907d6f85546f14", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3fb7e4520ee492713befc61f4ab3f65189e3bc8923987a6456c8ffe27087b9f3", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:b3e08d899475855bbcce06fec900d8d47e60d5b5bb367b9899cd6f6d8f300484", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:0f2cfe14588160f2414a57ecb214802af28a54002a7366ffcb5748b7811ed416", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:769b45988e14c7eb2145d29a555f81fe16c4dd4e1a95ed6b44976173677c8e43", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:9fb4ce889be7113ad4391104d5dca9cbef4242439e8e3736615a82132b457a4a", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:08eadea0e8af8656322a1459875b36031ae2319b423e470490dd1bd41fcf9e37", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:a1c44ad22e01258b2932e767f38dfdf80992dcae57506fa42671b275b1deecb9", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:b244c9448abc5e19a27a6f7965748f35240dd9e3af0586e9ff3e1db048e12fba", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:19b5fb6039da647c51e2604a805309aeb6c22448b5a3b29c08fb339127414257", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:0c0f53d2924cf6b86d23051b1baffa65467cf720b2adfdc33fbebb7274c74586", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:1a5226c79480fe9d9a7ce1dd8a97929f1c26afd169e71063bda4bc9368b43690", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b9da73bfc9d99349b96c082bcaf15748bceb81d37ec718251dd12bd134e9b499", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:086c36b732fb68953aad3cb4bfd00f909b82b69e0c77f21332e20bd846c8941b", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:36db51fa5054f472cfcd961cb51ee8afd2cd07df1af32c859c2b05c51eb9126d", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6c62f3f178f416f22715a1b467135f87c635135e9df243f84cad395a18b9f07a", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:3eae9fa47752fa7d5d92dba4ca5e675191e0e0f6be8ac81e0100fee72d70ad4e", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:100e8d8c9cf939d9719ac07d72afc09b76be1aacd85d8816e26613daf035bc1d", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:1af0d4abe48506bd99ecd2f9666c6e40ac30978d5747ba6213eb8a62c818f6f0", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:5a7cf966fe0853bc3411976087340e9fefe8d23cdd5d8804b498630de48949f3", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:5736cc8a4eb4e159bc87eb225bb51d3f5f1162a6a270d55e709b59f3ac409bb8", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:33f8bcc7b3b90dce3a7478942bb8b4a751b2a4a7c4e2a7bcaa9bd08d822a296c", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b0612567d7f5e65e1374f1dccde55c34433c93b655841b33fee229fba2caad8e", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:4359a7ae37756823539419a21b304272c2882aa88668979d6014d80d40297664", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:3c67a66ed499ce745a34db9697ed99253558bcfcbe8d68fb72d1aff7747cb5f6", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f097d245e2ade6613c2d3f924d6a0673092a47ab35f4d0a3fe9e965962cb6358", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c9a89bfe7c797c01ac31731bf8c356699da9ecf56b8f58ba2e5a15b57aa87416", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:289e444a04b2c161559d698786d2e9f144582d2b4e0a417014fb00ae017190bc", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:7c533fd8c184d969bf85aaa7bef22c0d7d3e2cb1f3ff903b5b952fdc9d4f3871", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:53d24b478d476781bf7f9894da3c77dbf1450ad0f0e45501ec722608bb3fbc89", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:62f6c77661eb59d92403794ade769dd284d400703422c562a4962a49f6298cf9", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e7ae388d34abed089efba6d62588c45eda770c3fe59c893dca69beb72cd87a73", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:58be163088891786ea256290024d098abd1c25513cd4402945ba4ac612ca1ba9", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5c68586a1a8ebc961ae9e9eaeba90584b765547e6d42596caf1db65887acb723", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9c7ac4026c26dc844e8bcf6a2cde1d8902f5871661e277510b6a1b801073d112", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a14d4d7f647018e94c7ad8e3abce1c2744f2fe282b0bd3c669d47d360f4395c8", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:683932ac650fb2760d342d9e0e84bb8de72eb98af7db736de46e6d47cda905e5", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:efd4914accbc9d4362a60c62f4d83564ab536a65f170c0b09f91748c4dcf9208", + "evidence_path": "evidence/20260721T153343Z-b9ca7df5/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 21, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run2.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run2.json new file mode 100644 index 0000000..757853d --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run2.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T153418Z-b09ca6cc", + "timestamp": "2026-07-21T15:34:18Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:0097880315c116e5ab8ed0d70fede45c4f4ce13e795c390fda86386b30c7f655", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a9857fe424bc8b7f55a4e7e421a015c63099b003249ee180d4dd509fb4da3b6b", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:759ec6020de6fe45b1452fbce71e6cd14c085539671aa8923060bb547f60027b", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:607955498801b33270e3243853e25a25f032e32c5e9b7883247cabc727fb94ed", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:adea1b479f358782beb56cc10970ddfdb4f1c03438da0385dda1c48e4c67fed0", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:c7ac84cb218a44e5562dc6e26de7b2016ceaea34a295e9630c9dc3955aba7837", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:8a44374c6af5dd7f13623669215e429d08e48a7f9e4958725f7eae71caa23ed3", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:156309e1302db01531a16a06889deda2586be68374173f52a37d842c5688f67b", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:02c217ef7a4ee0f640d28c2072656119497fa41663b46a9c5321cdc6f2770977", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:259f3753e739dfb29905e02b8f5bbbb1e9ace55d1d09fa92bfa5f004a2b3d2e9", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:3b80703b11ae4a3992dec0eceaa64e88baa96d29a7f90f9f8f84fb18a79709c8", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c06b63099a9c35ae3cc8624bca07b1fa204923b146b14c171ac79103e6bf45f7", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f476302608312cc4b3ed6322ad33a592c805f1eb9f603bab26c461f7b3cb5ba6", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9810a9eba57ca82c2a42624e43055abcbd536cb4b54483670226a15b7fc83213", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:bc25751af7a4010874bf5ef599fe3046a97d20e28080e3ad1f70b145f8699b51", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:27889e3eb474f16ba336dd4607fff74ffa178a8eee558cde063d8df4b315bf29", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:66d3c26964968ccd199608804e8c87728960b8055faddb79cfef0dabc2d9014f", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:636f5460c5e800d022a3d8e3c37da5f053ea183a1a28da7afcf03a6c2dcac4a7", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:2e9206883af59b5ce5151ff080ad2f50ae5720144aa450dcd49cb327d3552cd2", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:8d88845be0307ce8e5da2a48a1033f1c6dae402029422fdeaae4612376ab4259", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:6aa7ca9de9dc0884bb9aeb6bda3a8917bfa06140b732d58029712b49d640f007", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:145cc287b9cbbcc6e9715bafef2c512f198fc6426cb15ebe426a28a87595d199", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:edb2fb6ed53663a745f5a75422c97617d7a7687eb663b1667a6ff40148a602de", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:df056cf5488cb9ac7f75116a7ade213b6e25352a28514a41142b2f53813e6b21", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:74da784251646033c6a309cf5ecfb8e3743ed833c480b005943d77655e939eb3", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:71aec55f1de6763eaccdd4426293c12a3b98af05f929e3422991149375a307fb", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e50be7da08a5623a4b28cc79297eb5f9091e34d7f30b931a2443786749b4f473", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ad22a94e60593a4c4e05d9f3bfbb6abf6fb972925d81cd81064a41ac5e753f25", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:dcec11ae910acd8b39268b805c5fd0e15e3ca3db0465871d03bb4534a8b65ec9", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1bc10b411aa9f850fff71340512e7a4946a341637513ddc13e891424dbfc1d27", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:e99e95bf3aabcdf989c1697ae432e3898f25062722179b7df3f389e436086791", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:6477ba24017fa2e9877073cb21ed79c88d4902529101fa7a0b90ded0a24c5c16", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:158736aa5bf946b16ad39808c51483a47ef72bda9cf2efc8421f7bab14005b4d", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7ac16fe852f613f133704bd1e9a805e46e4116f48dacc39af8515ef41742b642", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7be26c88acd1b8e2854e6f34d3707a055b6a9aa7464d6ef92ca0e3e3464b1620", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:137a3fd36db742b6c27d10f8e241cde6700d98c2c1659f1fd3ce5e0d258a06b5", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:a969dc5998220886d6fe8e45b7e303393edd5cffbf006a93d0aa508ec19d153d", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ca5604a115be5e1251f9d3b7eee9c69301118ba7d8476add64eedf51b4ff4ce3", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:91e8fc04373c385cf5a84307cc45f0d4dbd3fbf376564ee6af541bf2a3ce11db", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7efd90529b2814178149dd028dd2edd23290a08a7d7c1cf16939363423a2990a", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:dbdee2b331f67f71d1891999c922470875f71ab5e75310719a81bd1bb09e3e6d", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:a5818cbd6f0c7a511acd6b05748e5ab41031d9fa57d1473d0fbcaf00ba67257d", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4b7d401e7ab2b9cc307c054272afe5d79375d27b48235012a766a2538bdd800c", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f2fbec225dd18b490799135b0aa1bdc0786917112dfba6ecf31779e014253757", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:72f8e4adcab0a4dae1cfdabcf9bc066546aae074da226ad6c84e75f6373642e7", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e77a3fc47f0e28b3305069a4ff4040ed9e7850d84d050f90a9234141c286bda", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7ba6c4319031badc68e3419b7fa7f9c8b73093b73370debd4c0a3c8b4ea0f321", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95722a9b92ff8b5ca203d417c2efb96d5c47bf1db0bb20c4f3ef354898463d74", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8c21049ea240ac0abc069e21e026d3ac6ba6565278102b2b7374971791304922", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:32f4e643686134b3ede3c6fa8bbd6bc5c1ca4aaddb8f2db4bd7bc004af34f421", + "evidence_path": "evidence/20260721T153418Z-b09ca6cc/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 21, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run3.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run3.json new file mode 100644 index 0000000..e0e06af --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run3.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T153450Z-9253fa51", + "timestamp": "2026-07-21T15:34:50Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4f65df1fde6917cf85c7e1122917c2a35e5b37be1aca3667c002f696183f9a60", + "evidence_path": "evidence/20260721T153450Z-9253fa51/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:997d72058f66eaf9034b1ce363861468cbf0a45d315bed9293a4642fb54611a3", + "evidence_path": "evidence/20260721T153450Z-9253fa51/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:930a87db7202d838297f48f74aade45f8d15bdfff57a41085ab0f01b7a433932", + "evidence_path": "evidence/20260721T153450Z-9253fa51/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:9e103632c03e3c672aef633c33beedd656e4646263146f4aa974c126db00caae", + "evidence_path": "evidence/20260721T153450Z-9253fa51/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:5060cb4fa979e330f09f642e12606db7f38ee8a98919460138e261c7b84b0a56", + "evidence_path": "evidence/20260721T153450Z-9253fa51/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:9e219bc35c488efe4411a52aead8e6529108df88bb8de0d822a6398a28741e10", + "evidence_path": "evidence/20260721T153450Z-9253fa51/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:fd909d8dc986e8ece17babc3362193c657a30bfe320a33e328fd73b2db490c5b", + "evidence_path": "evidence/20260721T153450Z-9253fa51/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:786bd730b0b7a6d55f72c6add5f1507912550fbc2f7cda8f244b5831b7f3c671", + "evidence_path": "evidence/20260721T153450Z-9253fa51/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:91a055017ad435928b0ad7d213cc9708cac2fee4edf37f57ccea5211b0b6adff", + "evidence_path": "evidence/20260721T153450Z-9253fa51/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c4414b0018662093789ae8e6f24c48693aa12e6f521b010467c52f2911136294", + "evidence_path": "evidence/20260721T153450Z-9253fa51/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:b9b7a33b99653c806afeccd8c86b53f6a9b3821b1d1a4599cef499b62deb1615", + "evidence_path": "evidence/20260721T153450Z-9253fa51/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c2c14513faf233962812d3ffed92415e4f796fe62a483e13e1727a65b0d6e88b", + "evidence_path": "evidence/20260721T153450Z-9253fa51/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:b0dedfa6a92010ac4e1ceaf0315bbdb1ccc9a84fe34496c404c38585e5b1ab78", + "evidence_path": "evidence/20260721T153450Z-9253fa51/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:abd33d7ab351ad3c438b04483faa7c793c978a313b072a8be467329fbea8fbe6", + "evidence_path": "evidence/20260721T153450Z-9253fa51/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:33dc41d89383e78fa41fe855a41f95fb9bab837c1ea39b73c5230ed6225e8a36", + "evidence_path": "evidence/20260721T153450Z-9253fa51/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:e0a43e08b9eab21108f263923232cd54c6368230407b197acc3d0381a968205b", + "evidence_path": "evidence/20260721T153450Z-9253fa51/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:bcd479de569eb9cf208bb1eb1145bd26230df65bce2982045074bb7b20887bbb", + "evidence_path": "evidence/20260721T153450Z-9253fa51/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:6f872267eaa3b453b6f8501a1b9cb91dd68608cc0f76c636b7b503837e2a41c9", + "evidence_path": "evidence/20260721T153450Z-9253fa51/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:246ba4520e25938036eb6586bd2ed51a1ba3cb31044a9f39e3befa8d96a68be3", + "evidence_path": "evidence/20260721T153450Z-9253fa51/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:258972994e4a4401aa9b3f04d638657cb0a7b20992200e9b1132ab6e4f304feb", + "evidence_path": "evidence/20260721T153450Z-9253fa51/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:abe5a44ed1f8d46129a3e0a71b0ac4ed774a484f1aa70b193fdd8abdf143cacd", + "evidence_path": "evidence/20260721T153450Z-9253fa51/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:14a3ab920640f7b35908c1c0478d4162c3ec4efdbc35e51b4b02827149da461d", + "evidence_path": "evidence/20260721T153450Z-9253fa51/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:3b709ac7d5a3361f5c556dd7ac7e6b5513052bd7db60859d083748e671bb9d1c", + "evidence_path": "evidence/20260721T153450Z-9253fa51/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:5f6993cc6f78716cabc066b8f94e2b56a0eafe3d822484014437f7a9dcfff303", + "evidence_path": "evidence/20260721T153450Z-9253fa51/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:81aaace51a362ab8dbe3f08cef3d513a1d9f205bd51fcbd349f327e09dc55f76", + "evidence_path": "evidence/20260721T153450Z-9253fa51/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:31a676b221fdee163900208b871fa5e5ffa658df12e082ed6bf16effc8512c03", + "evidence_path": "evidence/20260721T153450Z-9253fa51/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:56c5136a9717ab3afc8286c439f37104696d848eb10b745ac09466c31362c149", + "evidence_path": "evidence/20260721T153450Z-9253fa51/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3a6ef4960e1611fa0b72b0f01b3bed5e45a6f13349de9ceb7c9c9c88483c3034", + "evidence_path": "evidence/20260721T153450Z-9253fa51/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:276aa47fb9b521983b616d29916b1438800ad4bfaa21f0cd1a1342faf1cb9139", + "evidence_path": "evidence/20260721T153450Z-9253fa51/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e9167a84fbe40fe076fde2463a3ef317334583f245f8f1fafeba3207fa448e39", + "evidence_path": "evidence/20260721T153450Z-9253fa51/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:db1e752337996d16eff2587ee5cb3a04fd2a5cf6bd463bc1fbc92b34473d80bc", + "evidence_path": "evidence/20260721T153450Z-9253fa51/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:b5075d458f0d4055b2b211a4950a62ebdc8f6eacb9551b16e89befefbe80a0e4", + "evidence_path": "evidence/20260721T153450Z-9253fa51/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:43a4f280a7365dc5a9c51b5b37bea8b2619b227f1a22dabdb562b222cda6490b", + "evidence_path": "evidence/20260721T153450Z-9253fa51/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7ee61afbc33e35c77c530b7d136799df7e69a34549316bc817ede77d508656fc", + "evidence_path": "evidence/20260721T153450Z-9253fa51/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:642c190b8eadb8e6c410c295b7200de3e38e229ac5304b82ccddc5d945e49631", + "evidence_path": "evidence/20260721T153450Z-9253fa51/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:92f99a2f8e15693f07b7da02182dd40591176477ae808745a7173eebb61d35a4", + "evidence_path": "evidence/20260721T153450Z-9253fa51/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:84f0e280a428473cf786019bb1761f63fcfaba6119ba7adccda0582fbc7d5788", + "evidence_path": "evidence/20260721T153450Z-9253fa51/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e0c2d0fe09a000b56f5308143104a85c54ea8f7f1647296eb037a03a453afcf8", + "evidence_path": "evidence/20260721T153450Z-9253fa51/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d1fa0f686a2630819232308588e7ba4f1b40632212a3342382f403532c6b66e9", + "evidence_path": "evidence/20260721T153450Z-9253fa51/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d8ebba730b9c3a9be790e11d61a4d0c3adb61f38fa670e3c56fbe2c182179298", + "evidence_path": "evidence/20260721T153450Z-9253fa51/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:6af8c308bef4ccd68f3772e450294440ececbc41f8ee8828a91711e7271fe92c", + "evidence_path": "evidence/20260721T153450Z-9253fa51/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:7c173ddbf6606438493719d3f112809cd6db9e9a0781dab38ae563ee628e3315", + "evidence_path": "evidence/20260721T153450Z-9253fa51/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dcd70f55f5fce299055987d2ae941db947c5f3eb176e7cf2fbd09cd6b08500b4", + "evidence_path": "evidence/20260721T153450Z-9253fa51/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bba235af1b4db78d95450b5a6c7b42151e7560ea1b774353c863e750fad65d76", + "evidence_path": "evidence/20260721T153450Z-9253fa51/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:2bcab059e8949890e8043aa4f3372f09b7b0d8915a862ada0ad433d4fd36d13f", + "evidence_path": "evidence/20260721T153450Z-9253fa51/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a882d4564f2fc7d9cc4347edae592c41bda9e2e72c3f221e8efa38b5ee8075f4", + "evidence_path": "evidence/20260721T153450Z-9253fa51/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:369eca39085b3593ce2f6e0eef950ed691f488797bfc43443008d051dfda6a23", + "evidence_path": "evidence/20260721T153450Z-9253fa51/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:855c8ff4231a2b811d9ac2ee62d058435878453da745956ec84ac4fd04887278", + "evidence_path": "evidence/20260721T153450Z-9253fa51/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:07ad921ae297ae897c07b8a19397be5dc7977d9d48dfeccfd50edb332ad25dfc", + "evidence_path": "evidence/20260721T153450Z-9253fa51/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0774c06e6f443a499e6daf617a0a59372cbe65baf2270362c8e0b64071e20919", + "evidence_path": "evidence/20260721T153450Z-9253fa51/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 21, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run4.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run4.json new file mode 100644 index 0000000..008f621 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run4.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T153521Z-b3edea5e", + "timestamp": "2026-07-21T15:35:21Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:0d261c25242ac0a3e055c54251433fa94392d6c897e32b0f9e1759e9b9ce34ad", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5e90c90278bd37d816eb40d04c5d0d100a7dbc5977342e8d2aa48a59d733a4f9", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:52c21574d1587102cd1d40c6667dabe74ba630fd978103ebd1ffce5549a79d3f", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:c325839a730159b203ffe78f8ad3ca9446bcf175bca5d15bbe5ccd5bf08c0f71", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:80a7e49bd70d4d02c868aec8ec67867180c181533c12a29c944d64f922e85e61", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:e2f75f4a98d7e7b4fb5db20b44e14691e1c2fe15e8c9efbf44c4c331d380bdc1", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5120fe8c0952a2cd4600aa284005f239ad949e26737636c45d11f8b1d4437951", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5b9e9567c1074fa9ae50d00a685f8ff5c8153804c5f2a7e1f2bf8a050246185e", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:a5d9160a2becdc88e0db39aa101243965c46f492856dd8b316563ebf016f2395", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:66fc5c8d00cc246b5d3e2630fac43c390bd2c801de249c69b119ee71e7b0f153", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:5773432ae12d3367561661e1dedc1a05393a236069637c54a7047282f059ce9e", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c4567072564f0cb7a4999013957b4e7797c2ca502d3ba65f319e52564663074a", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:d97140a36a9bf6e980bc1b2493b7748ca96a6617da4553f4b6a2cf48114b473b", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb9a357fc12167b4dd43aae19ec18eff0dfe3e75089139be970d5f4527276e32", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:2ffc60d0bd760e8b5979ce05b607b943124e6b538c68dafb99c0a73786fdb5b0", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:0a3009afc25a0ea75444ca0be47aa1925ecd95c723d3106f59806218d2cfee6f", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:62243e8966bac48d78c2f3d5594f7b91cd26d8d088e820674ac0f623e451e3d9", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:ac3cb0abac641516c58511f5e8bb6a2f833e58427f271df9c66629bbbaf03cb3", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:2f0ba5f17d75593cde578e2a509799b067ed217cc8a8b8cfc2eec5ee0e2f6c9a", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:429ab7364485fc85ec56f1c825ef12f0ce65f0de3cf6e871d9732f15b4623e29", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:9e60472143a307e2f8e5ac1e11d9a74cf1c3a29b332bf05948d712f794a2c754", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:44559af13eec0ce70dbc240edaca0c9df475467efe06908c766c3d6fc1dfbff7", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:5a4dc8ad9d59cae78277128fbd82cf071c4319f649cb11e90ec6db6831e155b1", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:f7131fba9dda5cd9b70441bdf1bd0e94766b9414419729d341e2561cc0c8642e", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2c528b79bc242620e2f1499085b81bbcc084496fc38c441945e6e1d435d3d36f", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:06288e0bba996c17c079385d3b4778b6e235d5669a24a7b3753d319c7337f3d4", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f9cffe4d22d6ba113e9f0e283e51d9d10d5f904ac70ce120a5d9f83e8f26b99", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f22dc24ce1668f2f44c1e89e8ba5f1412b043c47182f92f4bcfe4e23c8644711", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:d507dbdc3da5ee732476f233bbaf9f3cd35e6e457d8eafb336375f2a4271fed7", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c60b580b656a3fdbb14b48b14948f07cbb3a5e8ce574bb4032a10edd387510c1", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:437bfb24cde1d9d08dddac414f1bdd4618da938007241b50ebf8367eec4c48e2", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:c162d922493b962f1a836c7e672524e6dcb5d91f5764b16a4616c6b089e934ad", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:009132e69b83f81d14bcc29cf59d67b1af5c558233c7a74b446aff048b002e62", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f0a1422e2213dba52b2ce3590050aa3ec450e7455f9635f036d5f24cf2408001", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8eb9967e63630bf66fe851487ae2247fbe9a9d6662a7f73f90c116d77fad6eaa", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:7ae3b9b887959d858b4a310f94c4897c5eb2d3fc71208d3792e3ea9daad3d106", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:e8393a5fb5f02a7f5cb3dc8cada7ee275191ec00c85442a66ada6ccb54b2b7fc", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:89737507d4eeb456d1a40de22f05abbf71871e1a8878a472da288fd03ab5a0fd", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6523c672875b2778987edfa376e9e07342b900520145def33d84e76098606f96", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cec12acb1e650f2bfc8e6da1c32d73182dc6796ca6ee97a324c6809ecc23c4f5", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:7ffe37a31b2b27343987388b7dbe78e08bc5f47ebdbe95c3269fa4f8a38a940a", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:e2ec7a89f8b129bfb548f3a9362a7189db6005e39f2dbb552770ef415e9a3f9c", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:35831357f41643ddbb5d0605beaa596dd83ad1b6902e14d6b956230a40814bae", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3f86137b2289155b088963b16d20bbea013055861a7915ec4836732eec61a1e4", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:2bbc5f48695676f6ed59484afd9444699e5453d287517cf3976e5e2795cf9748", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:19ea8b2f5bd7b7d837ed00572783521e16b3b2edee543e8656d75e6793327e1e", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2208d65483a654761af45ce511582cba3adc68da20a61017046f6131bc1c9929", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a7c68e5fcd7fb84528e0b531ecf0a820cf4e331ed816ffeb9b9b8ba826300f5f", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3129a8ae38622520cd8afbbf735fad8dc1bf630c8f79f4c7d447e803946ba3ac", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b61ef81dd11a848654351484228b1c0af6e2ea23e409f47d0aa7a6e6cabc9ad3", + "evidence_path": "evidence/20260721T153521Z-b3edea5e/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 21, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run5.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run5.json new file mode 100644 index 0000000..f663172 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-run5.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T153552Z-199985df", + "timestamp": "2026-07-21T15:35:52Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:e529fe829c23c4b896bf3ffd1eea1654ae21cde981ad5c637cf49725284ac7d8", + "evidence_path": "evidence/20260721T153552Z-199985df/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:2e10cd6d7b4ec5adb968b1e57724c708f5ee660c09c4ea67ad4c6a8596c4bea6", + "evidence_path": "evidence/20260721T153552Z-199985df/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a6fb5c8d6caf30f9497abb7e86a5a5cbc89ef56da53c6213efac0594499c4e07", + "evidence_path": "evidence/20260721T153552Z-199985df/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:1276e4596461e360344c75ff2ef00303b9483a23fe0bb2d2ccd184e913631394", + "evidence_path": "evidence/20260721T153552Z-199985df/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:f6c390cabe52c41dbc564cf10843d81873eefbba10a0295d5c5f7bc65fb5861f", + "evidence_path": "evidence/20260721T153552Z-199985df/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:43c07aebf6d6ddc1c256147ef7c4ca1a154e9ca5593bc5c9564054eb562f1bc1", + "evidence_path": "evidence/20260721T153552Z-199985df/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:e59923d3fa7f6ab5ebd5a610019c91b0ab3a10ec0ac5e943cf452aedcd808ff8", + "evidence_path": "evidence/20260721T153552Z-199985df/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:70cfd8bab63f59a2e84dcc305c1e914fb146a9282d8c3b345e59b2fded64e40f", + "evidence_path": "evidence/20260721T153552Z-199985df/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:04a3423de698f096c651068d986528b98634327369c86de0f0d6dc1b33767477", + "evidence_path": "evidence/20260721T153552Z-199985df/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:608ba56135aae32ec6a3f0d3b50970c8939e440181b5c4467ad494cd69912003", + "evidence_path": "evidence/20260721T153552Z-199985df/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:605449131d8f0d5aa89b4bd5c3a2a56f37b51e587b50fcdf2ebcfbb0f05b744e", + "evidence_path": "evidence/20260721T153552Z-199985df/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e02de69d9fb16050e820482c980545ce0680ea6264984c35f19c8d961de14f7c", + "evidence_path": "evidence/20260721T153552Z-199985df/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:210d0782440237e4a576f028e3a6bc501e5779abc1e395e5ac58e8706b3edf1d", + "evidence_path": "evidence/20260721T153552Z-199985df/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2478913291116c23645b908c8dbed2a034f41931d7a4a1afd493a6ead58675f3", + "evidence_path": "evidence/20260721T153552Z-199985df/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:91ff8a1d7513e131fc66683c1df637ce90a074c77583cb0c61a73b9897238b6b", + "evidence_path": "evidence/20260721T153552Z-199985df/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:439b96b327ffe564f7c517d614bec3559fdeb9dbe65fa805c7376e728731b11e", + "evidence_path": "evidence/20260721T153552Z-199985df/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:64e806a4f44773b3b273edd33ad3b63c535630dbf4d1f91df2ac03b9c1bf1658", + "evidence_path": "evidence/20260721T153552Z-199985df/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:80ebc1e892e16c9339688fcc45b75e15f182af13e15b6e12ebde566235716169", + "evidence_path": "evidence/20260721T153552Z-199985df/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:e31b5627dbee7e2d6d75b4ba4f8a715ebb1101aa9af15f99d887ba8aafcab81e", + "evidence_path": "evidence/20260721T153552Z-199985df/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:5de18a60b1987c0d8ea748dfe394d10417ecc27fdc2df3906a020c8a2f4c996d", + "evidence_path": "evidence/20260721T153552Z-199985df/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:7040855a9a1e12c037354746f5bc4ac1980ee3cf8cd07a3c8e536c3f66a008c7", + "evidence_path": "evidence/20260721T153552Z-199985df/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:da5d2d12b3df836e032214b5e28762c04d5e83beef71efeafbad86c757c5e3eb", + "evidence_path": "evidence/20260721T153552Z-199985df/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:579ae078d9e4d09eb3ad38fb58bb264345b385e750cd2e23b11e55b328c98566", + "evidence_path": "evidence/20260721T153552Z-199985df/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:07d573b465020b4c9d1f7128dccd0f04e218d19f0293f89a1b29db958ed2afcd", + "evidence_path": "evidence/20260721T153552Z-199985df/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:37366a47c40f02d6adda0042798feb4eb11afca79aa5e4fcadc62b3a965acc67", + "evidence_path": "evidence/20260721T153552Z-199985df/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:24a71550404a214b1a86c7bfe8340cd9fe61f33428225b2c7e3de10d7b87b4ba", + "evidence_path": "evidence/20260721T153552Z-199985df/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7f534957ecda228e2160e8df413aad776aafd63035c6031f4238ed383956d564", + "evidence_path": "evidence/20260721T153552Z-199985df/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:99215bc3fa517e482d1870981c6c06ceb428345473f73098f927080d11858175", + "evidence_path": "evidence/20260721T153552Z-199985df/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:dc3594d13afe796bcd4c50b41187f66d7f82e6ebd8f54035e6931c38a166a1e2", + "evidence_path": "evidence/20260721T153552Z-199985df/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1fa0fde79f00538a807ce7b4d9a54b029b644e3ce808e59aa0c91c9a7e18041e", + "evidence_path": "evidence/20260721T153552Z-199985df/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:176644126898eeaba24a47abaa71b7695421864d87204251154e1f4fd03b1014", + "evidence_path": "evidence/20260721T153552Z-199985df/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:5f133927c75b3e23cfbaf8a525212d4c37b0c776b64f176a20ba05bab4a6534d", + "evidence_path": "evidence/20260721T153552Z-199985df/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:437906e98b93f50650d2cdcd1605d643a49bdf9a19eee43e00921dff01e735fa", + "evidence_path": "evidence/20260721T153552Z-199985df/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9e9bb79522fcc611458f12ea07683559af978d64f347264f196fb320d7442caf", + "evidence_path": "evidence/20260721T153552Z-199985df/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9453c2c68484c39a7d9897f3d5fdb9b99aaf20b880b3039f7a805f5f406ebc49", + "evidence_path": "evidence/20260721T153552Z-199985df/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:cdb82736ecdb5a568f5fbb539e5e87a158215ab54df2701c014b47469407b1c3", + "evidence_path": "evidence/20260721T153552Z-199985df/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:45534edf01651f8faddb9716534472339f85d6aec3526d7aae8a560c9c02b39c", + "evidence_path": "evidence/20260721T153552Z-199985df/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6ac39f866709934e85daddcec71ff2c549cf4f6523a78a3319121120a23de5ff", + "evidence_path": "evidence/20260721T153552Z-199985df/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f49bce411fbd2cf09d456cb76053a30abc06ac4a4051a165ebe16882451b422e", + "evidence_path": "evidence/20260721T153552Z-199985df/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e504d4668f434ddffa6becf3939dfa22c99b39ad2c107a3eebb4f817a335f498", + "evidence_path": "evidence/20260721T153552Z-199985df/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:737e69bc898d57bf8bbb912733f024251e02b95097d9e041570f97998dde79bc", + "evidence_path": "evidence/20260721T153552Z-199985df/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:cab2fa9a179a253b316250e40deb8e920b8913ea5cdc480592f04103ba0a2f81", + "evidence_path": "evidence/20260721T153552Z-199985df/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f7ba9f65291a4db9b4e696bc40dea9305dd850853b820e5b2d4c80f9b60fd2f7", + "evidence_path": "evidence/20260721T153552Z-199985df/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e49678a4c3bf6d7d32554d51b173e2054e3fff6c475ce168ca5f46e23a08a1e4", + "evidence_path": "evidence/20260721T153552Z-199985df/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:ebb42eaf1a081c843f97348407067451a7b57d5be3a825504917575d46dbf1af", + "evidence_path": "evidence/20260721T153552Z-199985df/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e468071fca74e8ddbb24f9040f8a5d806ded6322f0ea91ae7464836a2ff61b85", + "evidence_path": "evidence/20260721T153552Z-199985df/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de17491df8ab712b2ec9dc814321c61af09d3d7768e04e2743b8c2e9e6fcd696", + "evidence_path": "evidence/20260721T153552Z-199985df/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:674a8acd0bc6f11c02c9c5eeea7b947fa56c7f068da804b6004fe9bbd08a04b5", + "evidence_path": "evidence/20260721T153552Z-199985df/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:343dcdb02b2713c0d1e75230a164567c3b2ede30bddff9631e45e58cf448576b", + "evidence_path": "evidence/20260721T153552Z-199985df/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:37a692d7bf9ea2dc9d417f217f81441aaccc3ab21292b5e9decf9502355511ac", + "evidence_path": "evidence/20260721T153552Z-199985df/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 21, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run1.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run1.json new file mode 100644 index 0000000..4211237 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run1.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T152942Z-0adab364", + "timestamp": "2026-07-21T15:29:42Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:0295d4fb2eb054b5d10f2da5a12cb3c78d6bcf45fd6de57a72e396751b54d519", + "evidence_path": "evidence/20260721T152942Z-0adab364/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f2087d1da5097cf72d67feb084d89d1ab5baea15d3a4cebe6072d46191b00a49", + "evidence_path": "evidence/20260721T152942Z-0adab364/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:fc8d53a1e1f1a1d61328381bed8dea0377f26f025f2ca5a2df361993132bc0a4", + "evidence_path": "evidence/20260721T152942Z-0adab364/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a7e501412a0b0ce618a1deb53579d19fa2d851cf9bebd13abd0fb07460fb88b8", + "evidence_path": "evidence/20260721T152942Z-0adab364/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:102f4f81082ee4ef9cd87956bc4d32926f78ec6dcf986176b344078af647efcf", + "evidence_path": "evidence/20260721T152942Z-0adab364/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4bd3b1b054eceede4e9adee361bcb4776df60f9a24f1a143012634e13f6ab295", + "evidence_path": "evidence/20260721T152942Z-0adab364/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:9bc34cefef83fb98ceacf23b232fff64af47142d74ae520da1d96a50b1b0513e", + "evidence_path": "evidence/20260721T152942Z-0adab364/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:57d8c36f5389799b21c80a701ffda98d51fbf636d464489175fc515ef4768a1b", + "evidence_path": "evidence/20260721T152942Z-0adab364/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:5437cac1543979f32810ff07f7bc1e80183183311197a6272bb2c7774e960243", + "evidence_path": "evidence/20260721T152942Z-0adab364/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2cc9cf0ec19454b1eaa66950b61b8c3e583ebbf5cc55281def8dcc680e846782", + "evidence_path": "evidence/20260721T152942Z-0adab364/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:46383629e8fc556df5af5268b569294ec4acad45fc4951cdae286acd6e464502", + "evidence_path": "evidence/20260721T152942Z-0adab364/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1ea5b72524117cf308f06e94f3dd26e654220fa4f9d917ce1f94678c3575ab6b", + "evidence_path": "evidence/20260721T152942Z-0adab364/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:3eeb0e4591ef912c57ac856402ee513444cfe2be19f2850e81b413b9652f6776", + "evidence_path": "evidence/20260721T152942Z-0adab364/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:040c0c119fa09dd03a672fa001d304a2690688c63fa7f678a897517457b89faa", + "evidence_path": "evidence/20260721T152942Z-0adab364/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:113e9734087e9c3e18bac83ce6741822cfac5977ac91f329e7cb7b3da169e6ec", + "evidence_path": "evidence/20260721T152942Z-0adab364/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:93502acd4b77b98bddff047c85a74e1196c3eeba26d871ba83ff094f3ca08b48", + "evidence_path": "evidence/20260721T152942Z-0adab364/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:a7cf0371e9ca2998c0ff4b7ef89e23f749189f94f5cbf96f1ffb8ce02bdce98e", + "evidence_path": "evidence/20260721T152942Z-0adab364/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:9fc9c2f14295b65c1ef0b507af2ed8f67b3f00266ef02f723509bf0bcd0a53cb", + "evidence_path": "evidence/20260721T152942Z-0adab364/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:717defa75161854d200ce2dc59cced387586ff6440b2527878093f92df53eda7", + "evidence_path": "evidence/20260721T152942Z-0adab364/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:f858a3d4c3b0512522c154cbf21c4da92706e5a1773e79d4ce8bf968b6a27ee1", + "evidence_path": "evidence/20260721T152942Z-0adab364/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:641d47ee0025ef83708d48d9dba4c0640399c88aca5a64d9d70fc06171fef0ad", + "evidence_path": "evidence/20260721T152942Z-0adab364/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ad714d6db8838a3c584f49a39fdae92db5e4e9e2d850e42fd8d283a96c994700", + "evidence_path": "evidence/20260721T152942Z-0adab364/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:49c930b10b52d21de95d215ec528222223cc7f69f38aa62a5ce9a0ed1023f83c", + "evidence_path": "evidence/20260721T152942Z-0adab364/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:b60049a7be569fe2b8c10c0bd9c894b8fefe7d3268b93e0ce111ceb9470d1583", + "evidence_path": "evidence/20260721T152942Z-0adab364/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5f45653888f3ad40fec7b8203a70c9062a59eb4eda90044b5f9647e9ca09a9ab", + "evidence_path": "evidence/20260721T152942Z-0adab364/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:98469e92bee8f16fe5f8371fc7ad26a015b7a3d5dd253cca26a461eac1228622", + "evidence_path": "evidence/20260721T152942Z-0adab364/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b5bea1903696e5ca758ebdcf3a945b257957297135a46bf367d6f0f21cc1eca9", + "evidence_path": "evidence/20260721T152942Z-0adab364/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2409b79c71a57678038188ec555a7e0a40b213956174104aae699564ae491e1c", + "evidence_path": "evidence/20260721T152942Z-0adab364/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:481a135164cbcfd44b22ff74054d6c7a9998e1f203c6dfad81886975fe34bd26", + "evidence_path": "evidence/20260721T152942Z-0adab364/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ee185c52993e72c187866c851cbfa74e50a26ab7534adae2e379de1ea3e3c799", + "evidence_path": "evidence/20260721T152942Z-0adab364/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:98323737392430b3b7791dbb696d497b7ce2e23392517be90fb9fdd18c31b8d9", + "evidence_path": "evidence/20260721T152942Z-0adab364/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:7db0031e28cd98063abfc82e40aa8d8151b4493a6c207aaab4611d767632582f", + "evidence_path": "evidence/20260721T152942Z-0adab364/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:a177b6f12213c6bf0d1829e9d1133211b667037b5a8f58aad9bea3fb18c917ab", + "evidence_path": "evidence/20260721T152942Z-0adab364/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:55e63752c21321794acaee9a79dda49de116e01739d908e55294f1cd726744fe", + "evidence_path": "evidence/20260721T152942Z-0adab364/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:97a79a3970376c42550428f6ed08174e18618ec48e55c9d319bb536f57b437de", + "evidence_path": "evidence/20260721T152942Z-0adab364/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:ee0dcaf8a64ac7094f9afc8f7baf8fa6b979bdaf6a3892c2ae3534a7381c90ac", + "evidence_path": "evidence/20260721T152942Z-0adab364/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:de4e2ddf99b7d14e508682ffe0cd91284b48abbc97b1956e2ec31242c2b75b7f", + "evidence_path": "evidence/20260721T152942Z-0adab364/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:47f81ba0b492e93dad5da837bf3929e68fd4310536c2a9c51315476d82064103", + "evidence_path": "evidence/20260721T152942Z-0adab364/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8d82297fd582cf33a5495cc0cf62e668b188023035c1c66285553d64d186c44f", + "evidence_path": "evidence/20260721T152942Z-0adab364/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c04b04a97e5954b5cbac858510e2cf81a08c23e2bd138797c7762cc71a6ccad9", + "evidence_path": "evidence/20260721T152942Z-0adab364/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:2c7ae13e3ec08d4d22074cc1392b79b3eab2f5b2d21666c712629fbc689fd6f0", + "evidence_path": "evidence/20260721T152942Z-0adab364/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:e5c7ec19ef60a1b4f63a7e5eda24729409102cb8840f28478f1be55f5e65a8ca", + "evidence_path": "evidence/20260721T152942Z-0adab364/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0b484e0662af0edaeebdcaa4774c59b3ca8b48bdd49309e6ddf96591183e4531", + "evidence_path": "evidence/20260721T152942Z-0adab364/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:52dc1730e3f66834f40e044fa132dfaef05b498956683ea6e68f818bde0dd6af", + "evidence_path": "evidence/20260721T152942Z-0adab364/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:98da89479f46c5d5a2e1928dd80c02562c01646ce183a61fd9ed19ff84612908", + "evidence_path": "evidence/20260721T152942Z-0adab364/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ffa6666d27c956dea30e4efe4dbe094469b6e8aff095ab081cbcec2a65ddbdb7", + "evidence_path": "evidence/20260721T152942Z-0adab364/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9571bc5ed7846aa59d865b8a45cd48b74eef051fc33c6519f346d0f0f69411ff", + "evidence_path": "evidence/20260721T152942Z-0adab364/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e870839e431d94873caeb6fc0f07c534718747177ac6b81e010a3c7b5369d9f5", + "evidence_path": "evidence/20260721T152942Z-0adab364/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2290aa496cfa33e20e7d482352cae049d069b99db9e6a938ba648cef93bbecfd", + "evidence_path": "evidence/20260721T152942Z-0adab364/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f3f3dfbaca6bfff79242c3cbc08a39c29d37ab3acb9c377a9d29ebb8ff99857", + "evidence_path": "evidence/20260721T152942Z-0adab364/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 22, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run2.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run2.json new file mode 100644 index 0000000..f4d4900 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run2.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T153028Z-6c5cdc03", + "timestamp": "2026-07-21T15:30:28Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ccabb07f4d234697f24f389467a1922fbade9de416b63c86a878006c3dcfce8a", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:bff0009abf6d7faaf9c3f531e5be6bb98255482a3c973f78fcae1af3150eef79", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:0615715fa56e7608a1d9808179095b779f54ba74fcd4154e987c2d8fcae04019", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:41396de16f3c258ebf0186b429f4fe4278926ca9d90774b6ad63eaae8d2df7a2", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:cd6ed63a1268e4b33523dea0c5c70342da00c1f608900650119f32191488fcc7", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:57aec34fd3956faa2fa586a614a7b00d70bd5c06d6b97b13f9637c591ed6cdce", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a6daee2d5c1dcf41cb6b02e12649976e56a16da9067385284ab71eefaaac3717", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:7a4134e18abcf4b38f9ced445ff1f50f6acfd74e6a10d4c7958b74979a12863d", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:ecc66c042a71a23abe7584f1ba1076c0cbd05178df6943bb171e76822ec2a8cd", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:be7e743da63e3fbe3ed15221a47b11c74d290c866c6069981f42c6153b042ad1", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:2b033d2cd97e002caa77d771fd653d74a7a569b5f61370a8fc62fabeaf974c0f", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:44cf08d6baa6c85a9fc477c6420340c192c78382cbdfdc2e1d796c4bd51603a7", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:67df51b7730edfe69e0c12d46f965b89477b13942304fa30a03df9d272d0bdbc", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c5bc3a7ad218ba22131b3fcf406be0546a1ec27c20bf995f9d47cb3c57ceac26", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:b7f947dc867579560e5a2fa1c6279db7336f7825f4a4deef77279d6d8dac3f38", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:f1b03caee119d322450797da66c79857146c05c33ba7109b00e23b7647cfc26f", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:6e8d861e1fab3fac9b22fe71b94ddb77a35e92ec424e693025a4f64e37a92621", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:ec9406d797c5679366ec6c0539020580ee9540282e6f5c59320e8fc63f6e16a7", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:486d44677647ddf07b8777a281b765b3dac9c1b1f54e25fd9ef14aa4b7787817", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:a0ce90d1e32782374931a811ff72e490e769d84f40ad26b4668f65aa96f0ed81", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:7cbbd0c61767e7db9bf9c107302eb0b9c0da5c62795394bf87893f2aa18b7d99", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:41df99c88890dadb81200a52640e5454f648aef6b915329f7ed7839ca6b54297", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:77ad92198ebb10ab387881694de3dce970fdc30e8c687d9ddb55ebb5680b2e69", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:fa1a82c7c409eb77cdf2e9da37eedb045f68eeef898959fe6574b58806315171", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d1a9c02a4a2598bea6502976cc1e13e21b72dacd58040e5ff458d8999c39798e", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:254ebb6827cf1794d510f83eb5aae777ae37ab087052fdc7e7ea3d4ec6eb9414", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:456d309f968a2daca99963b8e0aba2a5e6e8c2eb90b42986f2f498f3f427c73f", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f6ec4a38be9cffd206f9a5ccf1936531e682df87895825d31cd101d1f004df2a", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:d8485d2e11ab15f73dc8cd31961712c4a224997ea1b0c9e8a760d31ed5685e0b", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:00026ee23198cda60e312221c0ef6726f19f49ef7e70e057fb70c0972aff7b62", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:5aa7460a5b73d2350f3ac7f7bc2b295e9b08c59333c7ffad2cc88c75c8749be9", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:af794ca9db6aab917d16175e40c3a6055bf549c2f66201ac029da62b68cc886a", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:114f548c40407e7215f1a04e14f18f964c0f8c2cc32a1598310ed849f1b3c920", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a9ef101a7d718f5a5f18721d4c37eee71cb1423317da7676623b765ccb3d7b73", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f98a038abb0c327b6b40a37b2b6894836b131e6aaec88798a8770d01535d1d2e", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:8ef32ab3aac91f8855b537bb1560f21f40964b996be3cd8d7d6bf18a0a299d2c", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:8047bb60b133a73f77a7e09fd8c79c7b59f7a29f3839fb3aac6e1b9fabc53a3a", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c51ab33def01a29da80f7115dbcf8447f24fd6d2dacfce0e9b06e946811a0629", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9ebf6e7e3945217fe8851d39c15cd100bdf0de8173c9620d9941124364cd8165", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:135ab9caa9c7e0a02fa28d302f083bfa93ad3f2623e5dcb638fdada90eff62fd", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:c64d0b0139cbd5e058fa95a2afa6ca84daac2709d5a7b5ca604131aa8acf1e60", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:a7ce37d2139c65771b8ea42225895e193a53695ce4810fc9f19540f11fd2c1f7", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c58c00617a20faff4267349a60733af527562c0a3093474f59ef674e68a4d9f0", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:73b315b955083584e649f0e88b1c9edc2b084a0a046600d332f3b5dc5e7b9b4d", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:eb2900a903678923d9b5578cfed41b74d4cac8fe943f80666704df13f10ba276", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:02b14e1ccd51a6c9ce0fe8ba14db182e1c9becdd9e974a2083963932f09ca21d", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1633bd87893d676ba82ffad412ea754e1464c9c2f4558d6b193b7f136397e5ff", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5be65e43d12465a62bc51f25ac5eb7892e35757b42e07d6e63c5cddb7058a430", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0a2a7417031ff6cfdb2668266811666f7d1f39ef81e377d7eb5a3c685d543c83", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d0c95944c6f61320d1caeb3fdd786eaf00168fd39aa399cdcb3f558adc2b7335", + "evidence_path": "evidence/20260721T153028Z-6c5cdc03/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 22, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run3.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run3.json new file mode 100644 index 0000000..9ec0968 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run3.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T153111Z-3e52273f", + "timestamp": "2026-07-21T15:31:11Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4fa35bf74a8cccac95d8f8610165ce227dce2163ce5187a66dc9416e54179bb8", + "evidence_path": "evidence/20260721T153111Z-3e52273f/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:480b054c45d6313692218dc1ef6a56e3f66cc17fe56e7acc6a25e84a954d2591", + "evidence_path": "evidence/20260721T153111Z-3e52273f/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:2b21871e347b7669b1cd7945d61b7090e8b8c10ddbc1f1a1ed8e74ab14bdca8e", + "evidence_path": "evidence/20260721T153111Z-3e52273f/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f6add3b5c825a0c4e59cd8c7e491e51911bb5e112014c7e119f84a29d2afc017", + "evidence_path": "evidence/20260721T153111Z-3e52273f/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:61e8956dd9db490a65700c4246f8717ea872441772f2b884c76bc6ab87c863aa", + "evidence_path": "evidence/20260721T153111Z-3e52273f/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:7323b59903e0eb6985a048fa8aa9419c371dbd17f21e1dd112984a861b883c72", + "evidence_path": "evidence/20260721T153111Z-3e52273f/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:120f163d8cbd7ae58970daed83c8cad2f43b0f1824fb57f51dcfd7db41b8fb3a", + "evidence_path": "evidence/20260721T153111Z-3e52273f/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:89946622747ade53371c05ad9346a35035f29ab0bcfd3f37398d32b86d6d8778", + "evidence_path": "evidence/20260721T153111Z-3e52273f/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:6d9648ce3451a4a61c150a7312c976dadb8d9ddadeba851dc8e298c1bc7bf6d0", + "evidence_path": "evidence/20260721T153111Z-3e52273f/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:db92a7c8517c58ce7e7ef11ecf090b70adbe22b5fe798b6ebebc92ba15fe58d5", + "evidence_path": "evidence/20260721T153111Z-3e52273f/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:a7e66375fc1c9e0b9137dfe590e88a7abe97da684a241c0a1081a48387e65bc9", + "evidence_path": "evidence/20260721T153111Z-3e52273f/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0d6067ae13b4e0de4e276858962a65fddf7b1189d8109797ab7ada3bbe6779a3", + "evidence_path": "evidence/20260721T153111Z-3e52273f/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:17c299fd4a7932e9bb3b4c4c4be2a8cce29b688ed1d5a1193e21f4efdc513188", + "evidence_path": "evidence/20260721T153111Z-3e52273f/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:329c6b13230dac1e14b604c4df5adb6f5981db7ef6e23d4cc6fec2147cc9fd30", + "evidence_path": "evidence/20260721T153111Z-3e52273f/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:452ef59f6561715eda89204681175533eec990a5fb3d8c10e91e8e61bcfb4f2e", + "evidence_path": "evidence/20260721T153111Z-3e52273f/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:bfb113c94858f020cabc39dbdabedcb8f3e2946cbf396bb4314cce3c17b0abb5", + "evidence_path": "evidence/20260721T153111Z-3e52273f/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:900be6d9d7670f5a96e0728c0263f7a785a45d96b75be820ab772e6b8de4fbda", + "evidence_path": "evidence/20260721T153111Z-3e52273f/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:4a2b239f372cd69a37bf54f38bc69a3203be00f8bc4af3b2c6d559cc4c55dcff", + "evidence_path": "evidence/20260721T153111Z-3e52273f/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:226c185e8a4ba944f8e5beda0c841c6dd91692959482fb4109d9e89687d93817", + "evidence_path": "evidence/20260721T153111Z-3e52273f/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:eaf2ca1ed40d2fe8d49b673d639a90470adf3d0e4da13b3674cad06db89ce465", + "evidence_path": "evidence/20260721T153111Z-3e52273f/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:aea92b178d9f3aae737111b8071db1437ea326c391e3dc2d30b8c9be930eee9a", + "evidence_path": "evidence/20260721T153111Z-3e52273f/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:538ab3a921b9eaa89396712198c33c8bfad3d3c537c22be8272c31f0a0389caa", + "evidence_path": "evidence/20260721T153111Z-3e52273f/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:ed8e740ca7fb4c4e17445d21e9568799310deb1b1d1bfe792486b0f9ee018b7e", + "evidence_path": "evidence/20260721T153111Z-3e52273f/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:96c6994850033defbd9c23d5a0d17564f3d0e43995cd652eb6ee9f47b900bbb6", + "evidence_path": "evidence/20260721T153111Z-3e52273f/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:783c96a88abec85695839687975d60cb1d1e9e2266d5f1cb1a52859ffe0d2bb5", + "evidence_path": "evidence/20260721T153111Z-3e52273f/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9b634f97bae7add1a853df80d07a32455e1e059081df0da4ca7b080e50e3f595", + "evidence_path": "evidence/20260721T153111Z-3e52273f/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fbce671f126052e179fbe9ebbeee056de329dca5caa0fb09015a7698ffc3a6b5", + "evidence_path": "evidence/20260721T153111Z-3e52273f/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c67bad37299de4ffd23869da60c2e76dcf6b64c0bfaff058ea1bec16f32dfdde", + "evidence_path": "evidence/20260721T153111Z-3e52273f/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:614d58fab5a4030c582e190161960e74241580271b3b2aa1858930b0dbfa157d", + "evidence_path": "evidence/20260721T153111Z-3e52273f/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4728eef4d157758cea419db4332dc333ec5fb71581b37cb14720859da547452a", + "evidence_path": "evidence/20260721T153111Z-3e52273f/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:0d85572bbfe7cfe1959570d6f472528e4fa45c40df64eaa8e6d6ac525178db3c", + "evidence_path": "evidence/20260721T153111Z-3e52273f/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:a648a9b083f1a0d323acd10e980551123b5d359e32c81211b5f8176864931c17", + "evidence_path": "evidence/20260721T153111Z-3e52273f/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:4f41468007a01edcc2862a88d442ef669ac4d330a985153c35f3a029c6e40c16", + "evidence_path": "evidence/20260721T153111Z-3e52273f/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6699d47a36e14832596dfab27f471a5f6ab1119aa6cec70986a06af471d28b87", + "evidence_path": "evidence/20260721T153111Z-3e52273f/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:588d97d7ec9e54912e91ff5f9cc7003f6429bdf210427dac130866da96eacd03", + "evidence_path": "evidence/20260721T153111Z-3e52273f/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:f2e09a6e9164c49d00972635bcf09b891e6eae63c634fd8af2a8a0ec501402d0", + "evidence_path": "evidence/20260721T153111Z-3e52273f/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:5f1264e2db38856514911731a473a864932d4a944dd7b47c08fbe5526d2ff213", + "evidence_path": "evidence/20260721T153111Z-3e52273f/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:48d0006c60f5f1ae68d2809e7bfc4e746f35a09b622261d389d1d093fb2b02b2", + "evidence_path": "evidence/20260721T153111Z-3e52273f/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6512f5fbea2a0966ebc341a5eae689c4cbd8b86c7495918519fd7b0ff22f3dba", + "evidence_path": "evidence/20260721T153111Z-3e52273f/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de737987e7cc5eb09a4b16831f4a9863eb03c1be28231540e040cc667d045fd9", + "evidence_path": "evidence/20260721T153111Z-3e52273f/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:49c5b9b7e04cc70b93d09079094da246d5b82e6df5636a15697cd37d8838a4d9", + "evidence_path": "evidence/20260721T153111Z-3e52273f/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:4620e8f4a8843ef3431dc45928fce7f6b7c149b008e66af658fbcd397c3e0bb2", + "evidence_path": "evidence/20260721T153111Z-3e52273f/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8943b21b79b690bd449ef508ee83e6d3702acce73d4bd9d392cafd8612b0fd93", + "evidence_path": "evidence/20260721T153111Z-3e52273f/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95f7dc750653693c81079fd1c22ef8a155f0c73e17b307b08852e4f2582e9a3b", + "evidence_path": "evidence/20260721T153111Z-3e52273f/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:b7de9bdf9ad2492e6751fddc24c618150e661af78822b957336bb6a3f9fa9ada", + "evidence_path": "evidence/20260721T153111Z-3e52273f/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:743093ad837c0d7c6e16d756552e448a276685241d4edbaa9d221ea7b4207f2d", + "evidence_path": "evidence/20260721T153111Z-3e52273f/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0ac08961cb2478af67a291d20fda4807e5a86d8c29f6fa84a64e98dd7dcf4844", + "evidence_path": "evidence/20260721T153111Z-3e52273f/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:343eaf66bda10abb439a3a6c144bb0255190f3a00c122caf06437c77400eb285", + "evidence_path": "evidence/20260721T153111Z-3e52273f/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:41a86681bcd97e58e0f03d37673688cec8b727d11a88c4b1220859e928dc6504", + "evidence_path": "evidence/20260721T153111Z-3e52273f/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:23bc3e782de44946f5ce951415996b930637204ea8e160f91465407342020e3b", + "evidence_path": "evidence/20260721T153111Z-3e52273f/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 22, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run4.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run4.json new file mode 100644 index 0000000..464355e --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run4.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T153153Z-ab868c8d", + "timestamp": "2026-07-21T15:31:53Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f7b4a6d12ad66822bdda27a51237df4610684a7ca605fc99946ddb0f7e90c9dc", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:033bb026ffcf9ebbafa975c57d02368b7a148d7706e12eadc7097cd024f088cd", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:9e6705f737430a57eb587bbcff3a6e2ec9f63c231177115b10ae8993de7630cd", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:9fb4b6735b2c00b954c965324933c8e82bd2d753c677d47e0c9c8b50a7bf741c", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:812f3f48721f1da2eb911a6b401307e50213ce8f16e8a312bc9c01ff5d6fe6d6", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:89e812d87cfe21ffb3d6dbbed985ef367e1c524aa76f9105854a81b9d2b6149c", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f303045039d1817f5009f3e27392da4e239b7781d2ba0bf99464e69dc7556504", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:6b0b2704ecc3cd84031ce6cfba10fd000c3556ec6be4cd3d4dc823a898308105", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:a6841fc05b4c8f4b2ad3a88becaf585ef53acc6514ea2bab1248f86b8c59ff3f", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c64147b019471575c6df1c810988b696d2e710e80793af640c8583f6f1369313", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:51a88bfed8843a4102b88ae1fb79b480b51e13af12a4a92c10e3b0bbbfe34cec", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e3d5e6627a47cf4da34ec1ea60da5df54ca537f6a28eff71048013d9fe35e7be", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:fbeb8ea5cbb4514e1bf87f447e90f30ee2f72cdf2fe23aafb7b51c921563d114", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:011e95e5096aa414f66f614abb7b8b9b207dcea371c2430fb6d897b072c1db3f", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:e93998f804d7afd1b4a8270ac26a2ed20e7d37e3f0d1690f1db7d98f29aca351", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:101cfbfe91cdc31cd26e351865f6786cc8a953e060ccf6f54803aa600dfb7340", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:1501feff852ae926d0c8bd0f7e10585fa7a6b27861ef0bd731636db87523a458", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:6009e02f6f18aaf2a05e1e3c93bfb6fefb8061205b030b8f1e4ebdf05d406975", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:64dfae9e7bfac6207567795b852d39ddef57ef7b92bc2a40979bc843f423288a", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:4fef81b385eae4f7b959c574d5680dd2c5d62bca13771807d0a0bda399046c48", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:b2bdec84efa38affeb721838343473fbd12c55d798081f7d2d2a2d868c8aee44", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:e2c3030106a3f2e7bf573c66ff57ed76eb9bb2704595e35ef1b8a757bc661a06", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:4c0ca6480989400abc3a2310368d06331f6c29bbd6807f0e2b71324437688b74", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:4e0b1e2f66f8905e1d6ba3c9a713b00a27a2e04bea28cc9a55e4c08135cd0457", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8ff61e38151a5ad64f0cebaeef2a4392f54835761ea40b111135a1c130eeac5e", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9910f57aa00b729c2216f8d9661cfe9eaaaef4a4b2e0b1c7ad3409691e22c1d1", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f8a3c97be11d4a9a4d7218b74fe525fdfebf06ad8588101967031590be3da2c4", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:43e27dddb34994ffeb48bd89da15f314afee044f18d9bfa5ecbf57bcadebacec", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:db152a634df707b60b4daaf859877fb9907c6e4f712eb761afb21e463d5c8498", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:06e5c00f98d8caf931936e8569a696135219c7669d6aa7b8734c65753fbf7424", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:5b865520f2d2271be34791c6847cb0d50fe6259647ebf38ae7e606734bb57126", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:d3ba44ba32e163bd87d61c10b7521d17473ec88cb4d542f379dbf9bff0641172", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:8d095b8b242d05e7057c521610c7d1c583ac753ab3c46fb49c127886277bd7cd", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5256664d124da036130f69aaafa7d842c0d9bcdd9491328802caa866451c2397", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6942f224835d82dbaee7bfde6f1e6a8e7607d02bbc8e9b01fad554cd46320566", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:b05ca1780614923617d2291821eae4b2c99a2b0c78a1b5537ee8c8d4715a44f3", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:9b8a96e9c6ac126f0df5a31df7383fccfe213e2c0c7a6dde07c1f4dadc8a9043", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7a8210de0ae78a2b0c88dae0380932b00871f1668e22c2e81ff5399781b3115c", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:86361e07bdc4d670b41dc3c36544c5921cda7bbee132461d7050d3414635e94d", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5b9bc04817cbe8e94ad42c116846adfde228b3e0f755005a8f5174e8f9069676", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:c4d92012f6539d6379c5941b79e00e7be82cd21f1096f9b3f423a3cb7aeff858", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:aa5061ebb29d44e2b550a16840fd1a8354d87ee8d9f57004c71b593a25904745", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:13e9e18caf8e501c6285095363cf3da5a796cbdbee5df7574c2f390adbb9aec1", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:abfaf059e3d6867ad528ecb15dfb21a0bf9836228898b71ad10f3994df475f49", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:d2e3c89e7076bc789086a000eb0c2ba8c3afecfa3fa945540747125f0bc123dc", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1cb0a44f6e6b558c1e3c6b827b14c25190c34499bf8fb3b1137c65302be007de", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7eaf64f38596e990c1b2225d474f87d3a9f90d7964bdbc1ccc5934a201ed0b71", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:faf39c5c10cac488c75286a86a70e7c9de7fa38675bb38ea33532d9dc9633329", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:702ea576cbcd0e25a5a54fa3e2ecd2462982c04045332efa35480b2afc9ed78e", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6fa88c23d960c4f96d0909bf501685061097818e6c7ffa3fe5e76457284f3eb5", + "evidence_path": "evidence/20260721T153153Z-ab868c8d/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 22, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run5.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run5.json new file mode 100644 index 0000000..4e40161 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-meta-llama-3.1-8b-instruct-q8_0-run5.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T153235Z-5926cc27", + "timestamp": "2026-07-21T15:32:35Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4926df90a68f4a80e0d791273828e6d160c2063681ca7c39a213ee0da0c6df07", + "evidence_path": "evidence/20260721T153235Z-5926cc27/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:517cd75f192598b072c233f3cc6aca187a8735b5770d956491d481a44e50e478", + "evidence_path": "evidence/20260721T153235Z-5926cc27/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:234c89da3e83dfcfea9580ab43c86452d185a33cb922f249e5aa7138014e0ccb", + "evidence_path": "evidence/20260721T153235Z-5926cc27/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:e8ef30912f951b3e6bbc6485164ab6f47235a26df79cc1c27c376ed337d1b0a4", + "evidence_path": "evidence/20260721T153235Z-5926cc27/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:090d92eb301e63132618f5c6dd18f68396171d7bc47ca8aaf303b7626b86d334", + "evidence_path": "evidence/20260721T153235Z-5926cc27/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:84d2d2b1a05c6d75689a45e8d54b5e73deff8e6fb832586b046c027e4f0ac487", + "evidence_path": "evidence/20260721T153235Z-5926cc27/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5c6025e3e2e754490f9776edb319277d7aa6866e2bedfb326fe854f092a5e692", + "evidence_path": "evidence/20260721T153235Z-5926cc27/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:daf5dec1941147e1351e588580df53c6556ee4d9895895a2bc4461eac13466b1", + "evidence_path": "evidence/20260721T153235Z-5926cc27/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:233b7ac0e312a40ecef764480d651e1862b29189416ebef6c9482490235b2856", + "evidence_path": "evidence/20260721T153235Z-5926cc27/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a797de0edd2e300841dd48b7c46b25d22e37a24e94ccc717d17bfef1574045db", + "evidence_path": "evidence/20260721T153235Z-5926cc27/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:20a11e971f2ca5e5c4db1bc401eba56c05dc2133f441717543cbb02987c16a10", + "evidence_path": "evidence/20260721T153235Z-5926cc27/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:61e9242fabc123c564d63067ae7f2174e7e86a086d00a65913070f3fe9a50773", + "evidence_path": "evidence/20260721T153235Z-5926cc27/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:ae4aae33a70ea2ba581e60e586dc9e366e20686499312e44d0f5a6a4a186aedb", + "evidence_path": "evidence/20260721T153235Z-5926cc27/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:82af60e16bb7d23726349a6a148487c097e5c0235e3ac9bb5fda697b2c89ed6b", + "evidence_path": "evidence/20260721T153235Z-5926cc27/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:9fe7acf21492f3369ad495e6ade93a7b6e0b3246e9f3f44a1f9fc50df5cab46f", + "evidence_path": "evidence/20260721T153235Z-5926cc27/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:86be27dbac18e9cfa30997250c4e812540fe4d38c6b2d9fb0abbf0b35fa58711", + "evidence_path": "evidence/20260721T153235Z-5926cc27/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:ca6075c524c559493bb46f3c19b2ed2f082fda3ba073f91e40b29a361f757558", + "evidence_path": "evidence/20260721T153235Z-5926cc27/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:217f24aeb42c9e78d5203919377abc7c6716171465458edafdfc736dc5f025b0", + "evidence_path": "evidence/20260721T153235Z-5926cc27/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:ba79b08e4f95c54e65cb586d17aeb8fd02c9020c41c3bee2fb935a367e63bb7a", + "evidence_path": "evidence/20260721T153235Z-5926cc27/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:a66503255ab7e63a597ecd97616c3656e2a91c00f13f89d1232d6ca296c25589", + "evidence_path": "evidence/20260721T153235Z-5926cc27/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:c995cd9d8fb8feac166979cc68f1eb56b435a7f9eb140474710eebd50fddd226", + "evidence_path": "evidence/20260721T153235Z-5926cc27/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:85d7513ff575e9d206dfe63b68fb891284721a3320effd117ac3598d73182a86", + "evidence_path": "evidence/20260721T153235Z-5926cc27/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:17b71517361bddcfed91e41841a122916a5365a2c2f7433e377cb5fe5ee0935d", + "evidence_path": "evidence/20260721T153235Z-5926cc27/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:5867d6df5d5f37f4765c5c6d408ef9bc3e4be8e8185bb5901e78b616fcaa911b", + "evidence_path": "evidence/20260721T153235Z-5926cc27/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7dd15da8cf3ba5598bf69bf2d2e259c3abf1a9b8c3793309c6c5dac4e1d75625", + "evidence_path": "evidence/20260721T153235Z-5926cc27/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:153fccf4b8e1f4c6d8d6b0d37b63d88c47aedcef2383603bfa7a5e7511a532b0", + "evidence_path": "evidence/20260721T153235Z-5926cc27/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:58f7d7ea97a10d2e672aeb2359f4283b260568aac7ba5329a2f8af24ab492cab", + "evidence_path": "evidence/20260721T153235Z-5926cc27/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6bc3c324a32abe01ccd4cdb026115ce40989f2767954baef77e32eeb86b2ac79", + "evidence_path": "evidence/20260721T153235Z-5926cc27/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:239be05e416a2dd4884c2f946eccf59165fb1b26be72c8b613927c0cacc34402", + "evidence_path": "evidence/20260721T153235Z-5926cc27/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f6f81a0c5a75c9c5e33e71e8e376f792a5c6f32e73f796545816364c1ba87602", + "evidence_path": "evidence/20260721T153235Z-5926cc27/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:3394bb416c72f6cde8b5b83e050ac9df3db53af39c5b0f25b7cee3f17b5230af", + "evidence_path": "evidence/20260721T153235Z-5926cc27/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:efcc36db934cba4e422f3261716848fb05104b2dc99a60ba941de2f02685d063", + "evidence_path": "evidence/20260721T153235Z-5926cc27/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:4878daaf121b5ed4f2a4660690007d33e5219851c2871806146fb06f1864f835", + "evidence_path": "evidence/20260721T153235Z-5926cc27/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:59a42ea86dfb1995a09f6c3b16a0afcb14a7909edab5a8200bfc854f7c70e4cb", + "evidence_path": "evidence/20260721T153235Z-5926cc27/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:30113f0bbb9045e23c0d878d0674f4c5ba56c7eb0a78e9b3b5a1308dde175d30", + "evidence_path": "evidence/20260721T153235Z-5926cc27/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:ba198a1f7c2b1f421b0c95faecaf2eb23393a42ca66169d39910d4da687e47fa", + "evidence_path": "evidence/20260721T153235Z-5926cc27/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:7a9c66075602c7c15bd14d5a206810d2aae2e27fce3462683959ac4d07a3f4f6", + "evidence_path": "evidence/20260721T153235Z-5926cc27/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:208bf7c49f5cf477d9dc3b706a3f47f24331beed2133e493c55183e59e371dcb", + "evidence_path": "evidence/20260721T153235Z-5926cc27/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:97335dd002f72682a4d6f7c8b509de01d7520919dfbf000bef484cd3ab2159b0", + "evidence_path": "evidence/20260721T153235Z-5926cc27/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ff652ac11e185cab2f639f6844bcaa6529fd2dad502414fc734b3040e14ac4a3", + "evidence_path": "evidence/20260721T153235Z-5926cc27/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:3012dc9bab0dcac63b194a23f875e39feb246d255a26cbe6dbaca108a62922ec", + "evidence_path": "evidence/20260721T153235Z-5926cc27/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:306e2849eb9fbc8c0aaa3d032729e109d64b08f4e7e901895eeb7659b14038e1", + "evidence_path": "evidence/20260721T153235Z-5926cc27/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7abe6a8c31195b39ec010ab1e51571174bcd01e24bde27a8f0c1e72127603928", + "evidence_path": "evidence/20260721T153235Z-5926cc27/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:69d26c0d7159a57d757c7268b7880dc6a48f8299f624bec1e12b068c0902f8da", + "evidence_path": "evidence/20260721T153235Z-5926cc27/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:fc014d991f05e1ff0edd66ae0b7f63381fe8abbefc9373548c936909995325d0", + "evidence_path": "evidence/20260721T153235Z-5926cc27/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:75791478dd2939aef6ee8f9b5801cc55915530e35d931fd8b42d57a8e3e7bfcc", + "evidence_path": "evidence/20260721T153235Z-5926cc27/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6c03d8c45f55f32f4c9a38466fb295012f4623c9f21a09db96f0ffa601b82b9e", + "evidence_path": "evidence/20260721T153235Z-5926cc27/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:08020a9006bd1438227cdd68ec08db8366817f7b45e3b9536312c7a71da0260a", + "evidence_path": "evidence/20260721T153235Z-5926cc27/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:02cb8295a4b2a50722dab9f33c88b40f5c611af52800a2ba2127550471c146f8", + "evidence_path": "evidence/20260721T153235Z-5926cc27/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:97b3f5d19af570fa02da903b0c7b2f3d6fc2823d530df47d6be3e0fa89f30971", + "evidence_path": "evidence/20260721T153235Z-5926cc27/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 22, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run1.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run1.json new file mode 100644 index 0000000..da0391f --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run1.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T154018Z-e14ecaa4", + "timestamp": "2026-07-21T15:40:18Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:aa2c55c40cabef213f5e915ac4d633ba6237b7e52eff10fb2b19d6a9b5988086", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4ce04502a5cc1608f72dc0e3553181e97b8d5868b7d8a7f1e126757cc2531f23", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:534d64a156100b342681013473f727a83bf4b07be2b5ce72f01473a2ac5d5719", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:fe33aa5e6b418d0bf90ceb554efd6be5bebf801d54f9c2f8b05b9ce885d26e0b", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5699437eb43f8dc914a4d05cdf53a99496cf9dea55546836a4a724ed728a449d", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:53354652c08abb5d89d821dd1608d97041684e82d86661cba790e1533195a023", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:2c8557f07d142c7600cf316dab8027c9902e7da7f96a75cff31582ea651d3fe2", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ee6563a5daac154f77faaa2d65832f0cfd0691aa101eac44a685357e2b5a2f3c", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:252243da7e6ae1992b93aa3cdbfc6d9b1f2d69a9cd182be06ec2026f08a6ee42", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f1225c0a783619542e5d329e669477089b181f492eb5221eb5a1fc0766b01922", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:492a47120e2bb2ec7287616ad3661d58d606efb3e91c9f436629d68b71a0923c", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:535ee1e214170749d555b6a0b433425ac65f7c55c52f22987f4f4675674cbd8b", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:35d70c347796f6a99cb6a341efd6cf26fe9b75e4e0e92f8581c172779fa4830b", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1b54772f3c607d7de00192919cd16d6c2280c4be3f542b8417b75e7dac319e20", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0499bed6e168fe890829a44d4d0202d051353ad045239116a606ecaef67b5756", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:226a88d87b86403dbc655eed081b4afb8f0dc4b0618d7cb44796ff0d348186c2", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2360194ac2f766f7ef66f8293aaad6d4ee1df548452e6b3262cc1f6faf965da1", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0fef95270003ad03ecbecc8c011274e0dd65a94fd099994d42992146c2602699", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:86aad0817bd4c88b920e6ee6e25287fe27dc7d7b6480034302456ccd07766d21", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:904da950c63fb10c5ac3d251b01afa9058912242cb1f2da691637a075973bd93", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b16a4511e29b8e0f8528fbfe1ffc7437c6dae727a9758112dc77031be301223a", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:41b35d194259dca0f43b5f71de91b7479a292904e50aa1db0ea4cdcd80c06239", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c218d556c257129ba19ffb5862240f3271c9f77493c4b8ca8128d502bd568072", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2bf7b2e94ed8f85fc61109d48cb1f2545d30bed75b1acad7d867a88bdfbc77b8", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:08c1d212eb0ca9f23b10260271fcb89dd6d09d966668868be701f6d203875f78", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8bb23e66edcf63be2668713dd1a38ecc768a61b8dd81057e8171081c813286ce", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a4951fee5f8e98faa63da5ee3c649b1df7461c769811e211bf3b317fcb5e42e5", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0760a8129aee3ba1f8ec947cffd82f38609803b2685105bd531bda2202a9d432", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:144d7db360a6155ed6c1e25b69e612b7ae39c92cf5519371290b759349139a7e", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6930d29037289725ad8e600e3fb2222753c0ead84132226abea8e79e3fa2faa6", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0542119139bf67030a4fca2640e511e70126d33574ae187847e1cab7d8d33c88", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0d7dd3d8f4720eb2e3f620cf8873025f7331a32325d95df958ef8e5667953b27", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2639da455c27152fbf4f0d939ead0d5a8f758e0c2ef8cbb264dd9824245b9304", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1d1ac9058cf09aafdb5f63e8e69b7c7fe8e8e151f6dba2478fd28241f3e646a3", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4b243b301a024047564de21588c9251e848de219f26abe342aa3a1b62cf1a588", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e0860d7c65442624b23ff4e3d1a948d2a9a7f6e2b7045667534fb43e97dd9627", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f0cf13fcc0bcc7e566c749250d8034437041300336923a297cf5ae1f052d3545", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:caae925875fac90e95aa1fb4407b4aa3dead1b18f48df39291da2b6195644513", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d0bb4772eb62a32a40e9f2315cd8a45a171db8f96ca311abd27bbc42f1aee8e1", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c36dab516fe31185861609d8c87cb4fac43639d590f8bcea9ac63297a4e67db5", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c0af8b1a352a3a3341b3c9699e298586df84cd609ec864134ce35ee1a14e6fd0", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6f78f31d30360362ea973bd12f176078e1eeea0ff6d52418ec4d1f25e1ea4c8f", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d389d4534190b4251e13550e973f63b44335b3be26306bb60ddf938f47c24f1c", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de31a3ba5b3cce88d458def1304dbe0fa5c118de82f103edcc427cd70ce4668b", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:56e5bc18c1b705ac64af2c22e7c38db8199de2e5fbd1ceb57a9d0394610d7247", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b09f834f2ef1699d689c18561edb4393943e01aa13adcf825489abbd269b9009", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e708289c1c8cb8939ef660df94363f5c9821b7e0cd1abf0a805198cf8f6510d1", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5f831e6589b89e6d0c56485521570f80d5be42fac97a3c9683e11c8b2eeceb8f", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f49cb3ca9b04f4f3c9815081b9b5dc6a3760ef790e13643c5539d7aa29fe2608", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f8c807c6a77cca7533a647989fca24af5c42d3b3b83ce4d4c9d220fad767da29", + "evidence_path": "evidence/20260721T154018Z-e14ecaa4/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run2.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run2.json new file mode 100644 index 0000000..8c1b85c --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run2.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T154054Z-6ecec119", + "timestamp": "2026-07-21T15:40:54Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:3c1fdbfc5177cae3fb6a73b073193a2f3409cf1858329090237f5046869a99b7", + "evidence_path": "evidence/20260721T154054Z-6ecec119/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f324f024d954c73d58d06843cac17d9a213dd5d0e1d32009e6711a4d10f0f6ee", + "evidence_path": "evidence/20260721T154054Z-6ecec119/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:fb84949beadd6785ec324eb9cf637984ac7d604db7a9ff352329043afb13944d", + "evidence_path": "evidence/20260721T154054Z-6ecec119/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:68aadb1c3234df400881f3849ddd1be83053a5f8bc398a77e254c007dd8cbfe2", + "evidence_path": "evidence/20260721T154054Z-6ecec119/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cd7183c304692941871ecf6272a287c28b464e7550d186cad54fadb215a9457c", + "evidence_path": "evidence/20260721T154054Z-6ecec119/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:b46ae3781f2bad342237c7e01978b67b754259c43aaa5b4af12bcaed88c3c5cd", + "evidence_path": "evidence/20260721T154054Z-6ecec119/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:733b0b86c3f46bdd68867b5edf4f31559313aa069df08e3334810d512f1d5309", + "evidence_path": "evidence/20260721T154054Z-6ecec119/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:81e5c7c48bd5175ebae19326c8275450ec51751ff7d96f5cd3ae4adf58f1847f", + "evidence_path": "evidence/20260721T154054Z-6ecec119/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7cf749c6d1fecf005175ec394e3b76bc39f8378bf15daae2626ad5acce48c398", + "evidence_path": "evidence/20260721T154054Z-6ecec119/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ed8b227233dfc7146a073b205a27dcbd6e118a32e8d4648fc1f42650b3e33ce9", + "evidence_path": "evidence/20260721T154054Z-6ecec119/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2edcea31d8630a5592c495eda1fb73214857cee89db1f20e28ee8843633abb36", + "evidence_path": "evidence/20260721T154054Z-6ecec119/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3f27696e164a607d672400a3a24a17e3f71dd15677ff83dce10f451635a5ff00", + "evidence_path": "evidence/20260721T154054Z-6ecec119/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e5345d8942e5016cf0d2370db69462ec6aa5629ed4f55c8885c67ffdf0616166", + "evidence_path": "evidence/20260721T154054Z-6ecec119/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:31e53b1fc8eb539c2e22bd91f128744a667f7bbf22e61f2d23cca5fd645ae04a", + "evidence_path": "evidence/20260721T154054Z-6ecec119/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2c55b6806947077dd7875aab4c801b336caa453baf15cb715b13df69fede03a7", + "evidence_path": "evidence/20260721T154054Z-6ecec119/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:447f8b963e3703fccb9d054f5c90a2e5dd6905dbdf5d6eb01535c869a1a27392", + "evidence_path": "evidence/20260721T154054Z-6ecec119/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3dcf351d07aeb5d1b230e8c886e9ea86e4506af02bae154c862802f0b5ca096c", + "evidence_path": "evidence/20260721T154054Z-6ecec119/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:da3311ca11644f832646247befa59399f4424c606e0f8354b4f66309aae040ac", + "evidence_path": "evidence/20260721T154054Z-6ecec119/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:888fad2d7e5ce998e9f906a80055aa5065bf53e712b89eabebdfeb92a7773119", + "evidence_path": "evidence/20260721T154054Z-6ecec119/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c0b22814201d949a56390dd879c5f3329b5dd8e5799e92d3052b12d3357cf34b", + "evidence_path": "evidence/20260721T154054Z-6ecec119/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6b06b3ca1372b7383a92f4367d60156c6f0c42868a68bb1eee74455ac5350f20", + "evidence_path": "evidence/20260721T154054Z-6ecec119/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:55abd0750b9d20b4b542324cec97599eb1fb625f4ff8b0bc883a73e7f6493df6", + "evidence_path": "evidence/20260721T154054Z-6ecec119/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:09953bbb6895a13f508f84b0cedca259f0bad196d2014eb8d4b8d6047605c862", + "evidence_path": "evidence/20260721T154054Z-6ecec119/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5c010838a1d9317f692f907e8f879f87a32ea0084fb7849238b61d7bd661d3f9", + "evidence_path": "evidence/20260721T154054Z-6ecec119/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9b8f7f570a66c25eca1859dfe03483d5793d835873693874ed80e1519e3cab98", + "evidence_path": "evidence/20260721T154054Z-6ecec119/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:46aae70739a387f15ddc30e804fcd58a695cd1bf75f1193d24b69579949e7893", + "evidence_path": "evidence/20260721T154054Z-6ecec119/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1f2cf673bd6a833bcf620476ca1117ed783f44be5181e0c9eb2a905883a90a40", + "evidence_path": "evidence/20260721T154054Z-6ecec119/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8e8f00ce49dcb831914e776ac7de1af207cc4934dcf8a88f19933dd9d8fcf272", + "evidence_path": "evidence/20260721T154054Z-6ecec119/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:608e8dd94762cf33b971026c54d7b7fb3f38d03ea6dee0262705ffc182e2ccb6", + "evidence_path": "evidence/20260721T154054Z-6ecec119/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ea76cb80a296593ac72d93ef2ffa993193ecf85c4e5f266e167467bd23902c3e", + "evidence_path": "evidence/20260721T154054Z-6ecec119/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6a1ffd961feca8b2478f0e5ee7cb0800fb08097512ee6cafc2e82f33c4e762dc", + "evidence_path": "evidence/20260721T154054Z-6ecec119/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:87e84b2182f497c59c15be4cba04e5c5abf7d6b4897acc8076b1abdee2b22f1c", + "evidence_path": "evidence/20260721T154054Z-6ecec119/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:15701777e7527741597f136d7b782f2bf4ecf451278573cf5b65ecec5ba4742f", + "evidence_path": "evidence/20260721T154054Z-6ecec119/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:622ce65de660ffce41e37affac936cf3045504ce0f913996051d38c7daf467ad", + "evidence_path": "evidence/20260721T154054Z-6ecec119/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8a499eed3559abd7efea327c11809f6a434dd66e6345f9c580c216856e2d70a0", + "evidence_path": "evidence/20260721T154054Z-6ecec119/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:53108f0ad4f087ccfdf1aa75b68c225cedc7ef78d72f24bb72d29980fe720635", + "evidence_path": "evidence/20260721T154054Z-6ecec119/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d00dd1188073b81830a6268b93874270f1ede97870674f69b2d206889627b106", + "evidence_path": "evidence/20260721T154054Z-6ecec119/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f9b238ba58ba770c9109b16614e736fca9bd1a117656dcddab5bd3482e98a7e2", + "evidence_path": "evidence/20260721T154054Z-6ecec119/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ff2285fd3871d6e454b835bf2fcc9e0cf99d91e774f418743c73fa7f38154cc6", + "evidence_path": "evidence/20260721T154054Z-6ecec119/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:db501c21059fbefdf566549e56ac6c9b7b179d9f86f0862fd26d628dc48ed8a2", + "evidence_path": "evidence/20260721T154054Z-6ecec119/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:089d3e95258fe23df1c2b7e3fad64b2eca8b374465dd152a85c0baf55f88f705", + "evidence_path": "evidence/20260721T154054Z-6ecec119/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:28cfd8f51e2e3e2f76a9292fa24833f123b3f9bd2c4e2163c7f98be915ad9eff", + "evidence_path": "evidence/20260721T154054Z-6ecec119/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d97d3a8c03320f601cc9bd85e14d18f7f6f20f5beae05db2fa14db5cd8e575fb", + "evidence_path": "evidence/20260721T154054Z-6ecec119/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cedf02093896dba653060ee6ea9d7cb511e6912644410b4fbfc01061743ca2cf", + "evidence_path": "evidence/20260721T154054Z-6ecec119/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:429f7c3dd46724859533b8ee16c31355d73af037d01e950ec79c6beda2e00a09", + "evidence_path": "evidence/20260721T154054Z-6ecec119/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cebd1c74cbb1764133af33e7b4310c01b138e12a5b88c1eb2f31c8660359fa3f", + "evidence_path": "evidence/20260721T154054Z-6ecec119/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6e0e1d029abbd1001edd44bd6ef6d3970aefa500bf5f1217c8f842bf2e992a55", + "evidence_path": "evidence/20260721T154054Z-6ecec119/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:434cd88af58fe8710be06e7fcb8daeaeb41714db99bc595ac341a8545484d371", + "evidence_path": "evidence/20260721T154054Z-6ecec119/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ff4f5c8f31470217f57f3691d86c076d7a7476573c29a9b90149a628367cd778", + "evidence_path": "evidence/20260721T154054Z-6ecec119/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bc0403dd43fd07f67764f5293273e9d7efc9b3707f005c333fd5163ca2cbb004", + "evidence_path": "evidence/20260721T154054Z-6ecec119/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run3.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run3.json new file mode 100644 index 0000000..6e32ff7 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run3.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T154126Z-0103afc8", + "timestamp": "2026-07-21T15:41:26Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:7b6124f22e8abd917d564e779a84d0e1e66b1422587fce529025f95da23bc256", + "evidence_path": "evidence/20260721T154126Z-0103afc8/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f452453fd966d89a1427f4b8fc0a43e96a99fa6198a95ea1571622812b54b556", + "evidence_path": "evidence/20260721T154126Z-0103afc8/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:3566c4837807e43442687089f40344d1577a772407ed1519a6d9a0e90a8cbdbc", + "evidence_path": "evidence/20260721T154126Z-0103afc8/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:1a202ce08b75fad467dc81c785b869e17c829bff80be60dec4a910ed041209bc", + "evidence_path": "evidence/20260721T154126Z-0103afc8/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0a13b84f56300a5f8e354ef8b3655ce86631ac3261fb08913cb68053d62ccde4", + "evidence_path": "evidence/20260721T154126Z-0103afc8/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:8748555ce1cff927f361d09d3c9b6bcb6371cf006632551f5b89266a724956bd", + "evidence_path": "evidence/20260721T154126Z-0103afc8/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:96efa08e3c75556529b6e37a6bd6a122a08b1709df6a8337515927aa1f1ca553", + "evidence_path": "evidence/20260721T154126Z-0103afc8/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:490a5a31619e5a1b11f6adb46cdafffdc912b53da701875ace73f1fa8b8457bd", + "evidence_path": "evidence/20260721T154126Z-0103afc8/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cef961b4cd4b21e0cdfb30db554bf65ae729ea4205203836e2f87c3752d3789f", + "evidence_path": "evidence/20260721T154126Z-0103afc8/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:011a6e6d1f75de296306480817db91fcc57a4304e414f2a69defe60d952c057e", + "evidence_path": "evidence/20260721T154126Z-0103afc8/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6a673ad13dae0a1484f3ecdebdaac82182219c151bf3e7aa001b8722f0fc321e", + "evidence_path": "evidence/20260721T154126Z-0103afc8/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b71daac3b35971ec8e63de6538783a0d2ca69944d090dd5c1cf6e2a0e7aeec05", + "evidence_path": "evidence/20260721T154126Z-0103afc8/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:37c2a20c3868418814511850111ed2cb1d8376d38cd1ae58de6118906665105e", + "evidence_path": "evidence/20260721T154126Z-0103afc8/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e9988f2bc87431a796c1c3a93a841e2583193c61e63e4e11780ccc22c501c95", + "evidence_path": "evidence/20260721T154126Z-0103afc8/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b04d833c461cbf149eff84b488ecd80ffb4a16f2c4b614bcd99605f273147cb5", + "evidence_path": "evidence/20260721T154126Z-0103afc8/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:444c1cac5bc88843eefb43e983d748a106b6689e12b11511ce400279a28c9107", + "evidence_path": "evidence/20260721T154126Z-0103afc8/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:391e9e497ab39a5a68688d3d5d01d64b693102ce7c2965e555432a0b531ce3c8", + "evidence_path": "evidence/20260721T154126Z-0103afc8/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bd0fb8dd1616f35d938edae251995214080be948975f98fa591a9963fd6c269d", + "evidence_path": "evidence/20260721T154126Z-0103afc8/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:127c5a91bda68aca10c84d02e64b7723e52f5478f905ba0687d5db461bfd77f0", + "evidence_path": "evidence/20260721T154126Z-0103afc8/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eae80eb3df2d14b2015447f82d9d35c615edf6bf8a7a98f8265da9ef62288327", + "evidence_path": "evidence/20260721T154126Z-0103afc8/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a7f351a83cd15c7c425059d810cd55a26a1e6fa630e67bcc934691fdc9753dba", + "evidence_path": "evidence/20260721T154126Z-0103afc8/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f459f81ac5819edfbe0823d7e9ba461fc2e2c89bf545c9335244755ce32ee9b6", + "evidence_path": "evidence/20260721T154126Z-0103afc8/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b7b77df3ade5797c374f7bbd33a8a03a7e4a67e69c590429293184f9c6ca70bd", + "evidence_path": "evidence/20260721T154126Z-0103afc8/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9a9fee6ec424063792331bee8dec073db73cccb9e85f9a02c9209acb92d631af", + "evidence_path": "evidence/20260721T154126Z-0103afc8/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cdf716942fd6bfa2bc6f12a0bf65d9988e7dd907e278ecfa190c2692b09e1da1", + "evidence_path": "evidence/20260721T154126Z-0103afc8/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7d93b66e2a38f5cc09360097131c5d143ff2afeac62759b888c44c62a5447c56", + "evidence_path": "evidence/20260721T154126Z-0103afc8/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2ca51f3d51fb9beddc18f20f872f3904f5e33759b44cd45f5d9ace6feccdf600", + "evidence_path": "evidence/20260721T154126Z-0103afc8/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5f4bd8d73e17607cec46b1948858c353e8dbad5c942b9de4085d372f17906330", + "evidence_path": "evidence/20260721T154126Z-0103afc8/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:76ccda70451e1e23319927386af19f70100ae223ae78e201d4702ac42f24b4bd", + "evidence_path": "evidence/20260721T154126Z-0103afc8/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:efaaefe752e88705563959a4871b6c280939d0910058baa35e95a49c1774c617", + "evidence_path": "evidence/20260721T154126Z-0103afc8/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a51a04a9132040a496e7b1c47b233d712bf61656d9c178f81939c4c69cbe054c", + "evidence_path": "evidence/20260721T154126Z-0103afc8/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:564380f7040962210abb0faece338061b905e8e1beff3304540f14a4b3a06cb9", + "evidence_path": "evidence/20260721T154126Z-0103afc8/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f8277da87667e3985935b055a912ba688dec52e79a630e1f4da4495f1a843fe7", + "evidence_path": "evidence/20260721T154126Z-0103afc8/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:94315624566d43b38eb0d4d7ea24a19066449c867604fcc4126bf594f66e3112", + "evidence_path": "evidence/20260721T154126Z-0103afc8/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e1ac838d7086a0c91fce7d5085b2356f197b31a5992b35ecc11acc587dec1ecc", + "evidence_path": "evidence/20260721T154126Z-0103afc8/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:613df744cadadf6308a3d05172797f485d565c737e422e3debc33b5a5c354f7e", + "evidence_path": "evidence/20260721T154126Z-0103afc8/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:25c18affc13ab498b46d2e52085dfbe5f29ee1cd116205efaac2f60f9a5c7d83", + "evidence_path": "evidence/20260721T154126Z-0103afc8/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:58b2c6a05b8226875362358d9d8b52d4f5872c5e2291192b9fa9a9a0f7f830c3", + "evidence_path": "evidence/20260721T154126Z-0103afc8/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4ef819f52e44e39a2986de21cb1e8234df20b925c5c08f6dfbe49b2fc6e599c1", + "evidence_path": "evidence/20260721T154126Z-0103afc8/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:37cedeadc3cc0c4f2cabd8116dc7b35cd0ab272ae39c8be7b0c453c25107bb8b", + "evidence_path": "evidence/20260721T154126Z-0103afc8/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1c0d59c36fa5a756a2b830f9d54f3c965da0d106ebd0cb4aca580add94c5de43", + "evidence_path": "evidence/20260721T154126Z-0103afc8/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ceda335e02c5944269ff237cd2db993949d234e5853065c0465ae280b15fc9fe", + "evidence_path": "evidence/20260721T154126Z-0103afc8/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3648e3810ce25e903a19287613aeb2f1e6351d4c79bad241b84448e26eb45b87", + "evidence_path": "evidence/20260721T154126Z-0103afc8/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:81a3c07e65ecb8dcd6aee3fb117c1e6f51456f769770142ba69065d65f53d819", + "evidence_path": "evidence/20260721T154126Z-0103afc8/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:322a7f19487212687dc506ffcd30297b8c9a74231d3d2f36d3858fa0fc76238b", + "evidence_path": "evidence/20260721T154126Z-0103afc8/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cbc9e07069d0cfdaca8c95ef010cc32bbb8519bc39e3c1bf1786878fe99bb07c", + "evidence_path": "evidence/20260721T154126Z-0103afc8/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:65e54d6d452c8589ac1bc7395bc0db4127e2c7a6fe26732a94657aa6bc07eae8", + "evidence_path": "evidence/20260721T154126Z-0103afc8/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f438d4c44da5975aec6b62b7a2406a3a825e1b2efcf6a31e6d51e75a5f13bff2", + "evidence_path": "evidence/20260721T154126Z-0103afc8/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c164e70c45dca9108ec2a2c4484ad23580f5261e5bb05362d0724a676235d542", + "evidence_path": "evidence/20260721T154126Z-0103afc8/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0bacd8d14b9661af41ff090d9b9c3fe66cd228d7f92aa3003993ae438e8b4821", + "evidence_path": "evidence/20260721T154126Z-0103afc8/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run4.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run4.json new file mode 100644 index 0000000..2afeae9 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run4.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T154156Z-2b7e41ca", + "timestamp": "2026-07-21T15:41:56Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:40f5464b7d9d1b5c98f6caada2353826e1e42056b15defc49a41036781604363", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2d157f89020828c2a221f7a4c3bb3da53d912f70a1b2c2869fdf206abb0c751f", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:059d030bc48fdec3887c1a225cfcdc12521e4f5afbf4503514c959c955da49ee", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:2430d096fc501a40640c65c1d099fd8c23a1ddb9903cadd1ef2b9ee57421c4ee", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7a678a03f3b3dc02d5ec7b4d56ef40168ffb56515345d1d205f30b7184ae950d", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:999e3001fe6ed6386a8de4bf4264594662a06927e34a2fa9c89001c51d605096", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:ee3cf50fb8d68ea6bef597622468d35a823db698ec6e00ca81dda5394cc1ae32", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:814e1f814e46c7dd3b240338bc9d15c90a4d38b64563f8a5a4057916be671adb", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e00ed8494e0af5cc66822e553aa03a0d70e61883cea1e199bfab22e8c9c6ff9d", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d256db5e90e5b7aead4a5add4a5d575ed29fe81b94e5fd9f66633632c5736e09", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e845d338869fa2bfe3a3c9b21a2982b5e42b2d84705f4f07b84911c0c92fa1b4", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:efbddd918e33d42271349e71f8bb4b370bd3ae5c2cddaa355fe0c9bf66e4812d", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:50cc8ab8a48c5dfb32d149c8ec2481c31cf93c2a8f706a813819634f447e0f6f", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:53a25a64fec95c4da58213ad2507fb1cadd2992486e6754166a72860240152d2", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ab55d5c72d0e6c20b7e195dc9f80430d09ad2395248e90d89ed1b42a8ed604e1", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:47d79e33eb5bdd5b00656068e7400e7b2e1d887806f6254d87959c6218f7c45e", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1925bc47f0b1855d1c70944ad944dd672761edac8e26e56cd530e16ba12dc0de", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f388d03c2920f378a71056be65cef317cef3ab12aea522275bf65995e8204ba7", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:14c0f9d393ea94af36caf4c06bf135e11b14cb18471f8a16a7faa8f97bc88ca8", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:656fd33a3e7cca72433f8eb9e233bda65e46ebf0d4c957cfc36bf50e0863b877", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d4ba00bca0b2a9dfbb3c6915726723d08016e8e24310e5ba767a020deb415cd1", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5510c1697f9db60cee4a881038f13b6580e6f434568c8ad2487ae204443888b4", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:30390cf6d1e1edb3672ae25718e4e06ac7b239118f134e2a490662cfe6a21ecf", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b1c5a0c4bff1e1d5eeaf5d83a6ab08c6418973a3bf07ca132311b5c0b458ce3e", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:273d1e7fc12e47667f45697742224defe8aa3e9e059c4ced6d8db76e51fa7584", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0b3e5ab7c024975830847ecf42f6490d19c8f090fe2af97fb3c7da0f1ba93e56", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bad67e137235d7ce366edcfd296c21fd89e91d484a39ebc1be34c549865bf934", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5242a2ed6d44c0baaeb5a08b8c4ce09bf56fb180be4b52ece6b42777459696bc", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a4a8df64383f66bfd4dc2f04c4297d0511ceef03e890eb5e813756ee8e97fbf3", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8cdafdad84417df048f89a777b2667afd3da734ddb0b9fab2d03daaf7f4f5a32", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6fed12ff5c8678361d4422bba992f71dba4ba368e57930121749f1702e1b1b74", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:20859a17c7d4f6aa6c9dee6b9f4e1bd029a449cf4543f41ec609c8bbd0918b47", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:33a4e31dbb06b550b06da5bbc3b1be836e1e5a490b9078d199bcbfb43cc781ca", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dc3322c4032828ac0ec852aa929d1822880c0f475859cfbc995e902dd0aaea59", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f8d01d88896af3d9086c3819b6f49523cd8988f13fbbe6943e1b4b20e6ea1807", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:189be0936cbb7945f70365a7b683e7fbb2c735ce9b784acc0c5f067d6ee195d5", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4557749f3a0378260bc952d9eb9fade06a3df2f293e09ca20ec0797f7ffc4f0d", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7afbeaa7c3292359f8f9930d3a7d05c3f9d21335030846faa6628c8e13cdb052", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ef0a4dc87deff7b071f14a6ab2830e6582fc48ff4dcaab26222d935b100494c1", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5c35d7c77f50bcc5dd7415618b73f82df46bbd9b09b92eb664e97e66268d3736", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2b67fa906acaf9b8ac492510b024d67686c55a1303b07f9eb8a816b3ab02488e", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8e41aa7c410cda1f2c46d6180ea3c841c6f3d774e515610766438239806f549e", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:017d40b6119cbb45e6dd54f03d8deee46feee58e0fb1fcdaf936059ba7006842", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5d145aa52202add34f81d779dc55146dd94e3fa852e1beca069e240ae822025a", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:208a471fea6b18ef88a6eb8ffe78812288b2e5fa0adda49b7dc23223410382c2", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cd7ef35fc0eb5c16e3c1f9dd3182ebdf8a2d6d3ca7a9d87c3d49ed32310da7cc", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d1d69bd0ad30b64474758129eabd35ca9339bc1c36f1ea0f795b6d343a435f2b", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a93447d59bd38d0c3dbe50e3bfbc4bf71f8d52b599e0c1402ec7dcef7851c765", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8d4eb5ebc5186dafc213cacae64b7cd596686ce6e0e79d6a0f52b4c4a2d4af22", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a5cb6cb60d39a70bb940a4cdb4b0cd1f25b3932d68aca5c01cf0a39136805cd0", + "evidence_path": "evidence/20260721T154156Z-2b7e41ca/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run5.json b/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run5.json new file mode 100644 index 0000000..7472d8f --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-qwen2.5-7b-instruct-q4_k_m-run5.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T154225Z-b6deb78e", + "timestamp": "2026-07-21T15:42:25Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10075-76f46ad29", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:37574c9f6b0d49fb1daa1bb12c11866fd2947e8615d413310b24dd0934b94791", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6a0a81da92c42045d8074281d679b72c0e43efec77ab3276527cb7cf1810a1bc", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:9fd3a0020f23572210f6eef482f10d8fb0774e140fe9e2ad428555d4fd35436d", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:2c2b1abd2b5b82c42f6c050c4e827a65caebff87d961cc035f9e28dfe8584ebf", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5d168a7f89bd67c210f8c383448cdd9a28fae11f393c635ea3d6d32cd58ce9cf", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:aea98983afaec141ebe58248404b5a867deb63a879f0849e14193bb96f789390", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:d89bd31c4b735d15fe2bf52cc0d6441e248cd5ca80b91b8d40ccc0b4b57cc4ed", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e45534386b918e582ee29b1e2b75df72398f9bceff2d14ccb6ff011088fd3342", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b83b42aedfdcfa13fb0d67f6ecdd2ebbe1347e45ead694971a791a66a8f06849", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:798baa3d798486e0c98c885eae13ae40f26fdad0aeca45a77cab0e26b8adb6a4", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1b95676b7e657838f79071986b9f405dfa2d4d90a1a233be5e48c28405702cf4", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f4520650a465526611b14277f07f6e9bcbce89ed1232110aab1ddaa1e474fad5", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:db050e0ab9ca2d8bd55d875915998d82d7d1c02147b53ef28f70f5d58df4a8b4", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cf9a1d3c097bbaab5e9ecc77de57edcee1899765973b1dac0c484ee4f93a5818", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:34b8c53f2bd8518a8043c8a9d751443126c7048170eddef6d65e6eba1bbde087", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:36e7114327374906e79b34d896cf7279befa4defa878bd87e962e4cc41f93159", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb56fcf9a63f462097223ef228c4088f6e371c2cab8ad266d12a998cde417f48", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:84a1d1baf132281778c7e5c9af06895f45fb1a38c4bbd060ba88937fc05bcc89", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:333b46632a32215df1f7a5bb506f988f810d4e542222b5ab0675b8d70baeef68", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:88d2580d740a279e1a130a070638f666417b49bc67ecf980a13e5afd91cbd748", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:26de3514f8788115d3683ffb67fa15728f5b7a60d610172aeba2f7a6189e7c64", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2b7e7c93320fba6b6254c496e146aa2da7f6c04fdfc28dd6ec489568aab6427f", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:77f167740a1fdb5698d248dc3acc699f70cc6cac250136e1a7a63676bb0091db", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a984b3e05e3bd98a4a4d3a7bee4751e948909e190aa65218c9120cce971f3722", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:36009ec928c8ab4086f87ff9cdc942537f981515c62731f123c41e7fdba977a5", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9ebd67b784b4ec785b1a4729c9916a66c1d61b200ee5a2b612a7764af2c982a3", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:15c70704f551b85e27aab1b373a97582d1d5138a678fe5369c066632bf5e48f6", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:02daedfdc3bc6018ecbb71c23e1c508713430995d6f1b2029005f5c9802c3f5b", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3c81dbfbc3000c14d8900be290a5f5b1291425171b362b2275bae659714e9c3c", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb2e114585c9a1043597ae96ee718834fa33475ba42e972e1e015830f4a66b7f", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6899e10464012f96f13f0328f7488de6141685c99c758274368da8a67c9a7a20", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b92cca0e7eab7af309598f11d6e8acc872263c6fb2382fbfcc066887b5390f04", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e8d30bf560a0863de18b6aa2c35e30c0419e9ca660f2c16c192fea96f61ff541", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7de49b7a4ac637f6d6041d6af74420c418f66a72ae8f853ba5851af4224ca91e", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5bfb3e3fecc625972baaa41a9ffd6e256254008fec9115dd82b7e7676b9c00d0", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9a793a3dcda88f4e1b1313505a7a0690a181d61c2592cc2f3025c0607a8807f7", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6d13a5a9c7cfe1091af5083a22ab4a953da6c82879e9806a6c100a45ad1dd54", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8c1e4891656e7dcce645e8e611d72b9982e7ade1300d4d57597f2c63724a47d1", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:21eabb0bd0f253243faa8d9e50ea5e5b223ee18599aaff94387c3ed0c8ddab41", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:078f15381d89567da27c16e7d74784100366ebefbf4e212b9e340e8e0d8e660b", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aef43754b36a2c8374178acc4a42f74fbd21ef0c4a35d4cf2513a2f616851b2d", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2d25e9c1251fd79a46ac19bc3c52b99d2600352dc75417aae3f2fd727b0bd770", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b244be11d6a6c58b1ec4575f7a2eac5db747a827446555bc9fb1036dd2b94b75", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:44b8fd9782c18e4b0ef8c60643ac83fb3a1e0d0b439973d3920b3331bca3cbc8", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:125c71ff5959a4a0a394158bbc6ab8ff025e286ace0fc4feb50e9a949af80fcf", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:48a4c46674d2c72cad81ad02d81b666401ed98a0b54e673b501fab726addbad3", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e663489716cf4dd2baf9e5597c9771e8604834888d04db52a4a612b987144e60", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fcfa7335012935c0368144347f0b1e570b9a11494e99550b7c44fd5fc221e2ba", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8b641a1a4efaa62a3531883d51eff0703cf851d0728251bd526f2c6efc539430", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:abeeef665de7f31cc04c099a8ce977dd76f368abf9ab6f85a8cd9ddfb9a47d60", + "evidence_path": "evidence/20260721T154225Z-b6deb78e/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armB/llamacpp-version.txt b/docs/case-studies/evidence/replication/m6-armB/llamacpp-version.txt new file mode 100644 index 0000000..e4f3bf9 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armB/llamacpp-version.txt @@ -0,0 +1,2 @@ +version: 10075 (76f46ad29) +built with AppleClang 21.0.0.21000101 for Darwin arm64 diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed1.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed1.json new file mode 100644 index 0000000..e68e7d8 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed1.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T152308Z-16a92405", + "timestamp": "2026-07-21T15:23:08Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 1, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:e855ce5e845ed04ac2b1bf447a579864a42a03c2b543c8677ac7205b8793e954", + "evidence_path": "evidence/20260721T152308Z-16a92405/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_flight', got 'book_flight'", + "evidence_hash": "sha256:e77e7d27fb8e41cbbc9dd96d68da7cb1d3306afe7d6e7ddbdc9e31b420a9ce4e", + "evidence_path": "evidence/20260721T152308Z-16a92405/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:bef41d74f4ae2e0092fa5a654d90b07f28124e907427c4a3d21d4714c3da131d", + "evidence_path": "evidence/20260721T152308Z-16a92405/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:7edc30f0d4e8fc7a2eef08f20d8cec90d6e01bbefc32b6e7041fe08b7247a48e", + "evidence_path": "evidence/20260721T152308Z-16a92405/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'latitude', got string", + "evidence_hash": "sha256:e4dffddc7d07a2e3ab3ee9e6ae7ff280692248c77886c06d6cdfafc2c94c43c2", + "evidence_path": "evidence/20260721T152308Z-16a92405/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:b38e52b934ca851f7f022dc6e3df6949cee6a93bcc5ab1081cba76b2e1af8406", + "evidence_path": "evidence/20260721T152308Z-16a92405/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:26fbf10d087822deb98a4f5d0a1774af7dc55b51ff9646ce9b81fb1e15a796a1", + "evidence_path": "evidence/20260721T152308Z-16a92405/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:6077194c511f7417d7ce0a3ce88bb9fe695a19a4c45af473d5d59fc2fadb46ff", + "evidence_path": "evidence/20260721T152308Z-16a92405/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:5d6a3c9ac721c3ad677693831119e6342e63a4db56aff11fe8fd90064b037504", + "evidence_path": "evidence/20260721T152308Z-16a92405/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:534d17f7fed1575acdaf9ba1110b68c4b7470b03931b43f3ced5606f33424912", + "evidence_path": "evidence/20260721T152308Z-16a92405/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:9513cd1fb47d518f32a2a7cb2287a240ac153ccf2fe951d8e38793de507a1e02", + "evidence_path": "evidence/20260721T152308Z-16a92405/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:39cd32e3fb5b62a21e3aab7020a85bd2f4c5340f1f1cc4bf0dcf15cde00c936d", + "evidence_path": "evidence/20260721T152308Z-16a92405/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:e414bae66eeb63043202db98e8e8177e9175a0e443de4965a9594b3ffc8001fd", + "evidence_path": "evidence/20260721T152308Z-16a92405/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'city': expected München, got Mnchen", + "evidence_hash": "sha256:23cd4b5f98b9b587ffa460107049218d8fb9da7105a1a593f6f47b96d5f2e200", + "evidence_path": "evidence/20260721T152308Z-16a92405/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:869feb78f6bc25dfcdc260cd4e6884f55b24e6a99038bd3ae67a713245b0de1d", + "evidence_path": "evidence/20260721T152308Z-16a92405/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:91ea1915a99c2b2bd955af4e57e2c572ecb2670bee0763c67f590d5ffcd954cc", + "evidence_path": "evidence/20260721T152308Z-16a92405/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:24cb15656abba7abccd62e330e4b586623de3c4cdddbd0b822fe3c5a1b30ee1d", + "evidence_path": "evidence/20260721T152308Z-16a92405/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:4ecfb41714a5d72af0484e13ead3860a84999615c78d6d21401fdbe8f3421fd7", + "evidence_path": "evidence/20260721T152308Z-16a92405/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:85335f9459f717a546548b4be11c0ed208e03062979e6719a3740e42a787b9c2", + "evidence_path": "evidence/20260721T152308Z-16a92405/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:97b0f7553f799a3549ae79c1a8b554e43c063f74f1098e1bfd41a5fc29578cfd", + "evidence_path": "evidence/20260721T152308Z-16a92405/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:0a904367a2f85753c2b818c1de47e4f430ee9f2f0c2e56a07615f06bd4698fb1", + "evidence_path": "evidence/20260721T152308Z-16a92405/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:16f214f47f3f589336ece45339516d92da82247f4d497f832a44c6dc3e280571", + "evidence_path": "evidence/20260721T152308Z-16a92405/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:edfec1728c53fd75bec6d99fabce0a347114329578b5e194dc4cc5d57eb48232", + "evidence_path": "evidence/20260721T152308Z-16a92405/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9ed7ba1841d67cd85cd0d093fb2a96417a8e3140d8801f01704352b56ebfe08a", + "evidence_path": "evidence/20260721T152308Z-16a92405/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7e49c55b19b9409929bad8f1b82ce293894ca1233fb9a94b9ce8e44dbd72445f", + "evidence_path": "evidence/20260721T152308Z-16a92405/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a495c20e306d5e28106eef03b3715db659db64b701ddab21d5bb362cf178979e", + "evidence_path": "evidence/20260721T152308Z-16a92405/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a501342188a143168060c7591621cbc56a15cbb40a1fa1061364a8137ffa04e2", + "evidence_path": "evidence/20260721T152308Z-16a92405/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4cb9fc96a649804f7ce8bfe649dd171774884295d3fdfb74f4c6c9d7cbb23af4", + "evidence_path": "evidence/20260721T152308Z-16a92405/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:65a48bd9e134bae3af2f5186fbe6ce2b69d23b25318bc6fabd553ce751018295", + "evidence_path": "evidence/20260721T152308Z-16a92405/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6d8c914511ebba1fc7b8700c8ff6b6b73d93d096387f08bb29bcf5cd47c44636", + "evidence_path": "evidence/20260721T152308Z-16a92405/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6738b8ec9e2f6599e15c1fa5eceebe3baecbb46038dbc078075dad67b76ab5b", + "evidence_path": "evidence/20260721T152308Z-16a92405/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0becdb1debc0cf44e5eeab8e4056ec0ab546d4664cebce0a165dc3a3c856ada3", + "evidence_path": "evidence/20260721T152308Z-16a92405/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:fc2efd3c61de203068a22abde969cfa6416ada40afdd01eb40e07d205bcf7c9f", + "evidence_path": "evidence/20260721T152308Z-16a92405/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:22d3549306b2356a87269e6a9256b8e093e133179f650361db06a8dbd98cdd42", + "evidence_path": "evidence/20260721T152308Z-16a92405/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:359eedd93c52dcb9356d7169a94821a2be019de7737f7f28fc6fd313ce8e4ce8", + "evidence_path": "evidence/20260721T152308Z-16a92405/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c42750cc7ac27dfd0c3fd4c7cee2ef4a25089fba29255c1e024ccfe5fd38c8df", + "evidence_path": "evidence/20260721T152308Z-16a92405/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:ae3202cbfbd3ef373301cea8389ea709a13344e9e56aa5f8ff7adff8e38c9c46", + "evidence_path": "evidence/20260721T152308Z-16a92405/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f3473c6bb6c0f1bec3030945a26a6ca8ba12648bf73bf84f84ef69b5315d7cf5", + "evidence_path": "evidence/20260721T152308Z-16a92405/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c23b13fbd66a3cbd4a851e98f0eb1fd2d2361df286a06b555a952b81ad6009ef", + "evidence_path": "evidence/20260721T152308Z-16a92405/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: missing required field 'name'", + "evidence_hash": "sha256:23702809387a5ddd78304a27c3eba9fdf2e6d6d307d4a2a49b67be6085e2d255", + "evidence_path": "evidence/20260721T152308Z-16a92405/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:3f4ba693b749c38f229c288c58f560f9f76533748d93326b36b64a68cb56b226", + "evidence_path": "evidence/20260721T152308Z-16a92405/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:f91513464784577da86a4afbe5906103ec560d42ffad9e5b25b186303f6cb43c", + "evidence_path": "evidence/20260721T152308Z-16a92405/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b866e2da7799492fd4878a3a807a5dcb3d41b19124f1cba02b0ce1052b3332c6", + "evidence_path": "evidence/20260721T152308Z-16a92405/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ec9c11098f568aab917b3719fd330fcadbe3ae77d9377ea1c38ed8eaa9dcb1f9", + "evidence_path": "evidence/20260721T152308Z-16a92405/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:78c1197acff86544ef09b6966ebc1a3c395e68cf43b91d48f1592ea63a754874", + "evidence_path": "evidence/20260721T152308Z-16a92405/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f3172a29b54f6c9413b21869c64d3451762ac9f6f72b43e808cae34a0d93a54d", + "evidence_path": "evidence/20260721T152308Z-16a92405/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e64c5e5d8d8bb185f098bb82c29e4be86c54b7580cfe8be7a44cef21977dadff", + "evidence_path": "evidence/20260721T152308Z-16a92405/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a6b1414ef776b4b3527464bd37bdfa9cdbacfccaab7e1ac9a0c18580b3ef1d81", + "evidence_path": "evidence/20260721T152308Z-16a92405/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c512621d8e539b102cdbad48770d5fb1cde48702215e010aa6a9783ad796b7d1", + "evidence_path": "evidence/20260721T152308Z-16a92405/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:90506ff43851ef3d7edf4211a236c13ea4d12986aa76a8891d904eda62386aa5", + "evidence_path": "evidence/20260721T152308Z-16a92405/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 24, + "failed": 19, + "errors": 7, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed2.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed2.json new file mode 100644 index 0000000..90e5dd0 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed2.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T152347Z-9e1e2221", + "timestamp": "2026-07-21T15:23:47Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 2, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:fddb8d9a1a841918ce563877f357696d425e914182921762121c292151d2caea", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4c1709d17cd6a2e42496be0312989ed80dde954996263869b74e2bfce3c73828", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f18c3caedc3546ac9cfd2ff282eca72ecb2fe72a0ce9ab28621c55c441560cce", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'first_value', got string", + "evidence_hash": "sha256:9b1c61e6a2c21733c610f98aa07d27b38434383aea1f4e86f69a169cbfe07001", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:15f3cea1b1ad57c10cbef80fd93e75f5f57c3577acacab0335c32fda9547ffbf", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:e24178ef8641a4f1e727a03970f669788f8972f9b9276e74440891e38fef695b", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f631ce76c3b5cdf0e3cbb80e1c26d440a54f69dce04b36a8f3ac4b199154f5eb", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:8ebdaea4ed76c2b9f8145d7e1809ddbeafd8fe57a77805cacac77e3aaf6b62e0", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:c10046d7a9495eacced6c49bab8c0a5d39d34f1493453f1358e8c0175fe7746c", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b7a5012dfa8c9ebcebcadbbfd6263b7fdc19222bf2dc4456c7afcfd386ea92d3", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:a6134e9f6ec7067f146819c1251157ca7e506ac3f7f8334132d2a838602daa38", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb484787afd8a5a6ba2faaf13de1533530a9ed3b0442688e219fcc80385afdbc", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4e036a17813328ad962aec9773c93e2e9a07c374254acd9866838218485f98ca", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1b6007bac68252818ca27ee5563b02418f122eec8dd5660ec66ea10c8b2f6c01", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:c75db41ae144fc9c0084537894281e4422a57d322fcab1711be6ae5f4ad010ce", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:a53e58ecbdb3bc7c3ac8dbba8de7d257134ad15c9156ceba93ed176a768387b0", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:3c67b01ca58cd17387e1f81d33534b9c86cb2cf130b2b7960a5aa87661e0bbff", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:999a61970a7eedddcf87d23dc2d4ebc34f999eea05e07a67fde43f28f736d2ef", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:ada1a1582881edb20fc90b5945343d9a1d42c85e5ab7cb5fc5191ecadbbc541d", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:72cab73315db8ae5ea76f027dd3bb6556ab24ff71883030fef6676988a1fa639", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:a7b18ca95e7ae2eedb74cf57e1106a84e48e87af665f9e919dd34f3a3a160fb6", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5020e93d797de48e4bfc35b37c95b51e33ede1f4808a9fb69498e8e416dd47c1", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:200c3727c178844f168a866bc399ec36aea4dd5536cd321e94c945f8173b2a8d", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:cc6accf2c36e5b7c4933e98d5e24cd58fc38d0dd596d87613982571c8ae62067", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ea73b4bc734da3dfb403d9e173d629af844180cdad34a95c8654cc34f3c47ef2", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:39b400921b95d16dac487066b81bb3715607fff7e2da8577b5a0e2038d6c2a97", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:60ef31f7bea557acffffd42ecf6f9d2ceb460041944a6094e43bacf09e738d59", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fa0761c88e18772645b809b46f2a7d1a1d52093f27d489630575f649993b2858", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ae2881a976c7440ef3e227fd3ee38cac7c867b760a2634075a021e9dc925aeba", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:bc015b34219eb1b7ccf7f729211555e8d3a9c19d6eeb0459734735d976fb9110", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/single-nested-options.json", + "retried": true + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:35f050f28dbc6142c11f5f33a3163caaeafa2671a67bd8c8113738c1fc167c53", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/single-optional-omitted.json", + "retried": true + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2da0918510d0f2d82fff33676b33f5ae841c97ed7c81561e1bf01a8590285f15", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7549584ce2be3b3a882f64e902c71857499a5973ac8ccfe26cafed631e49c334", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7cbc6565ff1a525f53502f1b8e3e5f616095d54a65b31e58260c71c730a649af", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a643b0a8d22d2a7f3c0942f24379914d4b68c7f327e7f9e24841f70918d35dc9", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cc1d919ac6a777019637381c04642028bf8f2c0330baad3f9b3aae526c7e05d5", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:163bc82e7c68670e3374703b238af3c1334f5135f7dc66e906399558df401638", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:19c6757f8174e39949d79fd399c8efd3d859542ca0fce8fb065eb033ae013ef5", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aa27d1d01d2f3f264eca3b7eb7beb93c68325259930ae0d31f27f9bbfb7aa5e4", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "wrong value for 'name': expected origin-shift, got marker origin-shift", + "evidence_hash": "sha256:f7446055ff14c2e296d6c9c04837787b3b4b6fa8eae46df0245e3b5ccedc99b4", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:2e4c702e601aee4370136a449f7b741007ec890b913b25d847b90fc80d6c23aa", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:d7b9bd5e38f760b3cd5e154fb97d13d5c44406d7c816404c984c65d1a7071b38", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b049788b0e4a5966747fe81dd2054380781e10e30fbbeb567daa4a9b0d8be35b", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ad98931615fdaa1cbb6a9ce891485217c8259a4755e676f997a5b1329820bcb0", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:3b3433f35886dde3f3cad80bfb659e6c7b3bdbe513dd5061df1f0e559e9436bd", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:71f8dd97171bd6fc4eff58f1bbd292807d02dab818ffc4693c3369cb5340ead0", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7bf17eeaeb7074867419574db3d8b56f9dbb3546dc5c57e73b5f337d5c925fb5", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:88b415c45f5b103b52bf8b64eac8be539a27247913ea3e4d2443b05fa9a51e32", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3b68619c9358323f7329a5018631dbe9de502a04f56cff9d41ebcfaa4d20925a", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a16443bb4cbc73cf5a7659c240d1f54a1794c908a74026bb17b2d6d1cb279924", + "evidence_path": "evidence/20260721T152347Z-9e1e2221/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 23, + "failed": 19, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed3.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed3.json new file mode 100644 index 0000000..87a73de --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed3.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T152423Z-8787234d", + "timestamp": "2026-07-21T15:24:23Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 3, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:39d219afe7c94d11d6f09a5ce65f1060e7bd2cbab399a4baceca8321a65c768b", + "evidence_path": "evidence/20260721T152423Z-8787234d/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:8215d9a689138a86790063c6f3e2d224ec8df3f5557f7ef399b852f73767239f", + "evidence_path": "evidence/20260721T152423Z-8787234d/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f8c97de0f15ca5f1c8d108f03e1d2b293dac1f19064e861f89bdd5c4256e59a0", + "evidence_path": "evidence/20260721T152423Z-8787234d/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'first_value', got string", + "evidence_hash": "sha256:710205da66ccbea9f40c7441488a0b5b7be94da79c1e7551294d406875ed502e", + "evidence_path": "evidence/20260721T152423Z-8787234d/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:536d3e7c7a0fd30f40f8a76ad2a3e002dba101f5b1a2d7eb5163050e5d69a338", + "evidence_path": "evidence/20260721T152423Z-8787234d/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:93eda48550c69d51c101d2759c0b094934c729e6cf703a0051c2a8f0d0b308df", + "evidence_path": "evidence/20260721T152423Z-8787234d/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:71d786d61b1e32af0f73c94e415c59502e5094e6707365f880ccf6d44989dc1d", + "evidence_path": "evidence/20260721T152423Z-8787234d/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:b34acdf37b50f0214bf499f4b43ed670782bd7dba405c43350b2cf28ddc4e7c1", + "evidence_path": "evidence/20260721T152423Z-8787234d/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:9edce90860c50480481ec406acfbf0cbf57591073e727f54435a7fa07db56b01", + "evidence_path": "evidence/20260721T152423Z-8787234d/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:89cfabe09e7ce13c3798a964568a6c8b28c43460afaabdecbfc13cc975ed2333", + "evidence_path": "evidence/20260721T152423Z-8787234d/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:4510ea780f11be04ed021ac3e974af26e0b4fcfb79f38a20b188d45fb5af7985", + "evidence_path": "evidence/20260721T152423Z-8787234d/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:09feb7ad38ce88575c35c5caf946ea31aa5cb8fa64a21c8de86d5c59e3095cc8", + "evidence_path": "evidence/20260721T152423Z-8787234d/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f4dad6078d3341870e6dc222b2e2dfdc8b8f0ad51a0c7685109a91ba53b33423", + "evidence_path": "evidence/20260721T152423Z-8787234d/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ec43ed50fc52867907decbaae58a573dfda5520cb5c2901e950e0c60342a57fc", + "evidence_path": "evidence/20260721T152423Z-8787234d/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:b77ab53859db161f520cb46b0c4b9f85af8021e66b1f2584552e74d875025c83", + "evidence_path": "evidence/20260721T152423Z-8787234d/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:dc2e60c2634a2bc36ee5066969480cac223ff55d826afb03ca0e986816480f59", + "evidence_path": "evidence/20260721T152423Z-8787234d/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:021561cb5c8239fb2a720917db06db244a7bb656c968eb716648336ba90680b4", + "evidence_path": "evidence/20260721T152423Z-8787234d/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:dac730ab271f853d15bbcecb490058059215e488041954a2d3c0416a7e05e148", + "evidence_path": "evidence/20260721T152423Z-8787234d/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:87b6f2ac8a1bc269743f63702991f9e0c6201b65e096b7726109f7e6f8b1eb06", + "evidence_path": "evidence/20260721T152423Z-8787234d/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:c8b89a4b52808be41f5d7386d490e1e5b268fc5764a23c92eadaa8749755ecdd", + "evidence_path": "evidence/20260721T152423Z-8787234d/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:b9ee7cee0d06796275f1e539fe7458a6cc9e93ce714ce2d01f9d80d33bd4d164", + "evidence_path": "evidence/20260721T152423Z-8787234d/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:321600edc84e668114ae9a242a20595e23d2c796ddf417a5e30e9f2133071adb", + "evidence_path": "evidence/20260721T152423Z-8787234d/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:5becd07ad59d99e5a247fe95c4408a3138b217439fbb15ef8667827fb5d319eb", + "evidence_path": "evidence/20260721T152423Z-8787234d/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:44f7caf3d05764b7637f5331bb8525e91cfa833cac397abdb25f8c64539dd73a", + "evidence_path": "evidence/20260721T152423Z-8787234d/single-boolean-flag.json", + "retried": true + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d3e0ab4919d0ab15560da49bcc6de62b337099292365d8b64d9429d41f6ad361", + "evidence_path": "evidence/20260721T152423Z-8787234d/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:72d95addc57d288a465b2b1ed0ad25070da019b185328af5e5f1d0cf915ed0d6", + "evidence_path": "evidence/20260721T152423Z-8787234d/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b7ad8e68cbc284e0d43e80a59d83ced9264e6744d4df7843b0a7e78441a1aa15", + "evidence_path": "evidence/20260721T152423Z-8787234d/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1b85a41206a6840ca5a0e2696d45990f455ad0bba344f10adc055c26b46bacb1", + "evidence_path": "evidence/20260721T152423Z-8787234d/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b847a70d9a45270e06d536c0514be2ab5777c98c154050b44381c7577f6e1681", + "evidence_path": "evidence/20260721T152423Z-8787234d/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:104a6e6d222993acbc4bf2dd0ce96c9fbaf45456592fc99749554a4dcd2e8f63", + "evidence_path": "evidence/20260721T152423Z-8787234d/single-nested-options.json", + "retried": true + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got null", + "evidence_hash": "sha256:b007cb4d784fe0256121c35da822c2fad9435704ad6dfc19449b775c7f6e068a", + "evidence_path": "evidence/20260721T152423Z-8787234d/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e64a30195668ad3f01ae81f0670134e147da0b04a05e1a321dee9c9e6abdc20e", + "evidence_path": "evidence/20260721T152423Z-8787234d/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:66e87cbca45af79b417ebc577fd08f8035eaa91ac6441ac05346874a7a0c21aa", + "evidence_path": "evidence/20260721T152423Z-8787234d/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ae552cb510a30592292a194baf9b140b6d093e3c320326714a186253c512bea7", + "evidence_path": "evidence/20260721T152423Z-8787234d/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb63f816f0377b099a0559339b792d454964ba030ebf80c5ba14d11cc7c1ce12", + "evidence_path": "evidence/20260721T152423Z-8787234d/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0703507d9ed3f8130b82b829ae2bca26e293f9a64e66d7d971448e7a529ffd4d", + "evidence_path": "evidence/20260721T152423Z-8787234d/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f6536be0a1014b1b2f6b9eeb5eb408186451a58853dbbac8f5d2a3b3c80bc489", + "evidence_path": "evidence/20260721T152423Z-8787234d/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:76ddf2dbfe033898a6a2381446631920de0e22ab814581785a8248a6e7996322", + "evidence_path": "evidence/20260721T152423Z-8787234d/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a45a3363da74076aba46b218798d002effddda05967624aaedd3a78a9b471464", + "evidence_path": "evidence/20260721T152423Z-8787234d/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "wrong value for 'name': expected origin-shift, got marker_origin-shift", + "evidence_hash": "sha256:d39aaf7a04cd4af61d4d68adb46eac5126bcea50ed8f13674b5b1db064e48e92", + "evidence_path": "evidence/20260721T152423Z-8787234d/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:b8c9286e0a4323f5b3b2398ccd9a72a2a514080ddb46960f5bd75978ba484858", + "evidence_path": "evidence/20260721T152423Z-8787234d/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:549830317d5c1c1090288844d792b50a9d47adec532fc6f82f751e61adfc8252", + "evidence_path": "evidence/20260721T152423Z-8787234d/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6fc2ee363bf42d20185b893588625984c8285c94a22f49cd44252d8adfadfd96", + "evidence_path": "evidence/20260721T152423Z-8787234d/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:02b5f2c51d4f1ff5383a42251a045ea0d6a74b33977e7aefdaf1fa21ac4d22cc", + "evidence_path": "evidence/20260721T152423Z-8787234d/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:6b171a9caa7bc382dd3fa07c0057af5e48a2fdaf052b7128e9982380674474b8", + "evidence_path": "evidence/20260721T152423Z-8787234d/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:be243511e601e7fb7d53353051e51c69e246f5ce3e4d370c9aa887ffe3ab9969", + "evidence_path": "evidence/20260721T152423Z-8787234d/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:da39166b6a0c98b693cb3fe8e0f2542ae8891b01173e3f3e4974e256aec0b3c9", + "evidence_path": "evidence/20260721T152423Z-8787234d/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f72e40234d27c76ba6db1ffe8b410cd13ad4f6a4f87e92564a56a609f0d17963", + "evidence_path": "evidence/20260721T152423Z-8787234d/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3c44a2ab5c27dc428b5f1829cf8d785fc30d4fdf2188fe47db749ae62f765bf7", + "evidence_path": "evidence/20260721T152423Z-8787234d/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:92eb3c6119ff2b76ee61f1dbc0655908a0eee9b98b34b205d8f3ec21104f9997", + "evidence_path": "evidence/20260721T152423Z-8787234d/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 23, + "failed": 19, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed4.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed4.json new file mode 100644 index 0000000..8201e08 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed4.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T152459Z-4d8be60e", + "timestamp": "2026-07-21T15:24:59Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 4, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'get_calendar', got 'create_event'", + "evidence_hash": "sha256:d0f54fd4ddc499755b429726e9f43456c0acd2252503fa4b45d6548da22bfd82", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_flight', got 'book_flight'", + "evidence_hash": "sha256:6e9bc2f44036f7cf1809298c02a97345e50e483aa381e708ced8b19c38a85bc9", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5f45b0093d22ad3f52202a074d4773b05992f85a9f580a31593e7d0f8357656c", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'first_value', got string", + "evidence_hash": "sha256:cbc1f437978c44981d468e832d85f8220dfe929ad5874acb36ad80fbb2849fae", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:e8ec5f554b644e79ec361ff1d933436bd93301e911c615e04b0fb8a885ad8ca0", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4ef980aa73114f44d3adb132eb8009fa5b82cc909c2fe623ec70c1c39c3c15a3", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:1ce17b0dbdbeaf1c053e50bee26841f7ca209a0cfe6cac070636a842bbfc490e", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:88a58052587342ab2ca8ed3a5c1b7307e97a57b46a209f823bb912e577d62ea7", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:99c93229f47404c457f60b08edd0ca19e06a962031114bf52ca0ed0d6f917b93", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:41fd0186b4e48e86bdd5a1828d7fd2f5e42a4e33641caae913cd42296c093de9", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:175c5346afdd8babe2bfd173eda07e11c7fad3fbf58401d4fb4b10c31142bf82", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/negative-invalid-schema.json", + "retried": true + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:33d7d68f7c4c1774c2de9b32142ad2d3f21678589d3e9514ab1c8724dd5a2420", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:4e463f64e15e9a6fdd9540d439370df9a3eb2a59f60a8d73a1ea33fd38305f9c", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b03f3cd85b472850577d66c49b45a3c1899049da659de2e5f57c1c0b8da009ce", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:a3e9a56f2116ef5a731b6e94e1e003ba72f05d1dccfa9bb90f4684f672dfb26a", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:81683e748d1f8f47a273f7f5226dc337b9af6cd51491c6c0801261b8f486871f", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:5f18d116243c6c56dd522f64d23b1532ffa0a15122f8b71865fa366575e3d6a7", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:786d7b2cb8066a208ec287d4163b7572918c5db539264a12d8613858ccc0c46d", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:c08355b65e411b83fcf77621121593bd84d84ff585477ee192bf6d48ae0e0882", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:2885172a6f37ee24d52a29fe3f89c04c9ba53324b7e53c8f813ab792712fb581", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:a73e42f543dbd16373395dcca53efab3dd3f8876a5bb284d2695b1602b54f3cf", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:337eb998c4315339245930c6fc583a866e552d31f1d6e21821e08b7399330c93", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:218c94bbb67d998d1f34b60b135612c78b761ffbe1f17876cff86a9c530a9408", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:43b1c46fa64d7250b3fe1309212da5d9f98af721283d840b3cef77881c680414", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:044162243a05997f495816535f903ea8aeb836ef13932fe4248f058f92817ef9", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b8e618761dbdce7c88c9f9b79646920db43b2319aaa7ac64613cefa332cdb202", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1633533c7019287e6bb63df91db5d9e9d4ad4b83c30cea954bf1b5e31c449ffe", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2ae1603c4c9024d3f1f8de3f9bf5f14bae79d8187f6cf0979201634604f691de", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0cb1f8b36c562bcddbb24fc7342536cbdf5aef7bd4ee9e2f8e9e726ff40c1993", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b30f4431e28f5c360ed04d5e1002e73776b5bb25fd19930acb64972d930eed4f", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4d1282926ca1a2e22e7e46d02f7c3ab28c574434359dacad40e58d077ea42b54", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:84b267465f4f384bf318b1b00c5486803f53d97875a398cc2976dc321f8020cb", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4ea8a56e5bd98988e105d94ad20c4077a645a26ee16fa34d399150ec31dc49d7", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ad0e8b6a4247cd3e7b33cc8c744e19588a0a57427f04ba35fc8ba6157f7a5953", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:50c4d230315004dce2cc7aacecc743f9e55882cc95f58b7a7590382d66366a75", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:03b91a63f0ae119fa65507ca1168ea45b57f21fff2577785057e0674192d8da6", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1f4c196ae803c504e544e492ed3f4aad2d423b9a65d5a7169d2ebd8ca3dc756e", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1a540cc379bd1568eb4cc04a46efc606c21ebf8ca3a047c668b77b1c7b28eec5", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5a8b0c679c3417c56342eab818990f0e3d8a6c9ce9a4a8d94d94fae0ead49db7", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cdb9d2ce5577e51125d7d63df5335cdbfb97607e0dafbf269fe1250c29ebf938", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:61da1d68c4ff3c0e69bd47a5ef519a39dc0ba3528a89a6b7563d2613d2ef3620", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:4b7c73a77c086606db330d31859e672d7e5cf834ab45ff6eb0ad5e619e437936", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9436d59a5e92d499c68c960bb8ca378ad71003843a392c239f5fd8565b7e2c98", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:51c30b01f7ab5235c419c00b6ab9bb29341bcc5a3526d7efc3f747e1bb1699ad", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:62059b916b6b4bf3013540f16d7b993dac473249b62ae1caaf43431589254b50", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f76efad18fb4eec89dbe9bf96aa61c235fa3847aa290d172317184ecbba5b81b", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1c81fe0a64943efd4b24405c20235d87f74de50554e49d323ea8540b0e9a5126", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d187b3acc8cdb69bf5818d1a18b7b60435ba393b6ff54f1a30ac3755aefb9079", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1d72a2ec796b3f1e2494e3f0c248950019234ec8bc115f00d2f8fe7376875a49", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:50a8a740a1fdf60ca88140981f4767b88e1e7b790c9b1695f2ed4fffa2e9565a", + "evidence_path": "evidence/20260721T152459Z-4d8be60e/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 27, + "failed": 19, + "errors": 4, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed5.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed5.json new file mode 100644 index 0000000..31eeaea --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m-seed5.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T152532Z-35e81aef", + "timestamp": "2026-07-21T15:25:32Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 5, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:1e6d7f0a4dfd12d4214f45736e1395ad5a61338daff723b14b682f402e02afad", + "evidence_path": "evidence/20260721T152532Z-35e81aef/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:6e87a6beda734fc3cecf1707c59a5e5d0305448cb989985d7e5b6fbc63f00413", + "evidence_path": "evidence/20260721T152532Z-35e81aef/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:fb1e4c045ed6c8085d6ee7d0e8574ad1205b00d91bbdb361467ce00f30be0656", + "evidence_path": "evidence/20260721T152532Z-35e81aef/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected number for 'first_value', got string", + "evidence_hash": "sha256:108c37a752a12cf22c99d25a186cc26657bd96417d368a4fb9bc730f8d8a380b", + "evidence_path": "evidence/20260721T152532Z-35e81aef/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:51a31f6b6c97e0839c41d8f9a7f29588f262afa2ef11b7f5236a892815710107", + "evidence_path": "evidence/20260721T152532Z-35e81aef/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:62d47a04024d6f1ec0c871d0e6b35edec3fa9fcaa48af4e8191dd880072154f1", + "evidence_path": "evidence/20260721T152532Z-35e81aef/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:3679767fb0eabf1ebee5d5fe05c872ea3d6c3b8f9eaf5d03603f6ff36e0b1a4e", + "evidence_path": "evidence/20260721T152532Z-35e81aef/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:6c015054aa3780b5408a39e38991a43b5e662a4f7e91a1b7602021f11b15f332", + "evidence_path": "evidence/20260721T152532Z-35e81aef/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:2b34266d628c569f52d5d0d048943107706415f3575110a5754352356da0c386", + "evidence_path": "evidence/20260721T152532Z-35e81aef/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e004ae14ce589d650ac4b67bb135b2fe37229f8989392b0e500f88046595a6bb", + "evidence_path": "evidence/20260721T152532Z-35e81aef/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:fd15c2c8db728ba7521692db56e43e390f10928b38374ca95580d52928c0d560", + "evidence_path": "evidence/20260721T152532Z-35e81aef/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a889145acbde3133091e0670ed407b6770417e244e6adef8e7b80462a2a620f8", + "evidence_path": "evidence/20260721T152532Z-35e81aef/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a58519e675281352a7c5cf921f616d7d31310a726de58db420e78d65a4e0c955", + "evidence_path": "evidence/20260721T152532Z-35e81aef/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:af224ddfd2fcee0c4a1e1c2698936968aff448ec4ba5763b74081983ad769e4d", + "evidence_path": "evidence/20260721T152532Z-35e81aef/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:5952550529a967647c67c3e05d281924033545d01d45eec117007cedab6b2caa", + "evidence_path": "evidence/20260721T152532Z-35e81aef/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:605d9ad66f447a07f9721ef74a591ae97b7737ff8104d046d33f0bfb0b31da8e", + "evidence_path": "evidence/20260721T152532Z-35e81aef/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:987bb143d456e4a98ae1c255d4c41d0cad6dc6c8b96dc284e7037897d9b4b48c", + "evidence_path": "evidence/20260721T152532Z-35e81aef/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:16b942f56d5e73cb733633ab9dbbc0f2cb8e7f21db3913d8aea96e1507a59163", + "evidence_path": "evidence/20260721T152532Z-35e81aef/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:f676b110c592adfaf7817d077fe9f308aec90e08bf94d8beaa9747938fa443a7", + "evidence_path": "evidence/20260721T152532Z-35e81aef/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:3487c7439d0e6b1f71de035fed5d839477dbe7f622c2c8db894896c93f6a3345", + "evidence_path": "evidence/20260721T152532Z-35e81aef/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:14dc3bb57fbc2933a20bcb16636a8c5c15a299b6ba9c16981ca325e1c297e0dc", + "evidence_path": "evidence/20260721T152532Z-35e81aef/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:7769713ecedc8d37afb9dd5f9725b7ee8fc451ff0f6dd9c8666d4f42ca3e6fa9", + "evidence_path": "evidence/20260721T152532Z-35e81aef/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:62755cae3f112fe3e66672b066d608bbc0a25bcf848b770dffb9778d20baec9f", + "evidence_path": "evidence/20260721T152532Z-35e81aef/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a631da8e88c81a962e04ab32298bc54a3a040ed127f98d304ff2e25822cf3e96", + "evidence_path": "evidence/20260721T152532Z-35e81aef/single-boolean-flag.json", + "retried": true + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6fd0299843804249d599ccef7138af94386ba78df2f2b6226cab7f803be6d485", + "evidence_path": "evidence/20260721T152532Z-35e81aef/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ccd9726ace24f550cdb45a3cf149d2b80afe8e4090bbcbb1ffcd7936eb01a25b", + "evidence_path": "evidence/20260721T152532Z-35e81aef/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:03e71e4ecff0f68050e1febb5fcdae4f9009f8b0002fae9c04411da7f72c340c", + "evidence_path": "evidence/20260721T152532Z-35e81aef/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce72a399dd892b42253b136b54c36d5f74fffa2f4c7f2840869b6218d8c2b75b", + "evidence_path": "evidence/20260721T152532Z-35e81aef/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:592fffbdc6eb047f53314d263ddb27cfc5018816fbd80569c471e9b0acd3b1a9", + "evidence_path": "evidence/20260721T152532Z-35e81aef/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f218ccf31f9b751f2cd85ced4ac2afa31d6fa752f095f1a3df82198ae30cc6ab", + "evidence_path": "evidence/20260721T152532Z-35e81aef/single-nested-options.json", + "retried": true + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:7e90c908187c62a84e60fb3235a3914834e7842a0bc0e8af449cfeba6b02c960", + "evidence_path": "evidence/20260721T152532Z-35e81aef/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb1e03c713ce6e66f5562725a9c2b9cf44df91f3e0574615d9473997c7a6176f", + "evidence_path": "evidence/20260721T152532Z-35e81aef/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1628196b99ade4241110b7978339f760ef8dc91e5e072d828cdd495841d0065a", + "evidence_path": "evidence/20260721T152532Z-35e81aef/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3eb83523c168db15da98c2b515f773fa2854341a4b820bebbafda2198b29066f", + "evidence_path": "evidence/20260721T152532Z-35e81aef/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6c042ea71eaf82ffd6425cee42850a65a0289d8ed30dbcbd034dff82690987b7", + "evidence_path": "evidence/20260721T152532Z-35e81aef/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2d3a26429eb980883679f774afba21d65c5e58f94203ef8194d90c81ddb8ec9e", + "evidence_path": "evidence/20260721T152532Z-35e81aef/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:cb16aec9bc38a319d16c938e4b13387736995ca75ffb498236c6a76effc01baf", + "evidence_path": "evidence/20260721T152532Z-35e81aef/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ae787f016f68549c9e5666597557a453f7289e8b9b847438eb7a31d5b0d50c30", + "evidence_path": "evidence/20260721T152532Z-35e81aef/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3bfd163017468adf29445e66a7efb620753f212939e4097e6cebb4b9374ab9ff", + "evidence_path": "evidence/20260721T152532Z-35e81aef/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "wrong value for 'name': expected origin-shift, got marker_origin-shift", + "evidence_hash": "sha256:7a5e7530fa626f39069b2b97aa681e8c0c5b4978b500f15b363d8cc137a61949", + "evidence_path": "evidence/20260721T152532Z-35e81aef/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:b91ca34e91f4e58311710cb5fd840d1481d1e619775173c0f241fe6ef1bd0bd9", + "evidence_path": "evidence/20260721T152532Z-35e81aef/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:f5a34f8c1be26e50d92c81356347d899b31363525d4adb2f276a54ad7ff7d792", + "evidence_path": "evidence/20260721T152532Z-35e81aef/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a846123f5a2c2bb26567e84a55dda176bb77b687da15720f786728f80c2f8581", + "evidence_path": "evidence/20260721T152532Z-35e81aef/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d8adb3e25fb359b0d487011b8145002e68f2c0724f07fad039c9161dcd1dd319", + "evidence_path": "evidence/20260721T152532Z-35e81aef/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:3f084a3107ae0831c10d783b16c1b43ea0b5b619ef4e007178a3e76484c4ad62", + "evidence_path": "evidence/20260721T152532Z-35e81aef/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c68b99fd0656bbd9c62cf1668e850df8ccf42a1f278fa3e7fe50aa5bd4d46070", + "evidence_path": "evidence/20260721T152532Z-35e81aef/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f9ff0050079783c986e2fd7d4ee8a9c967db82444ee2aed1f57be79376b45045", + "evidence_path": "evidence/20260721T152532Z-35e81aef/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:57ec7e9be52d1b65fa1bdc0ae5a6055cdffab80dd543074c385b015fd9a77368", + "evidence_path": "evidence/20260721T152532Z-35e81aef/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d1c8a75d3ecf1efaa4d43e846172d9760a0b4b268995d1b52d807e751c78f50b", + "evidence_path": "evidence/20260721T152532Z-35e81aef/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d2e1c9221c48a2563c3db3192851962b71db80a8e416c9cd63f0b5ef2ec52e3d", + "evidence_path": "evidence/20260721T152532Z-35e81aef/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 22, + "failed": 20, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed1.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed1.json new file mode 100644 index 0000000..ccca0de --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed1.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151957Z-6c3c06cd", + "timestamp": "2026-07-21T15:19:57Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 1, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:25953b0e8ec2ac14732f227cdda9c5b95f56e3b0a5db9e9cd145848b4d809a2c", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5570775614faa2bdb0384577797222065502870093d353d5c21e1589ef3c71f1", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5f5d8ff0defd8f71763014d7c56a6e38549349628b98c5358391e9ad3b75e76c", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f5a94b912dc5d5b88685eae1dfa85bdd56b7161dc1bc0e4299aa94b2a1348b35", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:69147c9d5545ad105f886033aac341a1e7db6797961c3a9982005337885ae603", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:c33ba9f039bc231dd3681a5efe49b6b86dbe58bc3b627e40d014dd0231f2ccd4", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a11497327e2af485ec04f66114014e0b4df913db449dca3d585c6ea28175cbdd", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:da857501c97a676b65854723335da5368b5d3a2b404ecfec477af38e5cc3c47d", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:bed325e9688d5df466272e975e16bbcb90bbbe145e176688a919484cf33d3d06", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b729089484fc60785817385016a917e4aa195c380d0e630863fff386a50c7aee", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:62c1cd8260c3ff2351ecf10f0648fe4a70fa8ff29cbeb61f65254b0d46140572", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/negative-invalid-schema.json", + "retried": true + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:42f7df50b1638447050e7ce50b2a54c35ef6556b12f4a33a27ec06d34a42ce5d", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:3b41d48718d40a2104ea9e55105781e6e79b25e48b0bb4fc5c6a70ebcac0211a", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:47c8c39ea5559bd65e834b476553adc5aead68d68d9cd7c385372980422c7076", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:e32f98ef5d6ac3bfd1ca459aea2fd8d04a6a5786ed6405ab990ad5e71ef3845e", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:0337271f8c5df181ab03c9eb1c14f61f6e3a2400d97ade156133fa3b831a3579", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:792d9f2e217879b546f863d1636b428522fa7fd19cc6964620e6dfb8054c1063", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:d669a811eb0b63c8887d7c34c25e9edf16452159b4620712cd7c933f28dcf821", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:9cb30bc75867c273841f7e885b5bd5e4710fcd3baad720f9465948853b42d137", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:80643b6c4a184856636903ecc0dd2a10a15dc783041835ce6d8c530766491f8a", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:eef0ea1201484b7de528c0ea9ed9cbdd4e533a7c2db24b4be4c4d7d49dc58d4a", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:e5e9aed7f4b736cc694441357fe4676739f63d8f98cf1f8ba37de12ff9f313d3", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:467d03d53b4a43b0d7fc4c592faa4a97640f79775d46b9d459e12c05210b9fd1", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:34c9d0ac4562e3517bd090df1594f35b6fa7bb0fa2c833b723f7e9ae03cb205b", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/single-boolean-flag.json", + "retried": true + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4b2b6e2e9fae3ddd568306daf763b1c49ef43c4aea47bd59c265167a982c3144", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1501660de28f6975c3836166ae95fa11f7e2da27d3b3389a7b02e0a7aeb7197c", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "unexpected argument 'now'", + "evidence_hash": "sha256:f5b36ce4d07dda1c1422b9de5fb056801b5aa2aecbacc91b7226952de377f20a", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7413353bb97373f18b0f20cf4325ae6b44755842a63a3ea68aa627f4609a9295", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8551976e431fd240f4d2db0fc1c0150575a1ee0376ee8a732f951e3148179f3b", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:56484fb56273305fd11fc56b06cbe7a96ef5914647bbc9e9be16f6b0fe59e8eb", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:efaee4b6a37b453076a9f0037e17f3458050658ade062617965ae806b16323f5", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:8bb4db0365ab43cce1dac1fb473c21d3ea44f14a7c9988418c2a32cb9dfe9cbb", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:74c83d5175005b145fe2f7a8d61ed9c1fc518454bd52948ffe4c8cfa9f3fa957", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c4ad73e2a034859b422eb3cdec972820cd5d855d91735bb9063be00baaba254c", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected number for 'maximum', got string", + "evidence_hash": "sha256:3a36535ee61836759251ab3a173dadd46ea98fcfd5b4c805e47b2a9bf460795b", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:fd47a8f95af893615be92c8884aa2548918eb1d43c77e18e7876846274ccfd96", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:be49f87b595a3257052127a4f518c0916af46251c6ecb5367cf54afde24bb816", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "unexpected argument ''", + "evidence_hash": "sha256:f267b0de9e0ffc36fd493e6327c49a5c3b060301ca704e6be27e451602316bf9", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7a1057858e6117fce0c93efb92e3292ce840440fb49a85faa12c6d4079774e77", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9c1bf7915d83f8a489aaf1227bd81fcfa9c30a256123f5509656f27888a3768c", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:4d81941dc7740b4580f85b01c61db4bb7d11c2939c3b8f0779e7d24a8b549717", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:54686b6ca89944ab013c77fafa4dff8c61b79ad2d660f8f83d378486a7cc7457", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:92a5e70ccd6c79c38fc8338281cb745f899c7cb7d59c33ab1554ed9934147962", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:db32b8fd32aee07b5429c91321c852a0b354e68d58018b0929b032104050f1c9", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:e0e5dcfa192f3a17d3558ac2e84c332b6230a2ae11bd4bbdcc7b6b52c7445fcd", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7aa5f653521eadde610f4c67f59bbdaee3fa48f70c70f6785b2f3cdef1501e70", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:70615e61b4ccf0f6ecf89e8f9eb14c3870707c6c4279aebdcde710a7c607071c", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c8efba20d05cfd9839d30f231fca1b302a4a30aae67855bc9ea1c46addc92510", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aecf44d13f1cad725271c1738d8a2806ed60d0762ec4d19c0c54b274ee598822", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ed748af3274d4a9c8fcb7199e753946fb26561a8becbe0d76f8c2f79da9cd05f", + "evidence_path": "evidence/20260721T151957Z-6c3c06cd/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 21, + "failed": 20, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed2.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed2.json new file mode 100644 index 0000000..7f93027 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed2.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T152036Z-96088b2e", + "timestamp": "2026-07-21T15:20:36Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 2, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:d4c67bc190faa3a8da170dd15693c16a0a04a414ff57638d294f3d048bc894ef", + "evidence_path": "evidence/20260721T152036Z-96088b2e/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:2202c9b09a4ced836bc67931a62d0f0568ccd8861762c7688de81642fe0d3f45", + "evidence_path": "evidence/20260721T152036Z-96088b2e/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:af7ee6ca2af09502feeac742cb3dc42a084d0e2f1f1faae4c6570f29ea8efbb7", + "evidence_path": "evidence/20260721T152036Z-96088b2e/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a8ba3c6676198c0aa070b0b1aa8e519b6dd8811b3603312ab32c5fa79dc238d7", + "evidence_path": "evidence/20260721T152036Z-96088b2e/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:384a9f98b157b503826bd1be446896af193259c0b42badc8228572376b2d553a", + "evidence_path": "evidence/20260721T152036Z-96088b2e/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:3dd094d38207848c081bbe798f7103bdd335ecd40af076796e14113a14a708f6", + "evidence_path": "evidence/20260721T152036Z-96088b2e/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:6b76ee49c66d6e8bea76b4f484cdd3d90abc3507d34adf95e60bba28a248fec3", + "evidence_path": "evidence/20260721T152036Z-96088b2e/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:3a15ee654f1308e608b13004b13c65abb6ecf41fea61ef4bac82941643d2bb19", + "evidence_path": "evidence/20260721T152036Z-96088b2e/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:2d5906031ec649d6a13df73d79be5ae55293cb7b9565604ab01d406ebf7e10ec", + "evidence_path": "evidence/20260721T152036Z-96088b2e/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ea06fa6a14c90ec038bdc7c0120d4291afdf129ec48f176a9c57c6387b71872e", + "evidence_path": "evidence/20260721T152036Z-96088b2e/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:4a1fc01c1defb0188ef4d9891cee223c29648974dc0dda388c60f6a5220ac79f", + "evidence_path": "evidence/20260721T152036Z-96088b2e/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:70ca6e078b23de3ecb87600b0e05fce39d78f44d1261fc42d54e5115c1dd242a", + "evidence_path": "evidence/20260721T152036Z-96088b2e/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:430a8484c2364ed7e2f474efea081d5c76560736a836176215419c106c687598", + "evidence_path": "evidence/20260721T152036Z-96088b2e/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:793d2139937bd0070fec069f078d8758ebaa04e4c23c0f7d8ac370ef5a447489", + "evidence_path": "evidence/20260721T152036Z-96088b2e/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:d645bad0f495efdaa3916fb4df1396834bd32dbc4b453f083fc51e239b7c89f4", + "evidence_path": "evidence/20260721T152036Z-96088b2e/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:e5f731ff10282504f621c5dfab6f9ba96878f565910baebf25f080838f6ecd35", + "evidence_path": "evidence/20260721T152036Z-96088b2e/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:61ff24f447643fe3f5a3ba8c570ccc68175e7d8be0c45ecf9166a589487844cc", + "evidence_path": "evidence/20260721T152036Z-96088b2e/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:ed3344d8527a62146adc819aa55be7e236353a0cd01c21fbbe57e532eaca6e1c", + "evidence_path": "evidence/20260721T152036Z-96088b2e/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:4db296ebb1be57773ee8d9158327a17e410d57ba85b0ddb55f20117ed7c377e4", + "evidence_path": "evidence/20260721T152036Z-96088b2e/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:0236200cc757be315a04ba2335f1da53661a04dc815aac17534a2d1a013c9214", + "evidence_path": "evidence/20260721T152036Z-96088b2e/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:13cf754a94af7266252281b27a3cbcc4e8527caa3cb99323a5229e795d445756", + "evidence_path": "evidence/20260721T152036Z-96088b2e/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:934e03027df7b45f1597fc0e0bcf578ae6baf9a2354e7474e92edb76ffb99783", + "evidence_path": "evidence/20260721T152036Z-96088b2e/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:8f0275110fa27b37dd2a4a3a1abeceb5bf4c6a3c434d3be9279b4ab69b9b8c3e", + "evidence_path": "evidence/20260721T152036Z-96088b2e/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:da33b39ca4c531d212f301d599339a127e31737cce44bebc50d299aa2b9b1214", + "evidence_path": "evidence/20260721T152036Z-96088b2e/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9802fee4909965646613aa47a38aff86781603ede899836262028543408fc032", + "evidence_path": "evidence/20260721T152036Z-96088b2e/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1375b4796a64236ff5baa6876a2c5966f9d3a8a9448403867aea629644f01072", + "evidence_path": "evidence/20260721T152036Z-96088b2e/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0bb92115a8f8e9752c35e16c0877d100e55f3c63e45aab5f4520a862fd05c9af", + "evidence_path": "evidence/20260721T152036Z-96088b2e/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ebb206908873ea926a1f4621c58d716311128f56d5be7d2401184dcad3866307", + "evidence_path": "evidence/20260721T152036Z-96088b2e/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:449dfba90e797c221abcc5218f3078ab33ea48ea99c96f5c70cc2a4ad81cc201", + "evidence_path": "evidence/20260721T152036Z-96088b2e/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f4a8ed7b00ea78ba0d926314d0ef333faa5b05d3feb9dbf5ed17407b2d2e958", + "evidence_path": "evidence/20260721T152036Z-96088b2e/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:15a79164ca762d6e012bb0984447b42937d6a58639ca650e520557d1c6b83e40", + "evidence_path": "evidence/20260721T152036Z-96088b2e/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:348f31bade56c1c19a165e00faa8d00b04703ec17f085fefb44d23ca2c9b48ca", + "evidence_path": "evidence/20260721T152036Z-96088b2e/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:b3438d1fa9de22db42eb0155943f6b805d4eb45d3beb647f0cc3d3267090bb08", + "evidence_path": "evidence/20260721T152036Z-96088b2e/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:38ca0ad4708031dfa38d3e2c59344471d839567f60d39deb9f7b2991e7201ef6", + "evidence_path": "evidence/20260721T152036Z-96088b2e/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:51a25ea21f507e9e76b9c0f4122ae0385d18bbac867e386b972e22a686a1dd0e", + "evidence_path": "evidence/20260721T152036Z-96088b2e/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:70ef34a08e452f5b5cc8c079cc2a3ec3f0fbe83b350d8e845e9cf3f40f370d60", + "evidence_path": "evidence/20260721T152036Z-96088b2e/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:807360a048d4b21c13aa720b23cb4093610f71949cd5f378fe17dea73968aa49", + "evidence_path": "evidence/20260721T152036Z-96088b2e/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cf00ce8a99671bbe9226b0532f59ca837278d83193cff59b9a55d12124d4be4f", + "evidence_path": "evidence/20260721T152036Z-96088b2e/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e7d0d46374401bd019e55bf7c8e0cb3ec68aeee6f8739479848e464874eb0928", + "evidence_path": "evidence/20260721T152036Z-96088b2e/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c787d03fbaf60eb0c62b35c628f8b8a9e38081803254a0d03adb12927b3a41df", + "evidence_path": "evidence/20260721T152036Z-96088b2e/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:1b144ef717a74174b1a1b1095e3af5602ba93de57b2dafc0629a6ae8d2f9398e", + "evidence_path": "evidence/20260721T152036Z-96088b2e/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:80198c2fdf4d9ed8d02e795e8dc7f2b91f1d1d5543c8338db9c1eca3f1206e6c", + "evidence_path": "evidence/20260721T152036Z-96088b2e/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:df938fd340e4964c0caf0d2ae3dcdc9444c88efcf70eedab936a30ae476e0881", + "evidence_path": "evidence/20260721T152036Z-96088b2e/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9aa2c93d1996df53e843035ea2dc84d02f4af27794269c965a5aa5ccb4377853", + "evidence_path": "evidence/20260721T152036Z-96088b2e/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:cc70395c1b81048ca14e4077dc11d82491e21953f16440997ae919744920c24d", + "evidence_path": "evidence/20260721T152036Z-96088b2e/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:137b71d9b335e47dfb1cec5ff2305f2f72f88b8a77f8b85fd7b868ec07df6dc5", + "evidence_path": "evidence/20260721T152036Z-96088b2e/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7d92dabefa050c630a799b60b8ed6d7db78ea62e9382c7e6740d8aeb600cf9fb", + "evidence_path": "evidence/20260721T152036Z-96088b2e/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7800fe487787381f899a35f6143d6dd79b1401740f5a79502c01e4fd2a9a2590", + "evidence_path": "evidence/20260721T152036Z-96088b2e/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7862690220bcbf6a4cab13164cbcf35d377b78d163bc01f5451f31edc4dea4a0", + "evidence_path": "evidence/20260721T152036Z-96088b2e/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:171f78a27f6886dfb1d704f6574f590b85ef4dead656d996619340b550d56319", + "evidence_path": "evidence/20260721T152036Z-96088b2e/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 22, + "failed": 21, + "errors": 7, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed3.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed3.json new file mode 100644 index 0000000..225a97f --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed3.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T152107Z-ca646620", + "timestamp": "2026-07-21T15:21:07Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 3, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ab54314836cb90a26b2a992dce4302c2e2d7ac0bb6c84673f8a1072999f7fec7", + "evidence_path": "evidence/20260721T152107Z-ca646620/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:d0dc0fbc1f5988310687dbd49bc6b1da9d3449908de38d8be042da2b51184028", + "evidence_path": "evidence/20260721T152107Z-ca646620/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:998cd424b3d3d66d71ebbbb7a531e69913cfe7254a2890aebf43784a1c1fe3a1", + "evidence_path": "evidence/20260721T152107Z-ca646620/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:b499f37f1f69747815d4818a6d3f536561f1b1f4cac93d54daf52c0860442dec", + "evidence_path": "evidence/20260721T152107Z-ca646620/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:180a91e436db50e1757c9601600c620334a189c43586910b380b26f31799d4a5", + "evidence_path": "evidence/20260721T152107Z-ca646620/multi-turn-route.json", + "retried": true + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5d67abb63cc67325d86c36e260c291aadd9af4ed577a401a7c0523e4b11003c9", + "evidence_path": "evidence/20260721T152107Z-ca646620/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:913744b19e720214c1fc97e23d48379880b3aca9898ea260ca6a831c2063a7d8", + "evidence_path": "evidence/20260721T152107Z-ca646620/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:93b65c78748c5352b2d7f5be68cd3737ac0f28de3dca54d4fd785865ab3e2a49", + "evidence_path": "evidence/20260721T152107Z-ca646620/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:b466b42992810f1cc4db0aaeb3cf811e902f34aab78ec993dee55e466b1a228a", + "evidence_path": "evidence/20260721T152107Z-ca646620/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:72e633f49387ce6cea8bc5ed1004f6210dd1b261c8a40925b11c55d5ae26525b", + "evidence_path": "evidence/20260721T152107Z-ca646620/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:e578c5e024d8fea74b36f306e25d8d43c5b5b32c22d87aef1db0d9be2513da6a", + "evidence_path": "evidence/20260721T152107Z-ca646620/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4562ed9d87855059637b5e356fd3968c4b6b2404257c730d35e9a9cd36abc5ab", + "evidence_path": "evidence/20260721T152107Z-ca646620/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:6b69b870461f00d2587210c4ec713c42e3f9e322be100294fb7a080cacbfc610", + "evidence_path": "evidence/20260721T152107Z-ca646620/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:53f41c36d44310e958988c6372dda1c2db306a88fe4518ee5547c961ac4e5d8a", + "evidence_path": "evidence/20260721T152107Z-ca646620/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:f4b4f1359d0ee7d31a84dfd7be8a179b7cf105f5c08e841635ec203527e1acfe", + "evidence_path": "evidence/20260721T152107Z-ca646620/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:32a6dbb347d51a6d92c5a55322f9ef85087391907101fe948ad2ed358775a885", + "evidence_path": "evidence/20260721T152107Z-ca646620/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:25be56626213f6b872a2209f2bfbec9b32dad0492f06c9690933c6e25dd85af1", + "evidence_path": "evidence/20260721T152107Z-ca646620/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:e43946ea34949cd1d959c2003d0b2c35992163e98e5fb1ab1a6334bd05fdb287", + "evidence_path": "evidence/20260721T152107Z-ca646620/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:b17e288d1ec49f19c3d54ba21cf8e69a775f4099767560f54303ed6bf457d5b6", + "evidence_path": "evidence/20260721T152107Z-ca646620/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:6e00ac04e19b16e3f8434b35a2c0fed1347281354bc5844614f2146a4889e950", + "evidence_path": "evidence/20260721T152107Z-ca646620/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:cc49467b631129b2d4b2ece829ae14fc3957cc64d3f64e574e20fe88a2d3fa08", + "evidence_path": "evidence/20260721T152107Z-ca646620/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:178fcdd3221b50fb09c608702464ab126578eff27968ad54b0dab05a85ce1eda", + "evidence_path": "evidence/20260721T152107Z-ca646620/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:f5c6fa439de08655b85a4da4085e5c58221b5e7daff709f8dd9e21bdbb766394", + "evidence_path": "evidence/20260721T152107Z-ca646620/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:3addc8f8dddb7e9aa4d8ddb9c763eae6e1244ea2346e1802d92ff99ccf083237", + "evidence_path": "evidence/20260721T152107Z-ca646620/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:84efa20d47ff0e6cbb12a5989bc9918722657ded7f2ac1b3c8e3129253206618", + "evidence_path": "evidence/20260721T152107Z-ca646620/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1ae6b6103b18b1ea52a368ee9f5d644a1764ccf854a201bc8beef9331d3e9a93", + "evidence_path": "evidence/20260721T152107Z-ca646620/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:67647fead85f8a38d0901d69f5de9e35f36fa8945d5e9057b02573f0ca518fe1", + "evidence_path": "evidence/20260721T152107Z-ca646620/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:385999f204257022c33b3028b6128fedff1f231fb1ab57e5fc208eacbf902db1", + "evidence_path": "evidence/20260721T152107Z-ca646620/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:89d957638a8018eb5718690085df14c35cfc1c7d67474dd69eb990984520878b", + "evidence_path": "evidence/20260721T152107Z-ca646620/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6f7dda53bc352c41e87f054f9ff6869c99f95c45fc831af3e3ef4ab710200496", + "evidence_path": "evidence/20260721T152107Z-ca646620/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:cb05313314cc99ec07519bb3a796dc900e928e20c0411663c322ed6e964bec83", + "evidence_path": "evidence/20260721T152107Z-ca646620/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:b4c5ffbd92bb65290a5793842f3cbbba5817c32909c345f013d05548183cc83f", + "evidence_path": "evidence/20260721T152107Z-ca646620/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:3aade45ac33662e4902dddd1930211541e7ef5d8dfcba1100550d9c348c4edc7", + "evidence_path": "evidence/20260721T152107Z-ca646620/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0143fb0bf4b6c8100e9327dec5ef0da400c56c786ded1cb0faa4979113154bde", + "evidence_path": "evidence/20260721T152107Z-ca646620/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:738ab9bc5573d6e6886f6c4cb739d6fe7106a413dd7806bbf8991a131aa708c3", + "evidence_path": "evidence/20260721T152107Z-ca646620/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:f7e9d2f95d0256c547f7a380723a2e097d3bf7bbd00e4b2b1f35cab35403ed5e", + "evidence_path": "evidence/20260721T152107Z-ca646620/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:6e976748155593ca10cc426c69f4389a43d40eebcb6a2ac43e6fe14da1f81062", + "evidence_path": "evidence/20260721T152107Z-ca646620/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b1a100ab3b3e7dcdcf0d77b75dcc1264ed88778598758836105eba0aca391c6a", + "evidence_path": "evidence/20260721T152107Z-ca646620/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:253c7ba31b79a3e394ba216dd4de7a212667449afcd4ecc26d48fed9f3a4fd1a", + "evidence_path": "evidence/20260721T152107Z-ca646620/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d6fb011fd49884a3da7f94b0f0ef711fa59b44a8c1c458a5936bee20e15b8fba", + "evidence_path": "evidence/20260721T152107Z-ca646620/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:6a5bbc7e3a6cc1612649d4d6fc42d70c220dfc90ac5aefe263c11858d883aba9", + "evidence_path": "evidence/20260721T152107Z-ca646620/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:6e3927d06f4e1b805db7da71708e0ffaade0f0a9b4a5c133e717e0af56a65417", + "evidence_path": "evidence/20260721T152107Z-ca646620/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0d36790669173eeedf6e09ea509f65b7cd7f99b2bf9b7b7816e4586a3a2fbeec", + "evidence_path": "evidence/20260721T152107Z-ca646620/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5bfc6288294072a0b54da1ba374afa0ef0fa4479194246f56babe41d205dfa21", + "evidence_path": "evidence/20260721T152107Z-ca646620/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:bc2ab3507fc4416d1204448ea1e2062a2e6d073d11e9fd6c9dcfd7556a587fd9", + "evidence_path": "evidence/20260721T152107Z-ca646620/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7770f496ca1b30954356676aac06c1b65a323d07cc2a1820b88fa63465383820", + "evidence_path": "evidence/20260721T152107Z-ca646620/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:75789a933dff33e5d4db0d550cb2a659b6c966defba084ae350b65e54e0715c4", + "evidence_path": "evidence/20260721T152107Z-ca646620/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:89d1714e78fb0f579a83c5e33c1dd08129610cf2c82bb58cf87d1818194196fd", + "evidence_path": "evidence/20260721T152107Z-ca646620/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:48bd7a5f043b0b433926d5086ea8de3917b538ca732589f8f036516c5bd84c5f", + "evidence_path": "evidence/20260721T152107Z-ca646620/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:894c3003ebd011dd1ce455739ed56b72a71a42496c55b6b038a0df2d427dd9db", + "evidence_path": "evidence/20260721T152107Z-ca646620/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 21, + "failed": 20, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed4.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed4.json new file mode 100644 index 0000000..b507fb9 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed4.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T152139Z-ceba2197", + "timestamp": "2026-07-21T15:21:39Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 4, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:89bd3b96f2f999fae4a485b5d1dd512ea9ed2cb0b6c89d9593feb1eacf5016b5", + "evidence_path": "evidence/20260721T152139Z-ceba2197/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:2ed3eabc57e7833af653a8014124c585f39ae3098db7e5dfb240306d1d133f23", + "evidence_path": "evidence/20260721T152139Z-ceba2197/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ed466b281943d969076fa34f00a79697473d63fc88159c17e291dcebd9614cd1", + "evidence_path": "evidence/20260721T152139Z-ceba2197/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:10440a2f1a5c7c7dbfa22222afc1e0efe120b09a6891d9a1b6aad75cfeeca933", + "evidence_path": "evidence/20260721T152139Z-ceba2197/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5d55e205a88c4b8364c1ecefa419f8a28d7aeb6a5685a38015237c511e567bf0", + "evidence_path": "evidence/20260721T152139Z-ceba2197/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:d7cd2da97dcb9fc0ca3c185487ecef4429e6d9dc8641e5d450c5ca8949c93297", + "evidence_path": "evidence/20260721T152139Z-ceba2197/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:821f00a38e1f3c576c39916e56910daa1883359afa5a4e7b4c043295a37ecb9b", + "evidence_path": "evidence/20260721T152139Z-ceba2197/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:8246e47df86d923a7cb1baac2eb0aa9dcd6cbbc6d75bdb790e6f98c98676df52", + "evidence_path": "evidence/20260721T152139Z-ceba2197/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:208f3688cec8816124cbddb33dda1e4423120a4074d5f3de72297bc16826010e", + "evidence_path": "evidence/20260721T152139Z-ceba2197/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0717ff2436d71a38929319690b161a4c39106eaff77a8059f455d7ca3b466846", + "evidence_path": "evidence/20260721T152139Z-ceba2197/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:69aa8ecdea5ad261969f8c9b2d2d8a61f79546a5e403c68f05027216985f0068", + "evidence_path": "evidence/20260721T152139Z-ceba2197/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1a42310570ac0bc30aea32cc33efac90fad72b3be24dcd853207da650f343827", + "evidence_path": "evidence/20260721T152139Z-ceba2197/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f37e98ef506d3bd05632f9e7e7afeeb232dc6452f8ae4cf14adf1bb99d1af50a", + "evidence_path": "evidence/20260721T152139Z-ceba2197/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0b5182dff94f582d6f695a8209b06d612964552ef1e918a981ef4446fbf06a00", + "evidence_path": "evidence/20260721T152139Z-ceba2197/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:cb4deac91850ee979b41009c35930400fb190f66c120ba8c7a480a50e1505dcd", + "evidence_path": "evidence/20260721T152139Z-ceba2197/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:c50d4f4efd3ddb0f464af161a5fbc63f3559be4136153a1960ba74076f75b7f0", + "evidence_path": "evidence/20260721T152139Z-ceba2197/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:96dc6e200397ebd625b30f7c34a48159382d49ab978b721cebce8d818e5fc670", + "evidence_path": "evidence/20260721T152139Z-ceba2197/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:092f06fd961c3de07e5eac4edfccd93c9708d2bbeb94a3d9a098cb3ca62a4001", + "evidence_path": "evidence/20260721T152139Z-ceba2197/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:4967b2f400aae1b3f05cb42bcc01106324cf23cb7b77bf1c7569dd19bc1ecadc", + "evidence_path": "evidence/20260721T152139Z-ceba2197/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:fd890af8783fdbb1fc1b60d5aa02a8e3743b9e18b583aedbad0a90994196d2d4", + "evidence_path": "evidence/20260721T152139Z-ceba2197/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:35836889e7b21c8f4804a31396f9652d9b29ae26ab7ce60fd9d290e2fb28231f", + "evidence_path": "evidence/20260721T152139Z-ceba2197/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:c3506c00aaad81cf9514b1e736ca36b4be39ab5fff243a6f0426501baf33e55e", + "evidence_path": "evidence/20260721T152139Z-ceba2197/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:ee0702629a6950daac14f5953d1a38165737008a303eeaebd565952272ec72b5", + "evidence_path": "evidence/20260721T152139Z-ceba2197/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:769fc79a63a6cddbb9a97002d34aa9cf410688b996ba26ac0a34d7d2777f2579", + "evidence_path": "evidence/20260721T152139Z-ceba2197/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6572087f9eedafd47745410061a62cd87853fff849729e2da2e1e54f389a72f2", + "evidence_path": "evidence/20260721T152139Z-ceba2197/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bf1df811ce95014f92d018b7e684b857be925b18adb8c835412a4979d98149a3", + "evidence_path": "evidence/20260721T152139Z-ceba2197/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f3e2ec3b02fe3ec95b62072b790b09abe0bd8df31cfc9862732d6e94614ba7fc", + "evidence_path": "evidence/20260721T152139Z-ceba2197/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:32332ef7a55baa88d1cabb2012c2f6cb071d47ee0cd5042e127423bf650c1532", + "evidence_path": "evidence/20260721T152139Z-ceba2197/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:2d648f933117d405887d3be27aaa68f62961cd8d3f4d585e3a53d540496ddb27", + "evidence_path": "evidence/20260721T152139Z-ceba2197/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:33580ebaac78118dabfa4e58d255e98a7b4d7ea4e4b9b13a297392033f57837d", + "evidence_path": "evidence/20260721T152139Z-ceba2197/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:a79532b5bd1ce55b22767de5d30f54a3fcc3209e7a2fcb3392303a947e94b2e4", + "evidence_path": "evidence/20260721T152139Z-ceba2197/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:91e823fad305f6db13e6254130961cdfc0ec21bc9cd5fed3c2a872890bb92cc0", + "evidence_path": "evidence/20260721T152139Z-ceba2197/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eeda1ebb4ea664f2d17a034426f8f5eb2cdde3b0938c8e1783c81a38a441cc85", + "evidence_path": "evidence/20260721T152139Z-ceba2197/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:620e2e0dcb4c32784a49ac66429d6b5e8c12d00dd17b59c7c048fdc498eb0d1f", + "evidence_path": "evidence/20260721T152139Z-ceba2197/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0663ac2b440bbe13e7ef0f3684b71714f4d1716d0af8d487a6199458a69b7505", + "evidence_path": "evidence/20260721T152139Z-ceba2197/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f1f6f078c90ce4a0debe21c28a9a3a382fb09c8cc908d519237458107b3bd38", + "evidence_path": "evidence/20260721T152139Z-ceba2197/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b2a5a6c57dee249d0554a643f3a3e1b83273e26ce7a041629a1946564dfd27f7", + "evidence_path": "evidence/20260721T152139Z-ceba2197/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a8c1184ae395c6aac4e4c3b872b60f668929a65f6dd69bca4c0a61415e21786e", + "evidence_path": "evidence/20260721T152139Z-ceba2197/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5a86ad7d44aa81368acfcaafc51e688dc0d1adf657e6f99ff4e9aa4e713361ef", + "evidence_path": "evidence/20260721T152139Z-ceba2197/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:924e7fc519b07b033fc5777af1e27ee72aa6c1410df4994b3e00c0dd0c071232", + "evidence_path": "evidence/20260721T152139Z-ceba2197/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:54c991333150493a17a374e2b411da396afd708509f3614374edc78059850bbe", + "evidence_path": "evidence/20260721T152139Z-ceba2197/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:3ef724020f0c315f90cd63f0fb52f6207226c365d0fcf17ff22b0b04ff47e7e7", + "evidence_path": "evidence/20260721T152139Z-ceba2197/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f09d74fdece164dea575712a4fd2e423d86f592717e00127ff508b1ef1932750", + "evidence_path": "evidence/20260721T152139Z-ceba2197/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0e553cd5ba5f4079b7421ad56c2d01edaf987731e1474b3d64d412280b4995aa", + "evidence_path": "evidence/20260721T152139Z-ceba2197/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:0423844d411ff6a81cc5155b9d3d9dab7ada01f9b24999ab899b5eb3f3a91f48", + "evidence_path": "evidence/20260721T152139Z-ceba2197/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ac0248a953335f22ae705e970eff22a523dcd3957907fe0dbd546c80d1f4ef83", + "evidence_path": "evidence/20260721T152139Z-ceba2197/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c393a57c03ffe5bec998ff5c933fe5d597cb82c44d4bdbf049b8b0a6ad4a7d8b", + "evidence_path": "evidence/20260721T152139Z-ceba2197/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:969193061b387546f9802af3a5d34e08cf1d51e4a8b68ca6fdb1fc6382a394cb", + "evidence_path": "evidence/20260721T152139Z-ceba2197/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ca192e10e514d563f562f447dee66f335c5ad0340aca53cd037dc57a27a70263", + "evidence_path": "evidence/20260721T152139Z-ceba2197/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:53223be2a3e0b599534362c1fbbfdcbe88619cc17b96c3e1a3bdd63813eaeba5", + "evidence_path": "evidence/20260721T152139Z-ceba2197/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 25, + "failed": 16, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed5.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed5.json new file mode 100644 index 0000000..fc0d560 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m-seed5.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T152211Z-5f2beef5", + "timestamp": "2026-07-21T15:22:11Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 5, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:d4900aca1e1cbcb3806cf3e5badfed3ed9335d00481d5939ab4db82be3da0278", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:9271a43f8b66e0632e056136a0dd670075f576c5f4b6f276dada407f4fece768", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:43b922c1c84662b38427697213b96084ffba85ee161368b6a9e1c988eb58f0b4", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4c29c847a5f670ca15535c5e58512eb6979586e45b38aa206196e3cd50fcdf42", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:657902480e2f7b521293dcd77c61b22377f2b98185073e582be2dd501ac452bd", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/multi-turn-route.json", + "retried": true + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:95d11569a20c3b8f7a227517122fde1ed3f0099d293e896a801a51203caf2b88", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:72ed00662c94e3d3d55ab9464a930d951c05e05b92c2f8d69c03b440bf88d437", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:3b6949172d224abb82c1a2d90412cfde1d1de61dc4c66ed14a48fa6d9b1ccf36", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:43fa174b8b12a7b59cb8a66ca36f657457eb2c87ca57761c340d5447e41f6142", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f2d39e1f0339973ccbcb56d7aaf5c86aaa91ba5fe4b220c510a2f6bcf6726d47", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:c1c0253ea932d3494ba69fbb54ab225d0a08b4d8a22b325bb1e72c98d9759ab8", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b73426e3626fee3e061096b5ece479a91d5d47d88a76142bf74bbbea145a1326", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:4e5f3f2e17d7cbf7eaad97c5d163e665bb07cae4ff104daa4b7d3d80d9d8b7d9", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:21a10ea35f392df59772af885f011a3650c2d8a81cea81f90827fca9a5fd53cc", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:1e5ceb70f7e8ecbb0e0d605d2d3b9bbfd279a67d903f95e2f0dea2921682827b", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:6faae494738def1b8f287db0e1bec392b6a58de7fe849051a838fbb729600cf4", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:64965e59a388719664c0dfda9e3fc053bd35a5c5ad7d8671211caf428a7bef02", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:f758042a46792d34a2b05afd14cdd393d3a5bb365233924bd4587929b4c6c9f3", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:13351bb372ff3c7bb5a4c559f852a95031463f703f3caf5d0e248ce9b862b1ac", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:9e40daee2aed147edfbe59c58e290dcbf7281f3739a45a238d770554c019c97e", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:c06b4a7b9b05a51f8f93bf2a56859851911790e3e9f410ff9f203921dd16010d", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ce1fd778bdeb1b1b9c3fefa94767103d066f11585d3505b185826086ce878ad7", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:ebaf44513ffb71b11baf3fc3b82bbaf1d1d3677230d31e9f1f7ea569324852ca", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:2f957581732fde940d3e57d5e18fccd6646fb2ddde1e8cffce79ec964dce5f5a", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:825e7bfd517aa14146d4f4c10bec308f830c7547764562fb2c2c2146b44e8012", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dfed63edb4ab0b83a861466e647ed2938d7abdcb932216b5cdd82a14c110da93", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5e44389de56df72d754f2b672974491f600bac83d4c70e7c929fdea13fed7ead", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9d0f4489e794b77bb4e348f11428b574aec9d40f9434d5d030ffdfc3f7fef604", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:639319d1a54f600a5a676f32f99e3c797281b1e3e44085587e56eae1ab2a7ef8", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:def01f735c5942c14dc0c20de27820165120f9847c851f29fe89dff0dc53617e", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got null", + "evidence_hash": "sha256:34cdec71ab925bf5293798208c5170fb1a50a7b62f0c3fe269f4fd8c1aaac1f0", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:7fa12c005bfc529951138df2dc21bff4983f751bb4a108ce9799f91ffb13d646", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:1dcfdb23797bc29828ddb3f5237cfc20e75e4edbe6064b49995393499c36562e", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:82f645de09d5b21902915c10325216695f4b47b46b0319f10f676934b9282905", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:12f3794ccb16279bb9477b9a78e8865cc5db5ae85b31b79d20fb8a1a09ec35b2", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:dfb3826ebcb1851e2bc4258bae3e2830d7eac7b37b6b04135dcb010378c09c9a", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:075529ea638a96feb7ee17c78c7c9cb26155a1c9d4e3c9a953843559c3a1feee", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:923fe599cfc29d407a1249810fc10c1d29f331977d0ee78f924eae5e24e40481", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0818fb4535e481085d8620d30558704bc0158578897afb0aebfdbae41c9620aa", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:433293cf32ad4b349189b42b578dcb5d7781795e08e73baa8cbde202a0c11eb6", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:e28e13ef17993fa21e8544843caf6e757f37df5ed7b56ed083ce0aa137d38d2c", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:f3042dc421a2650e7931009511931beeabc888e7b0e738f5fec2532354f658c8", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0b246ae4b291b84af0ffa4d07b22929dbc530524f372f974c687f41883b1457d", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9e07dc92926ae9e807e444299eea25810a5d6ce73d81a97ec17db4854de09fcf", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:e24073495c46e57ca051f2f228093b7dedde9d45e8e8f6aeb05888075a7d8d8b", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:86c4ab295a8a5366b0c02a0365fc208f1eddf10b99409771e8b0b1a661dd3c38", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:00d26b4324d9af452dac030758d91f0627de41172072a7b6969675e5d6994613", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e111f8ba4135707e7e24c1b671b26c5669ffda35024818780dcfd3087d1b357", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:12a7474d2bbcaece6f4f90a1abf9ac4a59c5a42911f9fd00a10f59e8b462fd07", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e57cb51b3df2e0d6df69ffa56a7f0d9f1f0bf0e9d36717f52aa1a0efa00da587", + "evidence_path": "evidence/20260721T152211Z-5f2beef5/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 21, + "failed": 20, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed1.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed1.json new file mode 100644 index 0000000..3c68558 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed1.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151603Z-9c20c269", + "timestamp": "2026-07-21T15:16:03Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 1, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4d497ee68f98eddd1ae5f20cf96c22557948de5637cd742c9bf9a9073b971ea3", + "evidence_path": "evidence/20260721T151603Z-9c20c269/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a7f821e6a201db826b47453ec812b1d0d38e1ec0cf743a8d8fe4feaa4ab2de24", + "evidence_path": "evidence/20260721T151603Z-9c20c269/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: schema violation: expected integer for 'quantity', got string", + "evidence_hash": "sha256:733f276d0018862c87a09e80258324a459f1bc783320f855e3a301f8bf7cf5a2", + "evidence_path": "evidence/20260721T151603Z-9c20c269/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5144f4804a8f6fddd3d128f6f2423215d5731cf33bc6541119b727e27866891e", + "evidence_path": "evidence/20260721T151603Z-9c20c269/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3c6c9f1827c46d43f79b163d4cdb5ecedee2a52cac1fc8dbfb6f9bef990ae624", + "evidence_path": "evidence/20260721T151603Z-9c20c269/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:262522ef29bcdac0970cb16ff56a5a0b72b3da11a5ad765e8c8be5287514016d", + "evidence_path": "evidence/20260721T151603Z-9c20c269/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:00955504d74b2504892f7cbfe7e4e74f8f56a968e6f2829104cc51c5236d9bbc", + "evidence_path": "evidence/20260721T151603Z-9c20c269/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:2dc2818fa80cdb787a7925b7bf3967e42b335ae7f29726123ffaa04489a1d2e9", + "evidence_path": "evidence/20260721T151603Z-9c20c269/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:7e02dad985a22997fe809e1a0a7c629dd96b7006a1d252c135cfe8bc1ca345c1", + "evidence_path": "evidence/20260721T151603Z-9c20c269/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:49cf20dab948b7abbd7f6255f5ad3f51a563de543a23c748662ecd1bf04aff26", + "evidence_path": "evidence/20260721T151603Z-9c20c269/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:a185d723af00e5c1ae546bfdb2f5995227a5a4f8e9deb670f08ea3f1da640981", + "evidence_path": "evidence/20260721T151603Z-9c20c269/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:90978dbde4b36060cd2bc3a7ebbad4efd0befb2fa310dfcdb9a28d6170990fd5", + "evidence_path": "evidence/20260721T151603Z-9c20c269/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:58b818896c9455b477e925746534a893e639aa87588e04d2c47206030b1f4e8a", + "evidence_path": "evidence/20260721T151603Z-9c20c269/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'city': expected München, got Mnchen", + "evidence_hash": "sha256:22383b2d730cae3be5f174aa9236fb5ab5f6a1e690107e34e0076e4057f2ae0f", + "evidence_path": "evidence/20260721T151603Z-9c20c269/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:79d81376ba384710edf1cc05f73c9fba985ad92645435c87b7245001864e840f", + "evidence_path": "evidence/20260721T151603Z-9c20c269/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:ed12d41bd0c25d9ea6036f3bdf5d05c8d65aae26efd1c4b8ab9755333a4fcc77", + "evidence_path": "evidence/20260721T151603Z-9c20c269/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:6f41ece4e633cbf4af9255b0b94ebfef4c905741e485138434704f5fe557cc27", + "evidence_path": "evidence/20260721T151603Z-9c20c269/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:ef1b6c28d3070683db9652eb1ba4a58cab186902860d3c85e7fd1985a7566240", + "evidence_path": "evidence/20260721T151603Z-9c20c269/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:23b0672d59c623b565953ef5ab7bdde3caced5c5366e7bcca4f56723d4141ee5", + "evidence_path": "evidence/20260721T151603Z-9c20c269/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:9ba8953f7290623fef19eb1eb2fbe4835ea6f5fa64fe4365b88c0ae70629b15f", + "evidence_path": "evidence/20260721T151603Z-9c20c269/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:7d00afdd891e6a97a4144a6f87a644a9cc8c2fc9f486e261fa53d4229452325c", + "evidence_path": "evidence/20260721T151603Z-9c20c269/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f7e32e731c91d9057ef8ccda305fa069e64b588015d95ff24d351d80b7f90291", + "evidence_path": "evidence/20260721T151603Z-9c20c269/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:7d95cc93a7c7528df43809e4d45327487cbf2e1f9b411134524fb8d4344e1fae", + "evidence_path": "evidence/20260721T151603Z-9c20c269/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ddde3be5032ed5ae254fce432a94dd1c3bbf7715662abf2aa0bdcd24076598e8", + "evidence_path": "evidence/20260721T151603Z-9c20c269/single-boolean-flag.json", + "retried": true + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:08c8dcce3655b1c0a8da8523a4e04e4893c652726a3a09b553ec3396f8676966", + "evidence_path": "evidence/20260721T151603Z-9c20c269/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c0621fcc8493bbe96176df175a910bf4b35c6967998e919b58ed88c1acf28570", + "evidence_path": "evidence/20260721T151603Z-9c20c269/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "unexpected argument 'type'", + "evidence_hash": "sha256:27a510cb5b36b2c298293f4bfed8f268b61c09e77fa94158bcd8a4469f834695", + "evidence_path": "evidence/20260721T151603Z-9c20c269/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2e7fdeed0c675a8b1558c854887b90027146f5e3897ff2ce6db0bb93bb2f1c20", + "evidence_path": "evidence/20260721T151603Z-9c20c269/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:48885a80067eade2c4c8fa3c6dc4b76197d9a51f2a1f1621255d12f425de3f22", + "evidence_path": "evidence/20260721T151603Z-9c20c269/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e3bdf5be72e2ec3a60c5ce1dcbdca2695ce352815c9fbb6739c270bae4369798", + "evidence_path": "evidence/20260721T151603Z-9c20c269/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:c06ba883cb26057b4bd4def18cb1cd48a2a52b915d3c45587f81387e8123f108", + "evidence_path": "evidence/20260721T151603Z-9c20c269/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:18c5d09b58ea045a3e0daa4af71e1aa197ab09f06acba166dfe9737ef5920211", + "evidence_path": "evidence/20260721T151603Z-9c20c269/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a832502bf7e91c9586b1be8dad986ea69022fca7dcdb8be746e6ebe8923397de", + "evidence_path": "evidence/20260721T151603Z-9c20c269/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:01a3d54c3ed6ef955c7f6e5bcb009e7e08b0e09b035b7aac8676032ad466eb84", + "evidence_path": "evidence/20260721T151603Z-9c20c269/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected number for 'maximum', got string", + "evidence_hash": "sha256:889cb9ae523522b5b3dece7c63999055262f4999185546a001000261c38910c6", + "evidence_path": "evidence/20260721T151603Z-9c20c269/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:89c27b753425ce30bfa363bb11b465dd95a08212312110e7ebad6101d2003888", + "evidence_path": "evidence/20260721T151603Z-9c20c269/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:3f0a6c8c472e5911ffd446f85d9f27cff3595b92d2ecb4b40c3d48ddf96817cc", + "evidence_path": "evidence/20260721T151603Z-9c20c269/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bf6155100cd977cc643ad157c94a59cf85a7402908d86fe3a8afbc6be60d1695", + "evidence_path": "evidence/20260721T151603Z-9c20c269/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e2f07b95b26688517178a2dea6615f67a355f5e388c72eb0e7ab0ec780b38dd3", + "evidence_path": "evidence/20260721T151603Z-9c20c269/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:17c357fb0f623f47b3d6f59dd08eb2cac1e784d95837ec3abfd3eb4d248123bf", + "evidence_path": "evidence/20260721T151603Z-9c20c269/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:d5c3c789c7a7ee316b6716286c85a02b2e9dfa39cb74a261b8f1c1dbbb13b2aa", + "evidence_path": "evidence/20260721T151603Z-9c20c269/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:b7b7ee7d66cb627863527d656516054b2b49f870f1868fc71835943572c1a9d9", + "evidence_path": "evidence/20260721T151603Z-9c20c269/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:01427b9bdeab351091f02a2b471628c65487c601b57cc2a780e828bc89414b16", + "evidence_path": "evidence/20260721T151603Z-9c20c269/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:92cd76445bae03d60dfc7e8085490e8f7ca347f83e85b20f133c7bfe1b680108", + "evidence_path": "evidence/20260721T151603Z-9c20c269/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:ea5d1cfd6ae8e29d2779465a376fd15b40bfb45023594f5726d2d845032137dd", + "evidence_path": "evidence/20260721T151603Z-9c20c269/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b7e655852e7bdcb12c675415662c945e2e6bc7f2cc1cc9627dcef3b2254ca592", + "evidence_path": "evidence/20260721T151603Z-9c20c269/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0ac020172f100fbda26db29eda6d1be7f0279a5af1a9cd4d5929113fd2cb412f", + "evidence_path": "evidence/20260721T151603Z-9c20c269/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:920040419e89f09eeeae7db32e823963bce14ce9c2595017fc550cdccacd29b7", + "evidence_path": "evidence/20260721T151603Z-9c20c269/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f193c1a9f23f0c30b518250405b5a7b591037def0ad0bc38cc9ce6721dab4821", + "evidence_path": "evidence/20260721T151603Z-9c20c269/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected argument ''", + "evidence_hash": "sha256:918c3f21366aa8bce64ef27c6ccda9f00b87d5d15d63dc1fcf3895344bcd759f", + "evidence_path": "evidence/20260721T151603Z-9c20c269/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 19, + "failed": 24, + "errors": 7, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed2.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed2.json new file mode 100644 index 0000000..8e70003 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed2.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151648Z-ca4933ff", + "timestamp": "2026-07-21T15:16:48Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 2, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:0ce51bb2387be163369c347cc4d9bb0ea660fbea6d248560d605809c31424293", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:956d8444e72d3c644615a9b9f46e021bcfedcb231323db3718abb9145580983d", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:0f33da82b90ebd22dd23764cacf775df28290822013e8dedc26a03f2a967e186", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:70937a7d9a9db71b5c76bf4bdfe49ccfe0906d3dcc8bc0e8d1c0bfd813e46f99", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:ca50f487f49acee05aadff73da96375d699ac20ddb96580cbdfdbd0fcc5e2ed6", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f8ec2b8dc2bf83a9d0f94563170769fb530d9362ccd61d501d6700b2d4838e5f", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:573c3854ad1b980441fdf84450aea54996fd8ad40f8152ecfdfaa572dd9248a7", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:3434948f8d6cc7bc8915bbe0b0c785edded13451ff7a6806c2d61466737845ef", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:7b8de2e6f436172c73ea9ede2a577d8a6c7b8fe0c0d89163fbb42d314aeb674b", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:73f99b6ff2f28792e08a6ac625acb1043e76d360add51ea2c641aa7ab40479f9", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:c80c0538e66e6582cd62725e2064823f5da5916637a6257cfa0a69b38cdb03f2", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2216805462cccbe2dfe586cf0b31bbbc469944f17b0febcacb0660d030afba01", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:3626d92215cc77c010d5c29d14b5e21bfdc04e9f7fc10dacf2ca7c526e5ab87d", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:761d066439cadb643667639891fea5edf093f12c25d92e44bb80cc5aa699155d", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:6a52ba6affc516d580ec2f11ac276650d97b5841efc98c74edaa7a495cbab778", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:50c1b4a8e467cf3e66e6006498710f0bf33399d01d2f8d75b20a5dd7704fcfd3", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:a58e77d7590061f765f02d45ec2cb49d4f28d4c172473b9e635bbfbf19631f36", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:c60b1e7e2555748b0d22fd9a4a35c31c1d1afc3808ce3f2ac232c27e27e494da", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:c5b8753412cea1bfb0c76215afabd2b9100a36f3d7815f580da2af4a634311db", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:e70a941b8de9ea22c51cc2cc1e2a25f96f30d1b344698741dc1da8cd58592cc5", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:9aa83de6fd594ec606df7a0863e3a7a2263d5c2824fdaf8f60f9978d6c762857", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:7be4bcb6d90a93d5a86f6680ba04396acc5c474d3a1ca1553aaa93b310b0b70d", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:434d225006b91b2e817e8d68af662d6649866eee3e7a496b3f101242c5add2f7", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:04256c17aa8dc7d89617b61ef1a81a06dc3d000e94e7fffff72792293cd88e0e", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0ae158de8666f09728bdac56e627cea98fdb6a1b50f3d01816becc27c51a3291", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:be97a5c928900e12796eccb2bdc3d4ad3a8df4c4d4b0dec315aef427a3ff76da", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0e445deef5d844feddc98697eb1e68ea3112dbed1228d915afd72859e85d1921", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:21fda79962f5ad43fe3b9b0b80bb20220d50f687a3f15a439d2b3dd3832f5e86", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:712dd5c8ab6a58e1a6f15e0db481a2370eb2285da593ae9d1fddef0484339ab4", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:e56c578ff278a940a54f5ff499f31667d9f4faa6aa07bc11282727848b3a1416", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/single-nested-options.json", + "retried": true + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got null", + "evidence_hash": "sha256:db5d59094f93bda982aa22401c8289640c76faa3a6d4b6e0205b2307cb8cb158", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:be7368a782a3f8e6831c5438823e265b68f58566cd2e656b6fd50eb08764b64f", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:70e940bc09d7f5abe60b24b7282529d5d63516309b1b51c8219ad8babf93009d", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f5b4c85e508eebcabb86bd9529ea76b324abe4af2216b194906e6811ff04da1d", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:58f94941bdfad08bdc3c111cf7424ea741ec6b7990d16509a72dd90f202f0912", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:ee38709d57e844077e157eda549cd04f267e6d5584ce963e50799301dad65c4c", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4354ace71a225854b16210932c1fa7edf75ef820412105581a625d27bd89c5cf", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4a064f8408af4f9421530236f93d942382098db7ae654916a0646a2f79501fc1", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9acdb686bf9267f1a395589d0922867835ed644e608468676f82164de6765205", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0f5b8db0f3a273f7584d86aa545d80721c86b2265fe885d12e20d990ac4ff60d", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:547f162a24dabff5a42cebcbfef4161cd987b9c22c0aedf47864f870ba4f98b5", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:e0d9b37dc5d9365e1275aeb549ce18e7b55605595cabf36ce5f36952b1bb82f8", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ef5b0d4850f5b4d0a9635d45b5bd4f9c4ca00864e0a7b19c44be2113e91e4368", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:220bd5ee94a3a278e6e3815da59395ba7116b310b89f3444b01a6a55ad138e42", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:ef868025845faef2c340a774f4480c9b77d0d29c3046fa27d551412b0ba1b72f", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2521458380f0aeff569c938bb3fbf1004b4f23b107a2ae793d26888257311abb", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:47eb7a6f416b01d276cb4a2fba2984608121f671f819ae08d22a9fefffe010ab", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:be79febd6665dd4e4a893a325310e8c609de62941612d2f2471638011fe56a91", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d3f7fce5bdd7c071e98fa42ea9eb65433b58bf84fb1217395c3eee6606510ee8", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2265b61da07f4d97c6f529d5b1047196fcc43f9df54b2fb7705f23ed80d6a710", + "evidence_path": "evidence/20260721T151648Z-ca4933ff/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 21, + "errors": 9, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed3.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed3.json new file mode 100644 index 0000000..688d93c --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed3.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151730Z-cf217eb5", + "timestamp": "2026-07-21T15:17:30Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 3, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:1144b6e61d47d1a01fe6cd1eda1ae2b00221f9214a99e3438672b4165a8be980", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:919e66b248e8b9fc1b16605cff39ab6fb118624b65f9863b6f3cd063d7486a23", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:8c0cf4f97db83260d7220ce2093737fafcd39024ebd30c2a6757c3d720dd831a", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:486ae0cae882e1c5a2ae411a147a37f032f7f5e8d3ace1eb78a91ab8bb9e9779", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:664df149b259473614eaec5a464df45950f158d1103d9195fa1e7817b43d3ea4", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:800c30237d209e72a04199ae40046bcf0fbcb3e598df791fe56e887bb0dc9e5e", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:0c19171424d28d7d9dac10fb10380388ebafef7f12dcad081e9a21ab5af9b6d5", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:77b2ed6a2eb4ab0393036bdbe41c1f0cca720873938b20d6a83e8b7d157acba2", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:1238a75e9477f6787b3d4fda71fabd1ca68da8cb7ca4d77ae7fe3dbb4ad8e499", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:78365188312aeffbf983d476df48f3c81e8d916100b0c95b121bc7e73df12a8f", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:22a3e4c29449df37165fa7119a46ce6f20f930fb0cea79351ceef8f2542b765d", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b3175f132ea2665d8e01bfcc991a73c6419a343447fe2e177ad7faa7b3391a53", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:b2e6f4b2a549ed7c0a781aa672d8ff96baec8980d47c015d5771c6cfc9aaa2ed", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d77c3550080b57c661e37176f85096f90dc3118bc43764e1563cfabcd237ad99", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:9f9b2237b8483cad3f85923f6bba3e12cd2f5af800f83d0c5b8c2a317ea01b7e", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:ccf9db0a777af3e537699d871f8cfcd9fdaada6de7a7372488537f5e18c6a7fa", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:4b7a211a6ec2e26c7ce81627086e39a3c4ec5a512ea274891382781e59e65b43", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:0751d3392b38d16326513952481f4f4bf60b11a066c408cded94122e5fb21178", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:c62243f8b08907382f33f4b3a28dcb3b92499d611940001e4bd5eb2fdf2abafa", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:ee53ff9a87a62d4eb2478a2b307fcbc38ff81c6269cdd8c3c658fad492b82530", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:128db09ba580f8c9b768424842095585a89b80c59298e0818d6bcd5f0be4566a", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:86c39238de499d8d70d73e9b8affffda5ced574ea7192729d9a9b04533ee366c", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:0c51fff4e9051c4ba4928e749e79ee6c8194b0e77f3c592fd4ac288d6c2667c3", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:4ac4b329f9b6c5862f973a69a37b7b7fbd549332ee116f37a9068eeaa987a223", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:11f17c88b6cd3c29c708f95aa411e400822a9a82170af6a678efe1fc1f538985", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb8a5616f04f8bc33b3f21b6ae8aa6774511ca1e0d94b9d3b0877d0dd5906824", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f2f73abc8e54490422651de038b1a9c436bb979874db3557aba6b907f6ec1c46", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4454c6a839b91dce471007ac10942429898a032698d5da6c480eaeac67a060eb", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:fd1fe3c08f3d832a9ad4abe36b97bc2166f4bc775d39fcbb1a18f1bc08bcb287", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:171118e3f8dc223ade6c8c0df6be861287c5fe08ec445d839eec72fa51b75427", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:0fb04c064e45519f511ab9a8b1bcb58c721745f7b71ef909ce3b0ef90f0ec483", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:c613110b29e919aa502cfa85192d75ddb17d257168c3b5c247301b2f410f2fa4", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:de24f4bafa8d376865a86f430c52e5a102a888b111e9a1916cb9e1baa16baf0e", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8c30c45f7c4c7dd2336cefe1e2bdeaaf3e787feedd22f746fa94a2d0d6f6ac4e", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c7f7baf352e2e7559e2d8411190075fe3d895b9aa7f747b63806a86699866fec", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:e38d98339808705790b27635dc13949c957934b202cc73d918dc509c02f3ad6b", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a339dc229debc17ad754f0653b80e4efe2c4029f7b28ed7f70348755eb7fb6d1", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:094af776207d5ee4bc8718b7a434572395fb3a051cce9788eb963bc21dac88d3", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:48528d99f505a3331d8291ff7b98006e6678abc3a37777c238288ffe5330a32f", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:07f4c5ea44efdf65f74b0d127e4eeb488469d43c8a77c2a90181142a3eb7c5ea", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:5fcaaedbc751d1a5362ec325c22be71b148bfea83dc9532fc00e5c0c6dc93767", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:23b1ffa3b5890b8596c7888483aa2a0656c8d32037d2367c2d5fdd136f343a7f", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:36a78847b7cc73a19db58d10b27902996748d36bf5fb53f6d5183cf9c3e51d00", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:58a68cd510cb0c0a23d25c770d6ed78c7427326ad09317fad7135031e6dede9e", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:d4f3c509d5b70aa2730b64751a2fa8c7afdc354a3d99421a7c57806e735f5fe2", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7284af27b3e90fb25d1b4129ca5b160b06579f973debc01b99f06c82d26cb411", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b5e449b1b5bcd227a205c57190eddc1a7d01323738e2fd3d73a74b63a1ac280f", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:198464fc48fb1c9c3310d38907fd10e1d6cdda2002a83d378db56841151bfda4", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e4411588b6ec1f2767e4ae4e9d58dc35d79c4a96fc85651ae5f93dc50a3d3753", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4e20268ee4fef721535d6e72b83fff7f3ed30527d2673c011a3367197527f090", + "evidence_path": "evidence/20260721T151730Z-cf217eb5/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 21, + "failed": 21, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed4.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed4.json new file mode 100644 index 0000000..7946939 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed4.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151811Z-01f24ba6", + "timestamp": "2026-07-21T15:18:11Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 4, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'get_calendar', got 'create_event'", + "evidence_hash": "sha256:a2e699d532eee3c1ef4267e0e00ec6ce2a60aae58724c9b4875a3495a3aa2e69", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:4874e828287a811e20ec09409840d6c2ab6ff47156b4218539f7c23d2b28ae4c", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:1afba239e02719e39fad4e9cc03fed2581d4fe81882b93f755ffcaa0e4603cff", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:80c49ff3ec7b4048962d64a1fc0d8db4c861030a0f1af06bd876427bc48dfd53", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:1345b7e712443693044bb6e7fee05899fb861943e103ab0bf0102b65e8e71f4d", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:30ae2ce3955790ccd84a9cd34e3924113337bb8f1a34896594fddfd163052480", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:a6f3971262535ddd4c4b51fef32888400f37366a2c026c60b9ec34cd049423bd", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:f03e7f91e7b4067e5b774906657af4b91ede066867ef10b4ef73e2669cb32208", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/negative-auto-arithmetic.json", + "retried": true + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:49967d0e1a3fd475ef9f65bf787f2e91f21fa52c6afde57683154ee833c92efe", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4ba7795cd52857be33ce23643a7a14551c897539f811e66e9bef5159a50f59a8", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:630ea226249ea786473e5db60807231416bdd6736042a6d888f8888e6151e990", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e5cf3abd3016a3b492e89be1db0f57a8ad3d3bb8cbd21927ada63e6f789a3455", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'send_email'", + "evidence_hash": "sha256:12a8dc84224a62b1172131170321b32a3f3edcbf2ce9d6eab2a2895a1226f4e6", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "schema violation: expected string for 'country_code', got null", + "evidence_hash": "sha256:533055448ce504ac52ca33927ae589c56626e2718dabd38eccdf14ef1ad6fbc8", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:10b98b838eb85ba8a01aebb0119b822279023de7d98864cb88b5acc2eb9cd147", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:557b8cb9935a780d473da13ced00f6e6970ed6e7579a698d75df3a1c04f756dc", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:5b7e88aef4bd4e4fd3d96a762bfea60276b99c2c59f64212a995ced5c77cbcda", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:fb3f878b3abff74302044e64aeb51f6c6be6b9bf8dd1083b672a6b0ee016c0fc", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:3cdb048eca0872adc44b48537434043583b70ccc1d8237d4077a0fc9a50d32ad", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:9155674309d1a53f7aedce23b1c2dbf2bca7cc543126602e46a66a099ae153b9", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:e7ab83e281b5ed5bcc15f23e990061a46940e3c569dc2236bef4a91647c1cdee", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:71eb71e6320701aa4fa3228082b911ebf97230304c59dc9064154bfe6ed85747", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:3a5e98a2fb4d65b3365b6a29b586860a49a404b5e140b158a84a2c12769a40ed", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:cebafde89bbc74752c517638a96e0495562e0fad29560a37d2ea30617f947b79", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:32547f4bd987eee4c2721b465f66a59d8355b92e84bd42cb9f64d3e88aeb6c0e", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c420bb1e42ab32d12ff0e5f301fe529fbb9f25040fe7600b4f8c510fd0262a8c", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:06d8b3dd7947864805a2ef659c3e1f1603b95c1ccfb14fd36166425b2679c960", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3625ae8c97c6773811b11a91c672672e3fe18d4e4f50fe34d0ee5a697186b9ee", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:a748d25225cef42154b485f4f6b5f99e80f6aa1fb3cb5fd955e9c83ab3c4c427", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:52939076197d0abfe9f068e4ce7807d6d68734303401dcaffb20638814436f04", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:8f39817de96ad527b692a252bba68ae97e3105b9e123c85034811f51c6fcd69b", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:de5f4e260d8ad3980fa00d2bb0e12ccbdec88d8febdcbfeca02f7af7e7eb7e33", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:296f403420a36ee6ae4ad4f3e51d081e12a1f1aedcfd2d1e2a30f9b991d49d2e", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8248095fed48d3dc79b7bce3373bb029ded9efa16d07d28b9a6c295a51daa6fa", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d4eecef091ae5ab70599473eddc1a7fff63f95d93be52076b98e1efc7893161a", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:be5d7ba58115c81e9af8d3b075dc74deb78ccac4b983378ec3b9a774a1905713", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:82190c15ffe84d9c8f864ac32bb8f89214e08eda880f24f2826fc8c25e0a1e01", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:10eff819288e8a1cd58c2716fe56bc1db2811b3272476045eb9be1b32758205b", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95b9b2b7529544e0668e293b427a4ef6e1b513147525b12d7bb795f8aa5a1a1b", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:330d44c6a8e9c3b2f429f121b385f6cae121218464f83e1fb90fceeb1650f5ed", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:b5a8e50bd8e9a15888f84ab8de72e55e6b5f9d70c8e615c2c175c8ae66527760", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:8fa266c12cbda3a063292f2346dec402207b06cab1d7d24693e59fd0ab27301b", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1481f2743a5488831d9f326eca6047cba935ed5d3b0710662a404c915a2f372a", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1abc9e134f54b040cd28e841a04cce41c9873d87a99fe4adb300552eb6c16c82", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:c37db37df7b1d95b3ae20b488d2bc5737bc6e21654c050e78e70127bf2952003", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b87bde1f2d2378d72f890ff8001a6f0381617ffa9a7e14492c3529a63218384a", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7358fcab0006cee645eb002da5f6155ca0634181e1e002ab2a77ae2709677ffc", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:16fbd990aa77f9543adf0a2f0f3f003b1b4d10acee9788c309ff3be9c4eb6e07", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bcceb89969a050a0b8981e4aaef4737acb108ad93017219aff08ec61cfe00435", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a76808eb56def42fb9064da896610388527880740fed91f1203d51f266a81bd9", + "evidence_path": "evidence/20260721T151811Z-01f24ba6/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 20, + "failed": 24, + "errors": 6, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed5.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed5.json new file mode 100644 index 0000000..33cc045 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-meta-llama-3.1-8b-instruct-q8_0-seed5.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151851Z-b0ab2127", + "timestamp": "2026-07-21T15:18:51Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 5, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:ed1c5cfad664437e6196dfb9c58c1b7f9425dcf48bbc272df5d583d8c0118a99", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/multi-turn-calendar-followup.json", + "retried": true + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:baed2a17a641e70598cfa3585cfa65fba8dc201fbcf0c3b04c5afd771831daaf", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/multi-turn-flight-booking.json", + "retried": true + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:8f4d5b6423696ee14caffd3423a6189f28b70edac284162578539ceb778f58ab", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/multi-turn-inventory-order.json", + "retried": true + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:0a53976134432760f8eaae79c7ef7fa901a2c8c3419d4cca5642ada84a4983da", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/multi-turn-parallel-followup.json", + "retried": true + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:c2ee72d63afea3f55c4986ab1bb02a4896d27cf4ff65c24df00c49c008d13f23", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:aeb81ee0e6b5d93d6c50a6e8f522269c6823579c7927126e11197b38343b5bd8", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/multi-turn-ticket-update.json", + "retried": true + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:7ef2141b3ed32304ca760c6d270c4d46420403983e3c92616c741a345d2df7a4", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/multi-turn-two-hop-chain.json", + "retried": true + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_weather'", + "evidence_hash": "sha256:eac5862478ef2b062dfbc5d7eb5f7cd831c25833249fea25a029f67e4c38edbc", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'lookup_order'", + "evidence_hash": "sha256:256c7e4af94d0c58c1affe69a5f06238a20774692beb7320adf6d8b3615b55a1", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8bd6e81f0468ed2b0931feecff1f72913e72a31d684e70d0f6472338c4483c12", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:4eb36a2394b8911a1f8d2e33ada6aea2a9f9afd5410803bf00d0ca73adbf062b", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e50d24a43ffbca313a3907288a4db8d8bd088cffff24138e7f97d0ceaf9143b3", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:463508c9ce1d8a29259a6640984d081e299a9eca5dd5fe4b7f70621fb69a8eed", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/negative-plain-text.json", + "retried": true + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ff335970cf001aac3782c28bc90363deea64899b8dd6ad2670427aae666d3f8a", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:d161072be235d6a4dda929910e06593f8eb01889c22d78a3a0a5d822d26d1ab6", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:f21321a9c6ac0c13936aea79240c08337db80cf7ef6cf6d2601aa51d03cafbfd", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_weather'", + "evidence_hash": "sha256:5d8d1d5709b03ffbf478eac427d125cd41df344fa0a5b1693c13bcf5320e4ec0", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:7e38b484a74b53351ce583890d0d9240ff6c9edc2d189ed4dd256818578e7ca5", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:e2d63dfc205330dd43d58d7359d3f88b2a4da36431cfb83c6861c398c762fe2e", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'reserve_car'", + "evidence_hash": "sha256:6f426720b89ec5c446996faae18a8c48f5ad7494708954a3e24e6af867080dcc", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:cab630eed7f5e27075c26dbbe05ab4595f218145c8e41f22fade10b2872493a7", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "error", + "failure_reason": "server returned HTTP 500 Internal Server Error after retry", + "evidence_hash": "sha256:5f610276d3464c47e25ddd5d39b0b7ddebf4a21941be46dc2e9d0f306472cd82", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/parallel-weather-and-time.json", + "retried": true + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'tags', got string", + "evidence_hash": "sha256:3bb98ea9ba1958c47cfc2dffc341986d26355913f6a5d9ba6bccb6b9a411bdc6", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected boolean for 'enabled', got string", + "evidence_hash": "sha256:4a96cdaa6ba6e1defcfde30f2844727d97160fdd7dc69bb0b35dd4bfb18b2e92", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:86e138946f2d79ee3d029641049ff60765be4163291f911fd21750548a47a4a3", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cf8e3007bae070f71e4bf83b05ff9651d1cdbcd306d644815d5d1e96f5c1c5e2", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:32d414e7c371270ed153583d964b66014961929fc60e060d9fc5fe0b087c3e9f", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:472fe8e8d63209bbd5fec68337cf5a2ae3e39d51c8046afb8c474d03327eb631", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:dbbe6c8fb02f1acde2bc176d98484fd29aa3cdb2e20ae1119a6cf40223465571", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:22da412aec2767c38592a6d67a99be0998eee53e361811b5f42d8c0fa9fd8126", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:e12cb33d7cb4c0ad3701a5b43ca9d2229acd040d0144a34c77ecca1b7008bf84", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected integer for 'limit', got string", + "evidence_hash": "sha256:99a0e295b605e683f06c5e4f9336c3e977c45f3e9a23141f95bd7fa3fba19d9c", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "schema violation: expected array for 'track_ids', got string", + "evidence_hash": "sha256:31ed8787ee4474eefe0a36401db0df33d3959b4bf9e71ea8f6e29a8a2879b36e", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e9b8c368942e90c44234e6a4c1c8de2520bc2857e9023f1708695ecb02e89cee", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:98c9a1c3fad7fc2ab47212d5d0741d6cd9e5a14db0610a81ad331fe5f6970b79", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "schema violation: expected array for 'record_ids', got string", + "evidence_hash": "sha256:daeb58be19013c7d57e44a5d6d2d07663371ed39555b69be9ae5f3d98b32333c", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b8cb282af5874ece6c5ee0692f13dfd9856214abd39dc61d336ce020cdf6375c", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d2dfb78ba2885999be7237f031675c3f5f25772dce62b60ca2be878aef048837", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2a8769f71c81358dcad429a24826de88d9b3288e6ee2063360274908b17f988f", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:193b94ca355858fd448e096ef0ead234a6527d29afd80202174ce08b919dec23", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:a9749803a22e214d236d12a8ad75adb7e24ec9dce03ae16ddf3721fc3eb27871", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:f971de4b9f71c4e73da595eed5f403d39205dd42b9df2ef20afeebec4c4ff833", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9473d03412a5002c2a5fd6d61ab21d4a4f101a70e53fc0a6d53007ebeb62ec46", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:16aa53474498905df2dde3f49874c325255ceb0eda60d791bc8687a8d387fc10", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:e6695a290bf9401dd8a5dcd62e2469aeae87b3a9f012bc955e0126aafdd6f34f", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c3153bc0bd71fd84adf0a0f1c3006462ec28872b646f430f29cfaec8c86a0332", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:65cb53117e032150df03de46bce575f919f0844568324d4b87a90994e53b3ff5", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:90f1ca9bc52ab3884995d4fd6484eec2113890605a8f43d415ebec577f8baeb9", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de741b789e59be3047b4b9d7bfb2616cd7fe46df9a2bac7f1c818bd71c47b006", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:921b89034ccabf91b2ffebd471341ccc5535e8e98d20b5f22687c7c46e9a16da", + "evidence_path": "evidence/20260721T151851Z-b0ab2127/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 21, + "failed": 21, + "errors": 8, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed1.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed1.json new file mode 100644 index 0000000..be1fd03 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed1.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151414Z-0bf5f499", + "timestamp": "2026-07-21T15:14:14Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 1, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:0fe73162728b3e8897e8399f7d0c59094ee53b9a1be95c9d3815d32408fce7ae", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:9cd49b209bedafc09ff342ae401ae65ff32df06ce6541e6828ea68f047cdb3c9", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c8b70ab3c5ad98e165035bf57e2f06ba2048dcfc17e4d45922e566f4bb3eca3d", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: missing expected tool call: 'get_temperature'", + "evidence_hash": "sha256:cc59fcfd7cbff8ca747239b2016e5a79ee2cfd61a432af8dd339df0cdc5d0553", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:d8bb789362bddb95e504bb70664d4f5ea8816c552a96761bc7f5c515fadf185a", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:58729d4da023eea631c04634e5158699c3184cee9c5d2b12f8e13148cd952594", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:6252ed46e4c21f1be413642c9f471823a7c9f34cc428bdf44e75a5306e03f86e", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d4a0ddf3736bc23c766c0106a84febb20d1b1ae9ea0bfca1be70bcb9eb09c7b0", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb16ca446d946dbcf098c1d999abefa7063bf9c9bdf7373b5a343c99898d5074", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4609b4a1ba164805f4022cfc3a094a94fa895ae6501fa31a45fd3add65ab54c4", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bd084359448c4b72c1d0b2aeb8d05d1c72944b8460b8a3e29802530385d7dfc7", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:d47035e0cbcd9adccdfe914e50f195619aa26262771ee1e33a24b6046194dbfe", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1a55d7b829d1fa7a97633b55bdeb57ec3411e0a1c92462489a7dfa47132c337b", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2060439847283b26d00ff555e5a4f074af7df767574090ce0cc11b9f80cb09e1", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:8735e4589c2d17bd5c223c4800d1415bc85c1805e327aba011d3fa0c1fe11ae4", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c0edfe4bd1bacf15097ac25ba7224a56650a6d5c814f3688645638c76caf032b", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:90bee07a2db9eb81f05510a1c7ae05554d0f9b43fe25a63a329bb20b3218d819", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f0ec494976a7b4df1df9e1cf4779d9b265eb528d6511ea25a18a2a627b5f712e", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'divide'", + "evidence_hash": "sha256:8fa15c79960423ff3df82d2dcf4ba3d42111bef642edc9c648e84abea27fa09a", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:962df2a77cf117ff0e3368eafff415cbab54b49fa59258696de290646194559f", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'search_docs'", + "evidence_hash": "sha256:b29ed2a6d5ba61b762e74f482bdbbeffc974509505516c6a5aefea94fa98d0ab", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cefb8f3b8ed103ef1c169244aff9b2c929ff13754f0e989dfbb71914291d214f", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:64ac02f0c63183730133ed65af0cc329f4b6f3bd11d59e691ebfed232bbd2739", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bc9b66cca58e40ed699be22ce25da11721f6f64e9e7e8ea8c627577fe193f285", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:96ae4cf002a23cbc3451b2a7361479d4dbc95031529e57ad252d6b58a466f1d7", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5e2b4407fd7cc529e94349e4dd2b76d80fc112dbafdaf5613dc18621268adfb2", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8f98199cf04ee6dac905d94cda01adc9d07f77004d2349f5ce80790236437c41", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7125d16d95936455aa26c11a3bf655f6af615ffb0e4d8bd507aa2da63f4ba1b0", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8ceff81d92e48f364f42d01510d461bf3112e3f81845ebd9876ffce405381612", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:334184b3e7b15c309918f0d7d59ffa77594b18f6d178c10e09728c36bbea8052", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:81f99647251dfd109be83fcc56a814f809c84527a7d047a8e6d2d9655a21b2d6", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b4abfa6f3a202e3a92a56464c25baf9e101fb469c97be290e68fd4ff5d90059a", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2700f473e18ad94797cb3d52c4de0a16c4d610d9d2321ee95503fea04396df99", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cfbb9b9815fa8d0a73b4ab75b5d7d628dda18311a7ef68db40a11a200b17c3e4", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1dfe9daf1aa0d6e7ee06a1e2c85cd159b8f1cccdcaef2243b257f4d90910bf8d", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7a0aa5fb2dc74c63284dbb5a5d86dd461a2ee2ea5e75d546be9ec9956b20ec8f", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0b087ba6148d161ec8e17cae485e6b055422b33d811c92c749e1a00b672c1bfa", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f7d9cb5bc478e9abea38f5ba72485f29c5b1b0613ed05e865782c7377a8bb87f", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3f191c07225c46fb7eabd0155227939f61a9c1a3937d22e638032f7c5883b690", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a0d300f14de9333f4d46d9f52010e6059c7c051764e14c5b932685f1ae7d60bc", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dcc6ece94bf587d4165faf6ae79daa48c9ce04582f3dab175622b5bedfd03ab1", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b57a6780f009e833f52cc6e2be05a325765bd3ae557ce8b8de0aa73c424615d9", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:48e6fc3708e40d6dd5d89bca357938930fe2b61ec03a6eb25c05a8fbd65272fd", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d69c6677d1b298c8aa00f0129f4dc3986f1ac9ba1805ab367ce49a23c62b1d7c", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7d99cf2de366536fc1416e45cf6c6c970ef022eb58be2c8c4a923da63fce441d", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ee21325fd1b532faf5aa93d261c12d3ab15d2d752e7c7d8619acb7cd1f9b66fc", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:98651dc0ff2188da1750b81f35e889cddee1e8e003e372d7621c8b3c0a692a38", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6d75a984968be88ff8244edc1b563fac4ced9686373845e1e1da68a7ebe642ed", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:67527c9474e2438ea3065a356c4d466f5b4a5633e272b8c763ffb8343d387c64", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6d80eb36f1f85085620c3d6b5d9ddee901a25e41589af39f2041d45779dc6002", + "evidence_path": "evidence/20260721T151414Z-0bf5f499/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 40, + "failed": 10, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed2.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed2.json new file mode 100644 index 0000000..0d18489 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed2.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151436Z-a338fcd2", + "timestamp": "2026-07-21T15:14:36Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 2, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:9289d6aedd55b1926e7bc07d5a337ecb2565fdee135a0e73b680ddf11251f478", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_flight', got 'book_flight'", + "evidence_hash": "sha256:4588a470cbacbd9c7060ce01d71084e68a16279bb9481630e9ad664a178421cc", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d00e40605860024a4d7f44ea189bd494e73077c541cc586ba1c686da13554f4d", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: missing expected tool call: 'get_temperature'", + "evidence_hash": "sha256:2b411cd9117499146051e47782d7cd123bd2a80459d7a89254390403a75747ae", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:b09feface2eb14e134340fc80746cb3c5f9e61c0971821e4c300b7a8ffa4ce24", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:45bd94c65c3e4520486af81eaec1f8de74d4dac20f4965ad9821d122e28fa1ec", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'find_account'", + "evidence_hash": "sha256:e6c3bc4802901a09654f6df8e3e75536d347f8157317885a23181c73a8e0d975", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0797dacdc343c6b678fb5e265146a85c39bf570640d7a871c1c9f2468f96c174", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9483d3f92bb39b36f35af6015aa728c0bfdd2d0c459e638a9d930bae98003b70", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:debe92148d60cde4c5c781cdcfd98b9a4843dc634aff039997bbb3c9eb672802", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9d1920b4906038a5268ad605b094b9ee942aeff567bae0a2cfd74027f6d93084", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefgijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:13afb7e548a632dfab3eb6e77c6997af7774ad776c5da3516c5ff10c121d29d1", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c2c0b5268c5997576153f134279a7a0e7f89945c13a8efb7f00d8c56573f22a4", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'city': expected München, got Munich", + "evidence_hash": "sha256:75bc9131e9644add751c89c298e5eaac92538d383c6614f8dd850cddb0b82255", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:25d362fadf85bb2f312b12916106b9efeab0fab597ff7517ec4659fda3edce6f", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fba3c37ed226e5053b08669fb8559d0b21d3700a20de061aa6af5325857fd59e", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a926aa51e2e7a3ca22103652025e5d70d30321a7f2b5881ce1fdddede833b4d5", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:96adee701841c1b4de778c5f6c51d39c664fcdf24350c8160821fdb82d879f75", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6baa7c88795e3a3ede4508eea272441f9c55ff887cbda7b0e239799c0f8790f0", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:032bb84ef0ddd9b23883f55160760d1d3fd42deeaa2369dac502d534480de7e1", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:48b6cfb150471c2205e26f34955d773febdbb3a93abab10e1fde7bb6403339c9", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5ff73b262b70d00646903c69d2b5762bc387617c99f26e49dca68ce6e82f0547", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9a1e164a2f960300e7ccb7b11cb4422de1526cf355b6e49a7456d6637a819ad9", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:35d73bd16fe59595f1fdd1439eba16944a54db322a647504e3ef0363e16e233d", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:719772da8d90d4796f5082b2dcd69bc99e447ab4198ca4f56bb764dce9e24a1c", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0aa59534b819da7d4556f2207d2a71b8fc925f2688d69e26c4cb41a6cb8aa0d5", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "unexpected argument ''", + "evidence_hash": "sha256:c605435db772f8baafa2fa669c74e43f5b0e02ca1dd5769a545ea1b8be83de3b", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:284d537007f6bf10758adc58eba5323cd0cc6892490c1935470b268c3b87f08a", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8dfffa82906a3c51d235dd2d6f98a2163083ef368e942aa821e74cbe6c0c881a", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f30b3f6cae2145877d642f0d14fba7cde7e71589511eca311475084965c6093a", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:adafe01e195a5f4dd2237f85ecc2b66a520017b7149b54ffb0d42219f554c7d1", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7127827857038581a5f5ae79c028b5a30779cb5f1ffac9bc9d3c71568cc61ea6", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:46a6576ec182d611dbb434a6b5e1d61bd53f763f2101075268086044eaa0b6b9", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f2b7c5abee64de10b24e1008db4764bcb7eab93de8dd2c3da4c5543a57e89670", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c423f55301f7f94d6ff3e885d5123f44e24031df64a315967fd1ee0ad144bf65", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e857d97816cf6628508be68bdaaffaa7f7cf1c36857b4111cebc76bebd3db411", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:290c81dc7adc5fbaac970d16d4a9ab4a8777b0abf502f1c1cc1ec95ec9d5dccb", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "unexpected argument 'type'", + "evidence_hash": "sha256:f7d71b8939dce159ec3fdcbe30f56dfcd97842cf9a3ff726170ad116adacef70", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:adc67b244b1b8d1ad1efe2021e5c79a6c4938b82d7355af941c73dd27c25d813", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b5b35cf5de64cc85f3f0bbe398233289e735250785c34d7ec6c38877a4042775", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:73d085bbfac3b12c858fd800f933f12c46052b4178713217d45afba5e85da5ef", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b1dee59a0248f00931100310f9537b186d79e144f097fd3d8eaf96ffcf39671a", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5e57a9ff3c5286c0c921ec4b330fc27526f3751e9142304db4f3c2c97bb4163b", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:91bec476a4a266975d3cf0f4f0f1a3243d6b34af6359a57ef12d8c7e9368bf97", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:78060f4b0466462381ae66b430754c603eb9cb63d9d800323ec3b315fc3594c2", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:23af405d54bce9c94f20bd7c55b7cf6a39b4ffcb5023b4d60a5b4c4ef98e4d71", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected argument 'type'", + "evidence_hash": "sha256:dc0c9b26647d718b73437150160007f93b6322a8938670fc6576fe7e04d5f27b", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb77366696913be7d69da4caa9cf29ca4d471ea13f64efe9cb1610117d24d640", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:82d3fe6da461b9022e4a14064ae819a9f05ec1e520098605b66cc1199da2ae7a", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected argument ''", + "evidence_hash": "sha256:b385abb1fae60fb55713f43cdb18b9395d30c69f5ccc9b69fb92eb8a25c509ce", + "evidence_path": "evidence/20260721T151436Z-a338fcd2/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 37, + "failed": 13, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed3.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed3.json new file mode 100644 index 0000000..85827da --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed3.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151452Z-a557f977", + "timestamp": "2026-07-21T15:14:52Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 3, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:07721f69b7cab7177f0ff64af4311f8c921c0da6d9ff79866be79f5f6c0ab08c", + "evidence_path": "evidence/20260721T151452Z-a557f977/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8a1ee54a1d8b87aecf781159db783a526eb81927734327d4d536b068bc5af9c7", + "evidence_path": "evidence/20260721T151452Z-a557f977/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:2568a920bdd6d2f3ca22113fca22c510837e3645663c4dae7cfcdf49682c40d4", + "evidence_path": "evidence/20260721T151452Z-a557f977/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: missing expected tool call: 'get_temperature'", + "evidence_hash": "sha256:3d519c36126ece2c2e250df6184d39bceab121f01e36c8b79563c30f9ba02b2b", + "evidence_path": "evidence/20260721T151452Z-a557f977/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:c4921b01f136e4450bc553f796215340557cf870da6715e24c8e4bb8fc43495d", + "evidence_path": "evidence/20260721T151452Z-a557f977/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:33b60646c67410a1899eaa6a7f14e62940f604a0db759f57227cf67aae9240aa", + "evidence_path": "evidence/20260721T151452Z-a557f977/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:fb8651eeac155faaf4b12877bc0878d44c9fa704b60247fe67542eee730f4ace", + "evidence_path": "evidence/20260721T151452Z-a557f977/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:337446d49b3def364f861a9c69e2f6b5d828cb4cee8a740398fd4d6c9165c820", + "evidence_path": "evidence/20260721T151452Z-a557f977/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2384fc522f43fde2bf7bc94dacb448e256d2e02e27732914b4c72a189ed25be1", + "evidence_path": "evidence/20260721T151452Z-a557f977/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a4bde700ab12533295c7c52b202567af3efde7de7e9add664075cb3292401907", + "evidence_path": "evidence/20260721T151452Z-a557f977/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e526e6efb943dfe71faa0089915fb7c11cd1819858f0651811c27c41d291e225", + "evidence_path": "evidence/20260721T151452Z-a557f977/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefgijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:82e80ec446af7a9ed716b0892bd65f624908fb2b6c3777d98c22558ddccc23d3", + "evidence_path": "evidence/20260721T151452Z-a557f977/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d718eb45b25262c7103b9d76f097bd2bd6683adb63ae076633a2bbf11296b529", + "evidence_path": "evidence/20260721T151452Z-a557f977/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:90d3e304b9ee2a7449e3cba9231d2f4230fa59798634709ba86e7e91fc613203", + "evidence_path": "evidence/20260721T151452Z-a557f977/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:08d0e39f8ff977e91aa434801c925e0aabb487e00046da7e06527913acc20464", + "evidence_path": "evidence/20260721T151452Z-a557f977/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:947c1a9f1196cba3e2da0ea800fcfdddfc327e9c445b0b9b214fe186576c52c7", + "evidence_path": "evidence/20260721T151452Z-a557f977/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb00f33c9f48ff31a10aeca0e0dc367fea94ecaa3af97eca20936ada07b69d48", + "evidence_path": "evidence/20260721T151452Z-a557f977/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5d52beaf4dd8e6466ae0e02111b9f19b2cb0b24442cd33ca88ddff0f08e9e7d3", + "evidence_path": "evidence/20260721T151452Z-a557f977/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:02deec4d31ba235fa6f87fca98d590ea82a98da133e8ed470b20276c8185fd9a", + "evidence_path": "evidence/20260721T151452Z-a557f977/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:278abf072ced950b81c8bd00f10e56ef97c9f1b380d9958ded4d6d9ca59f61a0", + "evidence_path": "evidence/20260721T151452Z-a557f977/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ba7e0f19450c037af40b0e9ea9002700cf238488ee975912a738ea75e62384b1", + "evidence_path": "evidence/20260721T151452Z-a557f977/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:45871e0cd61fa0ec80ac9318af69e999369939d3bfc41c3e249eee216978f233", + "evidence_path": "evidence/20260721T151452Z-a557f977/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f461157661278322ad987d7c91a82d108edbcca62d96573fcb431cf3d20be190", + "evidence_path": "evidence/20260721T151452Z-a557f977/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6165f1ccc920f31db4a3a6a24f2360aab75ce8d147dc38b18b25e7923a1dddc6", + "evidence_path": "evidence/20260721T151452Z-a557f977/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d62361d7006a34c5b9a6f616f5dc9b46fd58bd6c8ef665c9a5176ceed87ccbda", + "evidence_path": "evidence/20260721T151452Z-a557f977/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:29c5152122439e2903fb62c2c4d2f6fa6d93f4c2740e658e1efee113ed765336", + "evidence_path": "evidence/20260721T151452Z-a557f977/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ecfe51603bcc507bced2e18eab45975a1f4e793e16072ee148e80d351a550da1", + "evidence_path": "evidence/20260721T151452Z-a557f977/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f32cf6fc80dabbfc8be6d7e871b228b21790662b60b9af99af5b076c32433ff", + "evidence_path": "evidence/20260721T151452Z-a557f977/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4bdec8b6cd8c49969bdb7152df0448624522b37873820e0cad2def33de568423", + "evidence_path": "evidence/20260721T151452Z-a557f977/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a89ba7ecd2f5ee5fe2e9d90e80de41a687c40246defae1376c09b5e9f4b05a11", + "evidence_path": "evidence/20260721T151452Z-a557f977/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2e92e084a48e33f84ec4283260104b71f3523cf44c87dc200b54274ec06589aa", + "evidence_path": "evidence/20260721T151452Z-a557f977/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:62578a7d0170c2813e6ee4281514db2043967489f17709b6eeb20a800f7e8929", + "evidence_path": "evidence/20260721T151452Z-a557f977/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e354cb6245af5d6ad1549ed4af4eb865c2a54bee802d12f9d56b585ea0b7cd4", + "evidence_path": "evidence/20260721T151452Z-a557f977/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a2801b48b9453a71d627d7798df728d7a09e58903ab6b3d0203b4a0815e7ae6c", + "evidence_path": "evidence/20260721T151452Z-a557f977/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f7c95cfc6b7eb387ee36cdd5f272d4f411c252aee39ec2bcc1c794f7dd2745b2", + "evidence_path": "evidence/20260721T151452Z-a557f977/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6618b3dce8d8de9661f251b24c4376b1828554e1f1665e84db04a3c8dc4754da", + "evidence_path": "evidence/20260721T151452Z-a557f977/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:870655095d5c6d5c1207121046c7e698b095caac67014d995b9da5a470582bc6", + "evidence_path": "evidence/20260721T151452Z-a557f977/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "unexpected argument 'service'", + "evidence_hash": "sha256:40c4873ad13fb0cf101ce338fe8d1c3bf58cd7fb804c36562a680c5183cc1429", + "evidence_path": "evidence/20260721T151452Z-a557f977/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d3844d097ee6c96aaf3886b0bac025d7ba402aa87baadd9c0e541f975355d882", + "evidence_path": "evidence/20260721T151452Z-a557f977/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2cc51c698906154e450e016e8fe8d27e2adcaac2bc0460d1a0b6cb9db09258d0", + "evidence_path": "evidence/20260721T151452Z-a557f977/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:82efaefb332e3fcdd6bc107b6d368dc7879272256c8328aca7f26f4599c4da62", + "evidence_path": "evidence/20260721T151452Z-a557f977/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1ca75ba0173886ecc51f0483693943178da35c16b94aedfa9ff8028d8f4626e3", + "evidence_path": "evidence/20260721T151452Z-a557f977/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a6962087a0c6984b1f90c4c1086edc53901199ba7edcb1ef79655351f4e72bd7", + "evidence_path": "evidence/20260721T151452Z-a557f977/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:435eadb272490ab62e889419128a54f34fabc587a1a67c6df863003bb180f679", + "evidence_path": "evidence/20260721T151452Z-a557f977/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7fd42e906aa355521b7c68b23e627793f4ebf9be5350ef1bfa198b22ea407731", + "evidence_path": "evidence/20260721T151452Z-a557f977/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1890577359037379edd1070b5018319795105fe395364cda26ef77dc43c11738", + "evidence_path": "evidence/20260721T151452Z-a557f977/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected argument 'type'", + "evidence_hash": "sha256:4ac297296ac6ee3ce66d0cd3d741b2bfd89d39e134f7116da65b6b2702148e4e", + "evidence_path": "evidence/20260721T151452Z-a557f977/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:702bc8c8015b7aadbb16915297c7ffa6167ac04930bb3f22d500352fe0ea4bd8", + "evidence_path": "evidence/20260721T151452Z-a557f977/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a44c20717134e1e90062e48020f3c16b95c5794d04280758b575d4d468bfc403", + "evidence_path": "evidence/20260721T151452Z-a557f977/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:53fa51110e97bb005d95eda709842629414888bde7f32f6b7eedd03c8431b300", + "evidence_path": "evidence/20260721T151452Z-a557f977/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 39, + "failed": 11, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed4.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed4.json new file mode 100644 index 0000000..9b683a9 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed4.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151507Z-a75a8dc0", + "timestamp": "2026-07-21T15:15:07Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 4, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:07178230cd80edd82630fa0166b73217863e24a9f3a19b4e7b1d32223ad1ed5e", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:76675d1c91d56b58ca5681d4367387f998f09d5e972b7c057427594581b87d4e", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d2ac3fdf2e28514c100cdb79595ace05b7b2ffc5cd73d57912854dce1f26fcc1", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: missing expected tool call: 'get_temperature'", + "evidence_hash": "sha256:1e3c937b5f5098b800ba87067cb27086b0f10876bc0cb5f1173233d10a65db72", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:5aa1ed5a8d5356765e3f6deb6942c8572a89564f5c03f00ed440bac33b668e09", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b86dc93dc0c27dfe0b3fc39ef43c7aad75b54f7ff5230f08c11e981019ed8ceb", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:c3412a451f6f35081aefa08723e3fbbb4dd69d3d4e505824e373b0bcbb9f7aa5", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:264ca9712a2fad3a951629d08e65409f3d5f16f4a372cb51191d903880ad315b", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9e0573b97269cdabb2eb286219624cf89e956b9ccabb61cd47a6af19d63ae471", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e8dab9336b868ccd3a0bcce75b31fc988aa413e24b19943e8172f8b5d855e36b", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:8f02bbf533436d1124c9bc12947bfc76db5d40433d96b077c8e0f43a9d5249c6", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefgijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:d843f26a932c9e870c40bc11bf915066da184092f196ffefcc9e8a16dc0c7323", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:88814b19190f15ffd907b52bb07b72b33574d6d7e4681f3911d187b6d4122e4a", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a8eb0f6e7dcc46fa6677aeb7995be652999b0ffb885ce13c5f458a4daaa2f102", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:03e92f110230b858363084a85f7a869e7beaff5bec397bd6906c431c53ef9f8e", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4a99a7ca2a75a8aa4f117e23edd3e01da869088624ef12a77f7a509da3e82efd", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:878daead4231bcd73089e2ac1d77399cfe5e164dd92e850b18a9859e23af4bc9", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:549b8c087c34d9e6d1f139dda2c7271f5976d6c525f3c2b4c0185c8bca607766", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7548e904031871d15bdd28ee0d6675f9c4d30d0d2fb32f08d68fcf9250b41fe8", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4d95d4c5ad43df3a5a6f9d7730218cc519aa8d583a5398a17ae235aed0240cb7", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7bb990b6b89294d919c8335318d33331f389a612921881014676c997824eb895", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0355c039664075712225ce55f94d78bef9e55ffd9a76d80cd78c49f3019ff6a1", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6faad45839f8b7bc14d03ddef2aaaef8718e6625256d404efa510fa65929626", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ba0da05433b6ce95d3787c79c2054477acd21c6efb915d1df59fe562d5c91707", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:99393ca1e21e4671481e49a9346d7a0c318d3f53e1ba2dcbb617c7c50dabe8e3", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:412632be62bdd2f657a73d9b4efd5a21e5d1109bce16213033d14299f1951332", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e15d3aaa7e0541cf2f3672677eca008421d16c88755e760d5eb8508fdcb1c860", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:73621459edfb9ab831e549662407b96847e450b6636b22361f82ddd178440252", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3fb6ed34741f6d322d50d2c7c839cc44c4b07ae26433ae47289d91be7502a157", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ef053bc2534c1135f492b1bd38f1db5176cd680b3c75d86bb4b5645812618893", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cd00a3a9bf187e2a0cc18e342f02e480adda1007f6c251a96dc5fd3c2edfe3f6", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:53a9dfcb4a142218dc0814331d862e89d3eddb3a05d58ea269719c689fa2071e", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:839c50e904069a2b8b3884a983bd30590280646325f23bf0f43a7b5c945e4e0d", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6cd87281b26178ec8907ced8d1c435a854065715b1da5ec6596f45fd41480088", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:122e95784f4e2adcfc82c3dcad22aa10010236d16f10ea8c72274c0fba0a3c43", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:82695923e5ebc6c59c2587e56e3df40dc34d80ed6091d76f6d0996ee8e175d77", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:74494ff5186746deffbf2bc190b161306c0c83226d12c105b0699c7b789dc331", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "unexpected argument 'service'", + "evidence_hash": "sha256:a0393b53dbf8ec4ffaae5c3501ce7ae629fbb18316157ee17553e81002a15b56", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:201cafee37481490e97be73244ebef2a32bfe25a22dd6aa15c168f155f0f02ba", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:581602222499aecb96e5b248db942a9c7a4402701a158749e1c7714f988fe56c", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:d74ddba1fd5b94d44c01782b4a8182f593605da1e16ad7c3594fe994a73cc39f", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1cab0bb50dc71f99370adee649cd0f46e22afe4bca0e3922167992596ea7938c", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d28d7920bd45bb234e6b9edefa1dc60654d370b2ea89794a186ae79fd7bc048e", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6e1f1194079ef9e36653fe8719633ed0ecdaae4084ce18ae754f8f9930e430ab", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1b7e4254fd7f4a126a73cc04920cae24e94830d4a96b9fefecfcb9770b0fba59", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5aeb63f5a9fe3f607efde66a0e080b3339113d8bc6b13a8f3535c5b36af1d538", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:68a1ef1ed139deac5f64e997997923c951eb8b59c657fab7e330f07e1d30b903", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dd626da489be1c8573795c77a55815698f55d0df8d0297c3b32b53986d78e1c3", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:09cad72558b4e4ab7bcbaea0d36e4c78b59d5394c2f817fc14c72fcf8c9fefd7", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9e139afc07c6c7c4ff52b2f8ace6ce31d1fb6dc3230e1895cefea304cc6284a9", + "evidence_path": "evidence/20260721T151507Z-a75a8dc0/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 40, + "failed": 10, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed5.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed5.json new file mode 100644 index 0000000..33ad1f3 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q3_k_m-seed5.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151522Z-781a4a90", + "timestamp": "2026-07-21T15:15:22Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 5, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:60875e05ecb2ca5708402d642e87eaebe1a49cb61ec8fde014c2b9caf186ae0b", + "evidence_path": "evidence/20260721T151522Z-781a4a90/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:d0ce3b517de2fe7448a2607456657cb0c7d988c7dc54256a19c29fe1299ae404", + "evidence_path": "evidence/20260721T151522Z-781a4a90/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:b6006d4ff42e25b7fc5d50c09458ac3d195a86a1eb1bbe3bc456634b50e398b4", + "evidence_path": "evidence/20260721T151522Z-781a4a90/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: missing expected tool call: 'get_temperature'", + "evidence_hash": "sha256:92b091154679423b9218657ce44149ed3811b4f1773bc8664677d1dc352f7bd2", + "evidence_path": "evidence/20260721T151522Z-781a4a90/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:8eaa12531b7bc3484b36a85c7b45a9a6109f31b731a3882074e18530b9d22e36", + "evidence_path": "evidence/20260721T151522Z-781a4a90/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'", + "evidence_hash": "sha256:46456aa26efc258bab0197e0b108f0c6136693fe5ef440ee41aab8feea091c76", + "evidence_path": "evidence/20260721T151522Z-781a4a90/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:56f9391648a1bbe9bdf6670146d7e52b17ed9130715e4a9c04a88f065f7bcd01", + "evidence_path": "evidence/20260721T151522Z-781a4a90/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f57ca60783d7e7c3c4285dbc1e38c7c73e3f0207da6bb76365960cec5de5802d", + "evidence_path": "evidence/20260721T151522Z-781a4a90/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:26dfa822dc3789b21998401d97b1c9896aba17709f45aeb36aa66d7d14fbddaf", + "evidence_path": "evidence/20260721T151522Z-781a4a90/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e17617e3d8e8965a531f96d4a6939a986f44f437216d343f77ef7a53611b6b8", + "evidence_path": "evidence/20260721T151522Z-781a4a90/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d3843505d6ca14eb88ca43ec74736fa4b0879c4061966706c08442c8a3971945", + "evidence_path": "evidence/20260721T151522Z-781a4a90/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefg1234567890abcdefghijklmnopqrstuvwxyz-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:6cb5c516406fb3c85c391abd0aa57d332f9f71dea2cc0dc04e3cfe1eb74f8462", + "evidence_path": "evidence/20260721T151522Z-781a4a90/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9e69734d881b4eeab8f25370d77c6f7c19d8c7db6b483368b815705c4029a3da", + "evidence_path": "evidence/20260721T151522Z-781a4a90/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f572ccc562938df89fa51f57ca1b94b5cae9ad53ee73a2f8483f4f3584acefb8", + "evidence_path": "evidence/20260721T151522Z-781a4a90/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3816dbedc9642b8aab51fe2eebfe292100a950eed88a3b987c9bbc0025839a8a", + "evidence_path": "evidence/20260721T151522Z-781a4a90/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:87b31dfe93a62e471fc054d6555e8d8d48b846e8819099b6d7d3063d500064a2", + "evidence_path": "evidence/20260721T151522Z-781a4a90/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4ccca9d757610638e7893fe84532b1af2fff5e425d21fc661ae6b354eb63fb2f", + "evidence_path": "evidence/20260721T151522Z-781a4a90/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:364f8c5ac4f156d152033db0c485af08282cf1f94a6bf75c3ee8a2ed1fd6a77e", + "evidence_path": "evidence/20260721T151522Z-781a4a90/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a5b43c72123945a18a467c84a67f4f7fc72da4c0a84b3480671f5ab9479ffc11", + "evidence_path": "evidence/20260721T151522Z-781a4a90/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8862041a8fcde4797e4474e50e79d77328c86640b8a882be8b320ee46769d2ac", + "evidence_path": "evidence/20260721T151522Z-781a4a90/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3b36163ad6688c5364ef1a343142d39581aa1745acc338f3f7443ed0ffb91fc2", + "evidence_path": "evidence/20260721T151522Z-781a4a90/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:587915cb166e3f81c1591488e9cec817aadb58c23ed706ee8290397a7fadd34a", + "evidence_path": "evidence/20260721T151522Z-781a4a90/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:41941449c7039cf2b335c4c3f6744007a431584f94e9164e756b89a3eb4b832a", + "evidence_path": "evidence/20260721T151522Z-781a4a90/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:41d8d37bbfde4a4aeaa0f6f55e3aee8c5e86a85a54c09141ca96b1197eae7378", + "evidence_path": "evidence/20260721T151522Z-781a4a90/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2e39a19dd242366d3f66a3573e4242bc491d69947cfe7b88d535bcaad423a5ce", + "evidence_path": "evidence/20260721T151522Z-781a4a90/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b2195e3f1ecfea3a8673f9d3f19fed5e3feab3ec69e15a273cd3f3f52fece7d8", + "evidence_path": "evidence/20260721T151522Z-781a4a90/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:22743336411bbc44b417f83dc882c7ff28fa0878395664ac1dded917ff4e4451", + "evidence_path": "evidence/20260721T151522Z-781a4a90/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f51a9cdf2e419c65e76b89c4023e516a6b529a15e6655787b0fb0e6a56cc7037", + "evidence_path": "evidence/20260721T151522Z-781a4a90/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d5b6b8ce6b04e4319d0f0ffeb2c0488a5ecaf0533c6d4d7da967b974e0d1c88b", + "evidence_path": "evidence/20260721T151522Z-781a4a90/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fe0c68b437a4a9343843eea25f29d5fe314f131fc015d14c24dad2556cfedbc2", + "evidence_path": "evidence/20260721T151522Z-781a4a90/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:019f404c4297c7f33058eb7298a7ccae96b832cf4dee3cfbb30470bb32771222", + "evidence_path": "evidence/20260721T151522Z-781a4a90/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "missing argument 'limit'", + "evidence_hash": "sha256:40dd987534dc1d3c629e266501c94f235bcea2ec49e1bd6afedea5440f68d7a0", + "evidence_path": "evidence/20260721T151522Z-781a4a90/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:829e2ae44b3c1ba7520010abd061d1b7138afd43914f9e9788679b04a5a95591", + "evidence_path": "evidence/20260721T151522Z-781a4a90/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4fb51fc367ee232624ae5bec4ed893f08e2b602ccebc1d30b72a79827d84661f", + "evidence_path": "evidence/20260721T151522Z-781a4a90/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:42ef769d968e84d0c7e4b1c3f1d664d9bcd677fe79d309087a01947cc8ed1567", + "evidence_path": "evidence/20260721T151522Z-781a4a90/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a336ef2c5a055df4fedd9df368bca6707d468acd458cf2bc02c2099df2918143", + "evidence_path": "evidence/20260721T151522Z-781a4a90/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6ea7502463e355b2de7f1c238b6c74d5938364580ee3e7e4f7455d9c1518e98c", + "evidence_path": "evidence/20260721T151522Z-781a4a90/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "unexpected argument 'service_name'", + "evidence_hash": "sha256:2032c1efa0c9c3812d8ef91d9cc4a41f6c7f4b04b6cdddc24eb09b723c02386d", + "evidence_path": "evidence/20260721T151522Z-781a4a90/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:57ed9020867b12777b981ac6065e4d46285574c6fd79927316f124dc5ee70aa5", + "evidence_path": "evidence/20260721T151522Z-781a4a90/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d0509752daf49a615260479620d6757d146e050b4c2929fb42b2b94d35fc38f4", + "evidence_path": "evidence/20260721T151522Z-781a4a90/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:105daba13c70addb09781601c87025a36b67fddd6c5e2dd6d62b3cf6b96bc63c", + "evidence_path": "evidence/20260721T151522Z-781a4a90/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:14959f0987497fc20629a90aee4af6a996c76b08300f7674daa342d9881d87ac", + "evidence_path": "evidence/20260721T151522Z-781a4a90/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a2b5137ebf69ef3a8d005bd46f1dbc76638597c0e5c99dbc99c0bfbcc8399a5b", + "evidence_path": "evidence/20260721T151522Z-781a4a90/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de3b82a20264dfdb522b0ab07e76a0526a842c6d4be49d8ac04f4a856e8c5899", + "evidence_path": "evidence/20260721T151522Z-781a4a90/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:958bfa6514acaa03fba0029036d4ffc025cfa0e302e48bfdda277bde129b3879", + "evidence_path": "evidence/20260721T151522Z-781a4a90/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0df6bb477bbdc884042f2316f0ce0b62414be326312f43beb239806fdaaa44c3", + "evidence_path": "evidence/20260721T151522Z-781a4a90/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected argument 'type'", + "evidence_hash": "sha256:35972feedb116cb75baf5978e5dc4e2fd29cbc7387c650430f7d89c8ff5722ee", + "evidence_path": "evidence/20260721T151522Z-781a4a90/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4e1815dd6216dea56db88025ab9d6dd7d1a0d46be1335afdff8c3383e424b255", + "evidence_path": "evidence/20260721T151522Z-781a4a90/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7d64bb42043c50cfbe0e8f5c8f31d9d1340d84ea83105569d8f27895606e11fc", + "evidence_path": "evidence/20260721T151522Z-781a4a90/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1d7178accdda966e165a4a974316a0c8e03bc4f62fc32d8da7a4c519a5e75d18", + "evidence_path": "evidence/20260721T151522Z-781a4a90/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 38, + "failed": 12, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed1.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed1.json new file mode 100644 index 0000000..de31aa1 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed1.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151236Z-7e333c54", + "timestamp": "2026-07-21T15:12:36Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 1, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:8d107c7944951d0bbb1f3cff4b28f4f44db0d355abffedadf3fbf8f2900e4906", + "evidence_path": "evidence/20260721T151236Z-7e333c54/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:44bf4556b510c16d773e6f03f88e938f659761c8f808efbfd3842b69b767e607", + "evidence_path": "evidence/20260721T151236Z-7e333c54/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:3bbbc6673a10cca1e9172bd5b41762fd98b43764f21858ff3119b0d093f6773d", + "evidence_path": "evidence/20260721T151236Z-7e333c54/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:d575908ca00c2bc3f42eafc432175067e5addc4ccbe976a444adab9e224929d9", + "evidence_path": "evidence/20260721T151236Z-7e333c54/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:81b71334f8679e10dae2d3292ec334e6d7cee4926fa66ac5280efc541f3067c3", + "evidence_path": "evidence/20260721T151236Z-7e333c54/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:5b8ba890a7674c3a525b4b63150d27d66fcd45fb225dc97ffb298f3b86fe1b05", + "evidence_path": "evidence/20260721T151236Z-7e333c54/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:fc7974f200ded343872fae4e1d6818de7e7919fc3d3555d1efb0813d79aaccc9", + "evidence_path": "evidence/20260721T151236Z-7e333c54/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:46fcc8c4fc252d4d81a4b04f9ebcd61952364fb8d0f52236a075fd0f97cc9994", + "evidence_path": "evidence/20260721T151236Z-7e333c54/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:737bf19e0f2f9e7592d0f547ab048429302bafb1ed0982fc23fa29911238a832", + "evidence_path": "evidence/20260721T151236Z-7e333c54/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6a2f73f4f0684b07bb2d245dbdd8bcffad254890ca1d5bcf1d466e96e58334b3", + "evidence_path": "evidence/20260721T151236Z-7e333c54/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c1035b711d260686b83ee00b40f50169b21c36f785ea5d0d8f60e55fe4a6668a", + "evidence_path": "evidence/20260721T151236Z-7e333c54/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijkmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:78e04e2e815b5622cad0917d78ac806368b909750728731b7a0cca5bb4f7b18d", + "evidence_path": "evidence/20260721T151236Z-7e333c54/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a33a097644ebf467cdbc9a79c22935b23a942799a4f7a563ab4f47091d4d747f", + "evidence_path": "evidence/20260721T151236Z-7e333c54/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1f9c0aad4d69a3a6094b1608c644eb2f1954bd331a0fb1c0fca59285ac92d0b7", + "evidence_path": "evidence/20260721T151236Z-7e333c54/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ee7d1d7919e4c12e8fb716f03d0406f4949a4296f782a31b7d25ce8dc4ae3efe", + "evidence_path": "evidence/20260721T151236Z-7e333c54/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:27e0a7a5cd328bc7144fb7e4d587e06381be0b92648817c4840665ec6806b31e", + "evidence_path": "evidence/20260721T151236Z-7e333c54/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a6902fd0127454fd037ba7e8836b5f7a21cd27b0ab6a12b872795d3081c74d36", + "evidence_path": "evidence/20260721T151236Z-7e333c54/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9142855910611892900a82af8f8edcc2a4471cfd41de9fbec0a3159c5ca13c71", + "evidence_path": "evidence/20260721T151236Z-7e333c54/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cc843f532e8eb183ce0ce7d487d9d3241a7f3707f585f38b6af10de923c3e187", + "evidence_path": "evidence/20260721T151236Z-7e333c54/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:982624a24c32bac80081c9fe888f28fd2e5be9ff3866b4209387e8f7e6c59490", + "evidence_path": "evidence/20260721T151236Z-7e333c54/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:30cbd08c4a654359cea3648c63504216c1c7d1c328e544bdfaea2901a5378da5", + "evidence_path": "evidence/20260721T151236Z-7e333c54/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4f18faab3fcdb17874b4c432a5ab22d7f4c0c700709f004788eefdd900e04e00", + "evidence_path": "evidence/20260721T151236Z-7e333c54/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e4d4b9f23ad29906b9c31e00665fb7a6f09efceebdbad678d635d287d57f8f9d", + "evidence_path": "evidence/20260721T151236Z-7e333c54/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:96f4566de437bac74ce320067bd6a10423db29d8774cc684555f2f0e97ca0c02", + "evidence_path": "evidence/20260721T151236Z-7e333c54/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:049b35a34c352b05743a5093aeba498231b60d3aaa9b2df6d61361afef318d88", + "evidence_path": "evidence/20260721T151236Z-7e333c54/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:16e22efb54527ef41a00cb281c1f13cb0b43fb86fdb570f3fc2173c02c9baab2", + "evidence_path": "evidence/20260721T151236Z-7e333c54/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "unexpected argument 'service_type'", + "evidence_hash": "sha256:edc5bea178adef38effa11c74437fb7be451a6c8936b135200a66ad1b9fc00cd", + "evidence_path": "evidence/20260721T151236Z-7e333c54/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e6443119046b35441b3d1486c768644527a95d7ace7b68bca2ecefa5539a35ee", + "evidence_path": "evidence/20260721T151236Z-7e333c54/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6022e88f59eeccbe8cfa4602a8a1fe81255423715896b7b6e23848adf36bcf98", + "evidence_path": "evidence/20260721T151236Z-7e333c54/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8319c1016604e77aaeaf7c8dcbc0d2ee7ab1cb4af304de92c9efb35ad35c8ca6", + "evidence_path": "evidence/20260721T151236Z-7e333c54/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2a1104ce146410a5b66735482ee4c70879f6ca4a36187b191b6e60e4e0dd7397", + "evidence_path": "evidence/20260721T151236Z-7e333c54/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c04f080711bb5aa641287c420ddede6d3792eb00cd1affc00aa0aa56feba6c0b", + "evidence_path": "evidence/20260721T151236Z-7e333c54/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d3a5609f1e642aa8c4d4c45efb2609f442ba76e8b2ba210db990f3b995561a7d", + "evidence_path": "evidence/20260721T151236Z-7e333c54/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:5876f1532af6dea4f459753e9232ce91c237a15103bad1548cad94a1ad3fa89a", + "evidence_path": "evidence/20260721T151236Z-7e333c54/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:034418e9a24eb8294e1ad617eb65f61c2b329b3f16e684bf62ef284bd7d18501", + "evidence_path": "evidence/20260721T151236Z-7e333c54/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:87864b723215012574ab369a78adba8121a9a4a7580fc565c8576dce4ce84e09", + "evidence_path": "evidence/20260721T151236Z-7e333c54/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:47fcf79250cb5af6393533a7251b2ee75c28253b03f3dfa2a2e5a23671c79731", + "evidence_path": "evidence/20260721T151236Z-7e333c54/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "unexpected argument 'service'", + "evidence_hash": "sha256:adea5260b0632a5b5f7e42eed6b8454c63fa1535dffb133e4a8323617e8aef0f", + "evidence_path": "evidence/20260721T151236Z-7e333c54/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f6f67fb3be31f3745876b5af3d83ca007a9308ba5af57210bb138e33c9e74b55", + "evidence_path": "evidence/20260721T151236Z-7e333c54/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "wrong value for 'name': expected origin-shift, got origin_shift", + "evidence_hash": "sha256:0986d91c8e68158807ca5e6039b67f88e44a2b08a751fe267f96dead1e8049c8", + "evidence_path": "evidence/20260721T151236Z-7e333c54/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7568964571709c88ca3d743cddc36fbfd09b6f0f52fd4634aaa56d9c0de3aab7", + "evidence_path": "evidence/20260721T151236Z-7e333c54/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a18796afac145c625c6383cbab4a0d9cbe068cd7c8f96ad48bb1be3721a777b9", + "evidence_path": "evidence/20260721T151236Z-7e333c54/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a7e9e006bafe9e820baa88674450619840e50a62801bb6b8b898ca255cbcb24f", + "evidence_path": "evidence/20260721T151236Z-7e333c54/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e0e2b2baf3e010932d165aada19355530052b555cf32e8391996e1e3d565c143", + "evidence_path": "evidence/20260721T151236Z-7e333c54/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7287f2019f00f279dad9ad161b9d5e67c00f2a853cb50f98a651f13de4d09ea6", + "evidence_path": "evidence/20260721T151236Z-7e333c54/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2315d9f423db39e8aa4f875db818b639f7d53f47d4382c07fa68c7d233c16253", + "evidence_path": "evidence/20260721T151236Z-7e333c54/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1ecdd6c8f2968317d8b72db8e2574d747365a8729613d74232f0068522e042c7", + "evidence_path": "evidence/20260721T151236Z-7e333c54/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7f0b42d1cfbdea06f7ce76de218706bbf7b61386842a0635fb549347bf72faf6", + "evidence_path": "evidence/20260721T151236Z-7e333c54/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4e12f34be6af21528e4a4ce1de769853cb98bdac1828373e4d044df107f63846", + "evidence_path": "evidence/20260721T151236Z-7e333c54/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected argument 'type'", + "evidence_hash": "sha256:20bd52ee161bfdba85ef8b9eec3e0b8f9efe78c281b2f714e3006e7e933626bc", + "evidence_path": "evidence/20260721T151236Z-7e333c54/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 37, + "failed": 13, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed2.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed2.json new file mode 100644 index 0000000..327c304 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed2.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151254Z-132cb9eb", + "timestamp": "2026-07-21T15:12:54Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 2, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:0444981110a1352f3aa1abc3701d7c237bfa71035f19e4711ae21ac2630e0027", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:4b0cae3ea6a28ffb80ede647d040fd41a8777dc83fbe666fcba2e24fc829404c", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:b47e941ebd92ea341545138764fc15c3c87ff68b42266a5d78f1f89c18a423f9", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:5e158a7706aa78472327baf7dca50a27bf7d0d632fa0780a367b3103cdaaa7fc", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:a2319e52dc95b1c0af4838bbbe74b12ecafcccaf882e28cde2e23eb9c13659e9", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:52c1545b50c897c2cc4a110588d34b4dcbd47b766fd4278d97c5b72d527d04b2", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:383818d07147f7fed4747bc524574ff63c0b87cd0054c1793191644b99a7881f", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5d4a386365c604e2cceddcfbe7ee5bd3e4fc754ad9c98d228ef994ace7b98cd5", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fb630d78e42aa7c934477f8753b111f5a8d70a0565079834f1d7ba3f817b0cdb", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1144362ded6c3b82eae59f5e924e3348b79a035b0d8f1f37db08db56ca28ec51", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d86b781afec1184cc1019d094ce66f6346a6a2ba483cdd3b5bf9852dc0fe7dad", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefg123456789-abcdefghijklmnopqrstuvwxyz-0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz-0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz-0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz-0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:68b9f5ae2c9fd752b64f3eff2566d79525417a2b828c92db0726f2f7e9219b6a", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0d3d1b21c4e0174bbde366e8d64a5e1e6df67c4bde60537db6b254115dd2d0d8", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5cfe7a70bb9b8ad47e1958191d0d1413397262cdc9b0cf59b45953c944b0792a", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5033c9d7521c1aa7a5dee3b4b8de3bbc0003b459c69d84c7653f4fcb10f44343", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b079688cc7dee5507e7a4a2ca38ac141af1d16e5ea681f0cb9c73c9e2a828a9d", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:52f7c5ca19a36da9a000e0d39ad8c6c93359410b8e04846bdc5a31250c20ead0", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:05635a4c94c0e0d5531a17ae85f84d19893786262fe465a3758d3205b31c11bd", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:be78192a88a772e65ea22d701ff529d9ac07c0ed3eb2bd77686ab38eb8e89ec1", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5edd73a66043927bcb066ad5ad2af47eb9f81fc7b8dabb3cc91d758077f2ef14", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6a53c7c4c82b626716c6805e7b28e70384bc930820cc8773eb90e43328c47a3f", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1eb08ebcb3623b5abbe7cfc4220116d7837e6d94caddb0d78e42895738a90d2a", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a1b223622908586a1ec218ef7e723419c19b43b2c858af5af2d05b94eeb519dd", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:68533b03ff09a5d710784d929480d808f798123bc52bf49929892e1922dd3035", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:77e34a064edc5cd5dd5d70ada6dca5a25fe1af2e48886fa616d893e87a9bbaa2", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:862f2f5483796432c4e8c3c9f910b180680dab1b8980b55cb3620b9b02ba4d80", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "unexpected argument 'type'", + "evidence_hash": "sha256:66a2e02f587cb6a06fcd0faa168d1b2be18d00d2ae480bdc1218adbf728241b3", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:07cdb54fdde3a5a79646c2f701c4a42b9a667b53e43fb6f50a689a0d8e89f0b5", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f66be112429faa89171433a2af54eada15d55f819e1f72da753799a045c9d70", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5864267e17227ef78375fcb995b4d2122cefccc8ee30f8560b217e7a0b49065c", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dd45f514d9f1918716a8e6a75f02efec4b330f2d6817b3a1f6552d69d04d4545", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d9867cd89314e5de1bf6198994131cd4382d087e810a6cc49e937585cc54e8f2", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "wrong value for 'name': expected road-test, got Road Test", + "evidence_hash": "sha256:503da6809972c97ecafb9aef6cd653e0ecaf667f36b70508fa1d90a7428b14aa", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2b891d8396e6ae15a3ec9d25f21ed18b113a76cb833b3fddaf5a1c64f652d7ac", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dc981de1f8040515c23f511caeb4ef29c0f0643fb70e7e1d7b6860c06b447a2a", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3cf6dfc4f00dcb78632df818e6afcece023e6a86a13ccc1df0282291edd893a1", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:128ce7e1846155047b1bdfe295b3f2d376ea5e8248606804e91d4f01d1ffc155", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "unexpected argument 'service'", + "evidence_hash": "sha256:09d9c8d5d6608430124138a4c470982670dfd21af61f38f48f5438c4640fc3bd", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:779bb5920f47ff46dbdd53127ab70b35801fc6bdc5623654e86bd0d11910c903", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c5a4e484ac2c47119e3dba6f1e281cab793295e78aae0e6dffa15fdf51d6af16", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:71d761dbcefe94033b6191284f434453bedca81c78d4e3c2bd770ac15ff94273", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fe51db66ed3cea5291b97369027140ec5569c309031156471f0df5cb47fcbdf2", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8a89d78632c8bfd75c37b513feb821facf0207b2b0fd0ea36667fd0d15c7ee42", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fd894a83d19889d78fe84c33d391f30b9dcfdfb885dc08c391b790b43d8c9065", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:61773724a6fb57eab9c49ae84cc3059311bbdc966a3d98f80fac8f724b8816f1", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b75005ef4da35afe1e4bf94e7dec95842d1369d287e373121dedb426c91c2c04", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected argument 'region'", + "evidence_hash": "sha256:0c66a1f04c88da7252667f37c9a650fa7679e6a2caea1520403396ff550a6ed1", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:016db0417aefdc87a7b19741798514eefc00cfd5492bc6e7aad65d8fb601aa47", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2a928c5d1a790e3dbcd04060af23652bcbbf36d8f8df6451bdbd3ffe52b0bf54", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected argument 'sequence'", + "evidence_hash": "sha256:da8a25397cd2598df4694be315bea2dc125e77004184e062bdd551a666203fda", + "evidence_path": "evidence/20260721T151254Z-132cb9eb/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 36, + "failed": 14, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed3.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed3.json new file mode 100644 index 0000000..1fb3ed8 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed3.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151308Z-2878ce93", + "timestamp": "2026-07-21T15:13:08Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 3, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:27e763e2a53fcb0167f06e117e131e8ae7bf2fb1d87159dccae4608961d9d866", + "evidence_path": "evidence/20260721T151308Z-2878ce93/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:ed1cd2038a7a06c69682fb044371a569b596254444b8318b37ed65ea29f3cd3f", + "evidence_path": "evidence/20260721T151308Z-2878ce93/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:1d88de0ec45284ad90e73ed473b1ecf6dea8fd9e9127bb14b9ca66d2542a1dbc", + "evidence_path": "evidence/20260721T151308Z-2878ce93/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:35381898d34435689566068f9f6e9d24e350f34395a49f1ba055ebf748fa554b", + "evidence_path": "evidence/20260721T151308Z-2878ce93/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:67e9e18a69e697ca480fdc3fd20202e85317a0a132d1c3dab586b7057e143544", + "evidence_path": "evidence/20260721T151308Z-2878ce93/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:4fb082d489bd3fe730fb6176695b155f972c6434ccb480968b0931aa0d7d5df3", + "evidence_path": "evidence/20260721T151308Z-2878ce93/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:3c771598b9270ec1a8dacfeb4450f698f128c8b386b8a9b9d343015d484246d9", + "evidence_path": "evidence/20260721T151308Z-2878ce93/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3ac0c0aa96e6ce87bfe46229ceaf33af40e073a1767e18dabaab5437dd1d3489", + "evidence_path": "evidence/20260721T151308Z-2878ce93/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2c826286a7c1179272dc46ac0868f386685a7a320f0855d212130a371ea08448", + "evidence_path": "evidence/20260721T151308Z-2878ce93/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1640ccd253432d58a32c4af20e8cba729e653841e928a7432ceff1820fbae412", + "evidence_path": "evidence/20260721T151308Z-2878ce93/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ebb86fc98bf545cf2767615b5fd19ec7188a48bc7e5ecb5f673febe5f7612d54", + "evidence_path": "evidence/20260721T151308Z-2878ce93/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefgijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:4d96267e2ba0f728637376da91e56ae223a4096439c431557cd83be8afb67a2d", + "evidence_path": "evidence/20260721T151308Z-2878ce93/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:32480ef381075a406c5bae874cf719eb42e68d1d772cb5ac91aa946fef22717e", + "evidence_path": "evidence/20260721T151308Z-2878ce93/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eba7603e7aebad941159169d10b8838707db9168a7507a08a975ede236b3bbe4", + "evidence_path": "evidence/20260721T151308Z-2878ce93/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:266edc928e00f626fdbeafa63ed24468ab880d14c46732fa3a8482955fc06beb", + "evidence_path": "evidence/20260721T151308Z-2878ce93/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e7448947d7c0a746d1f56690c0bd98570ab87a9e0dee33ffc750bb09c618733c", + "evidence_path": "evidence/20260721T151308Z-2878ce93/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f536d565eacbf68f702b6ee2e0cf7f858ec0058a01f9942efba7da9f98596108", + "evidence_path": "evidence/20260721T151308Z-2878ce93/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:023c117b26611a27fade299b5b93b2b5034c966d6ef6f24e26fb11a601644ca3", + "evidence_path": "evidence/20260721T151308Z-2878ce93/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:93ac9d52ff04d466f1855b67f96dbd5ce92f5914e395bd99adab094c5dc5a0c0", + "evidence_path": "evidence/20260721T151308Z-2878ce93/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:937a04af4bf5ccbb96ee11d1a4cb7188dbce6297476462124364577f5b8f8af3", + "evidence_path": "evidence/20260721T151308Z-2878ce93/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:56f36f3fc76d3d98985386f9a74611a10f26099b2372724d8b4b7215149f33d3", + "evidence_path": "evidence/20260721T151308Z-2878ce93/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f749a31bcd60a613dae9ff59c8982e476bd47bbfdd09ad0959e150a490f83fb6", + "evidence_path": "evidence/20260721T151308Z-2878ce93/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6bf05ddff73a47517f15df1679b460418faa67aeafeef65a45d7f007b8b8ba55", + "evidence_path": "evidence/20260721T151308Z-2878ce93/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:64680996e91eeeba50b220465fa7c3ddb33c0932cca76e66206b82c79ac3e8ef", + "evidence_path": "evidence/20260721T151308Z-2878ce93/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2c3f0b4ec101a01271fff07639a2e3551446b2da7e20c5a4acb12768f4dd0699", + "evidence_path": "evidence/20260721T151308Z-2878ce93/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:346ddd700964ccc58961238fbb0104be4262b5ad3c815d6771cc7a6d827b3763", + "evidence_path": "evidence/20260721T151308Z-2878ce93/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "unexpected argument 'service'", + "evidence_hash": "sha256:0c3cc6724b2e40321350e2ec201581ceeb5108308ebad7b68b3b572faeb69c94", + "evidence_path": "evidence/20260721T151308Z-2878ce93/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:46f122f0a4202648b1eea3df602ea2053c88f9b1e24f80c643df311d9a315bbc", + "evidence_path": "evidence/20260721T151308Z-2878ce93/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:620769625668f6223296b3421da88c405ff08afa869b68ec03836be921261b9d", + "evidence_path": "evidence/20260721T151308Z-2878ce93/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:97f84c55acb411c808aef03c60300607ba5358f4afdaf9798d24080538a07e07", + "evidence_path": "evidence/20260721T151308Z-2878ce93/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:49d6621936a5aaad2409575d5ce401b6fb885c44b7d2bd4da2e8aff4687c4986", + "evidence_path": "evidence/20260721T151308Z-2878ce93/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:69b4068bb2134aed1529ae67d14f5fd5e49477b3719dcb4c97f30b779d7cba40", + "evidence_path": "evidence/20260721T151308Z-2878ce93/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1d70a9b17508efcfeed89157c4d70920b84629524eb56479239d35957eaaf702", + "evidence_path": "evidence/20260721T151308Z-2878ce93/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cd1f1f34098c55316a09eceb43695702b2245a8ee60d9f3d2ea7a3f8f5d2fb8c", + "evidence_path": "evidence/20260721T151308Z-2878ce93/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:02be8bbaa2f8c1739f8843d840a068738eac4d571e81378b830ea0dfe4fb97c3", + "evidence_path": "evidence/20260721T151308Z-2878ce93/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4595ec1007b63053dd058c3b53f003d53c72e66b7636b0f37bbefee8da1d1db6", + "evidence_path": "evidence/20260721T151308Z-2878ce93/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c04d6dff679051d9fae7af06b3af2dd302c86f8ee9c2b911ce84159cd81ef409", + "evidence_path": "evidence/20260721T151308Z-2878ce93/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7ec57c5086274703a3e907715a96e762d63d57117ed02c88c7af4baf832d2b9e", + "evidence_path": "evidence/20260721T151308Z-2878ce93/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d508f9e2497406140d9a51ca28abda8a567d96563087494345c8650596b65bc6", + "evidence_path": "evidence/20260721T151308Z-2878ce93/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1672538545aae81a3ffcdb120d722fc16c92ba8741927adcc574bd104d0a508d", + "evidence_path": "evidence/20260721T151308Z-2878ce93/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9b7942c044c009de9a680366495c31a25e6da1ffaa7021890576fb3c7ae50325", + "evidence_path": "evidence/20260721T151308Z-2878ce93/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8c66e1091f27c6194ecb18fdc67716dcc42d6cd2890f86951eef4928e152cbdc", + "evidence_path": "evidence/20260721T151308Z-2878ce93/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:33ba2d3a3396028195b5890b5e23ed86b92e764255f499904a02f668c43f4556", + "evidence_path": "evidence/20260721T151308Z-2878ce93/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ae5112e2293dc93221feb851e7766d40193c647de8340372eceb5e0c476330a2", + "evidence_path": "evidence/20260721T151308Z-2878ce93/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:2985ddb151c99683baa9000b43a892808090a7ae5deecdf7d893ab7aed90e402", + "evidence_path": "evidence/20260721T151308Z-2878ce93/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4af01b3cbfc02203a01279cb81c2bc104115010134c8c760801c2009dda07dd6", + "evidence_path": "evidence/20260721T151308Z-2878ce93/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected argument 'region'", + "evidence_hash": "sha256:25adbf8aefd0012e35183dc4a8cbb38848af47b050afab904c94de4094d2a6ef", + "evidence_path": "evidence/20260721T151308Z-2878ce93/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1129887e7556a7911a110f4141ecaf0f206c069636d8c66dd999db19451d4acf", + "evidence_path": "evidence/20260721T151308Z-2878ce93/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:76257f6c393d1d047ccfaa5aa43fb9a0aaa2174da1181bec1f86952624c5a9f3", + "evidence_path": "evidence/20260721T151308Z-2878ce93/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4fdd6c1d9f781f5609f42ff35aed36a18a42ca722e9ebef510b26c202f56875f", + "evidence_path": "evidence/20260721T151308Z-2878ce93/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 39, + "failed": 11, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed4.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed4.json new file mode 100644 index 0000000..54c601f --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed4.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151322Z-a56ccb2d", + "timestamp": "2026-07-21T15:13:22Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 4, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:fc584adfcc3a759b055655d95d0c8b9c29dd8f088f1bea8769612f834aff374e", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:581edcc86b3da968e7e9f565276b051d9a528596e8a80196e339a860ee3bf9b7", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:992ed2397b668157488662d8c6a977b4e1d0827b2245366ed5695557ad2e9900", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:5a4c73a0906a5a222325400e008bc8cd7863bf0c27b8ede9e19a405717665e08", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:7f01f6ca60f2e850ac050bfdf72abe1ffa513d4a1b68a9632dcb18109f3b587f", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:67b8064a0ba76d4b182000e284251901dc6b19d1072effc30fef614f137f4229", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:b65420d4b02d87ed5e3e133cda32db51da0958e5e0cf1b236d05cb63691fb21c", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7a5941d82c35d61d847ff6858221c4150e047634943a4413aad0bbeeacbd8888", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5af857be5e8444271982e2de07e66c43ebaac4a03cc99e5a2211d02a2686ab04", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a686bd3ad5ecc68b8539602dc88daaf4f28c31c079c21c34cd1e22e8d243cb35", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'set_temperature_unit'", + "evidence_hash": "sha256:67328d55d37058fcac75ac547f73724a380999818eb1979c02bb7f1922d5aacc", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:d0442597b8b1d309063a16c3f76274db77b8696da73223372b6c72a2a82998e3", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cd73245c13005010122c454b405400ce84b0820d8f8304bc2f4791102817b808", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:51bd377e054124cb062adf4163d7aa7bfb36e69caa423b9aa19f7464c47b61b6", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:92d4bee48d40f59c75c91b92cdcd69aeba09a7666f2cb3e0d07d42651382e3cf", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4b0b68a490cce4e305af7e7ff0c640b52f99adc6eb45a7b6ee68024efba60b4a", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4de85786d9a6914b0ed6052582d4edd8cbc0f55a40707dabe22eabeb40c3d9a1", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a4d1e8e33c611086ee1829e1cd8b319c3ce4b7dbfd556f2319dc4a79a2e0fbdb", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8f306eea3a24729916781f52e9276c6bf9c6177bdc5df30b271c17abc3c91ab6", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:04e26b7a9efb958854b4e76c2889b4e563723efddec7e68eb79dc33c54fcc613", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:94a1f4a2fc62edda4e20dc2617a818b1a73151300a7ff851078ccd49bab288fb", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fd3d9b49b1044c60ec26a058572efac2cc0e9a80e159615154f25c1643549345", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a56b151f9ffb29633f23ab6647c88b063ed9a4be1c9a7f5576a3144791493da3", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:57668553f0013b2a199831988e236169cae19c07beceebb0e4c941c668ad353f", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b243bdf7d1c0f0256f6db55f60c2c72db6af6d1729289396b0771f3eeb4253d2", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6e60bb6023a8fd15a58ac4d1edfb6428808e39c290562470d83f4b230f152ec", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6ba8b22ed628d048ed4bd08e610138e9389aa9e5009e3ae15b5703d56af74ee", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ea8b9dbf0574393657c682747824601af8cbcd2588fc0e17913a472b1df03770", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:293f6fee5a8bc677b6c63ac00a69be9d5e7e22f82455c14b3127d36ead23b7f4", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:73772d65eedc97e0e3f2d03080b93ca3efc78cb5d82b095c7a0e196fc76f7fcc", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cf733ef23b1bd86fcb056bb128780720ca44cc21c2ef323121cd62bb93911dbb", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cdcdb333bf03cee77b326d9f6a3226265b526a09c69a59c25910f838d9665595", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "wrong value for 'name': expected road-test, got Road-Test", + "evidence_hash": "sha256:8499d8e852c3737c110583626c5be676113de0cb3518812999f64b2a44da68cf", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:005c8930d3e87f8060e88be4776371edde3012cb92fb47ec1c8559c57e83b210", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e626ac4b379c6f09d55ddb789fdfb6509684dc8700e088f1013a3af55b226e9", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ba5ad8f1d902ead1465935600bb8d2ed7b70bad95023a3a7b0c2a7dccb7cede6", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dfc1b56366586c45de2980b58adf1e805958eebd9e1a941c79af1efce0a35d4b", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "unexpected argument 'service'", + "evidence_hash": "sha256:b84e134289ca3004d2749415ec44c755ad450864fa1b5f19ae39da973913d6f7", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:758893629eedca78b44347057e97d0f481f044bcdd0b160dfabf61a5044353ff", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1ce8da5ca3d45c7da7e3275836e777c8f6107a193283c020e5a317209ff1d5ee", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ba04b6a3bd52cada16461c0dbd7b3fd753d9094e7413a80b8de94b2fe2dfdc9d", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a8af4d3fd104420cc82565772543a81cf55fc1c8af685914f5c46438d0591464", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bf791c69914e3d6c0df7381b9879e75f9b048a4b475a7da5a2ddbc074d24a27c", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6dd0ff9423945feeba944064b7671fbb5d0d037e946104286e4019087e77e183", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e87e3c699d068357177c14e58045928b49a218e6e33fa7d0386a00df9a1cf9b0", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:455f10885df44c44725a31caec66040b62bb5c7332a0f4244b72f732dc21bff7", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected argument 'type'", + "evidence_hash": "sha256:471e94cfd2ac6bbbab1f2b157504f6f4e64fe3b0518916311a3fe5a873b3ef39", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dba08bf8c2627d27116f0b42cbc67a0496930020f80169dd74f1a3b36ddc6fa6", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ac3a269dbacfa35a0b4197c6a6b08ffb6dfbe2a2f813d3189c4ac75282140491", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:745111149a0d9a27f47da19b54a6256801bcc22f05f6a8f6dd1337ec604c950b", + "evidence_path": "evidence/20260721T151322Z-a56ccb2d/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 37, + "failed": 13, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed5.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed5.json new file mode 100644 index 0000000..1a3a545 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q4_k_m-seed5.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151335Z-761d5af5", + "timestamp": "2026-07-21T15:13:35Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 5, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:94a0e12864fe84b394cf56126ff2acfcaa760b1cea6297ea4674206dbc85a8ac", + "evidence_path": "evidence/20260721T151335Z-761d5af5/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:679f06fe3178a3fd00c6a65356d76b7b6be65d649b9b0e196f42f5bce3df5d34", + "evidence_path": "evidence/20260721T151335Z-761d5af5/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:f6df606786e561609c2363d5a426ef85d76f98e17c281dc5fcd1b9ccf9fa99d0", + "evidence_path": "evidence/20260721T151335Z-761d5af5/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:cd9f2fdb6a647a56bc26ec96362270c6d1991b3e6e0b586e56f1ec0a55adc585", + "evidence_path": "evidence/20260721T151335Z-761d5af5/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:231536be677cb8c7c98196d6b80f5efc0a0a40d4ec2a379da6615f778d2db988", + "evidence_path": "evidence/20260721T151335Z-761d5af5/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:54ef0993d5952af83bd7d9355bd7f82a9a445c2d7be162b409ecc6aef3a4185e", + "evidence_path": "evidence/20260721T151335Z-761d5af5/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'archive_account'", + "evidence_hash": "sha256:52bc1e2c5d436778368b774315e0cdb8f0eec1926e9ded6ce504c3a2cedf3ef6", + "evidence_path": "evidence/20260721T151335Z-761d5af5/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cd562bb57fab8dc95eaba74c00a9af73cefb35441c19ec4be753df6965827b43", + "evidence_path": "evidence/20260721T151335Z-761d5af5/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c51f8af0fa999505fb2bbc4796b274a4af98bb5e04a798a7abc49ee85a27a65c", + "evidence_path": "evidence/20260721T151335Z-761d5af5/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f81fc9c693fb4095b55665adefd85a135a299b48e6513861b7423f3a6b847752", + "evidence_path": "evidence/20260721T151335Z-761d5af5/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2e63a9b230fa90751d7eaf9b1c1d700a963db7d9d1be4af42b8d60f1e0de2544", + "evidence_path": "evidence/20260721T151335Z-761d5af5/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefgijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:6c114d598ffa5d20ec573c8b8b2ee9ba634498b71e7db334bb7d2b419cbda08c", + "evidence_path": "evidence/20260721T151335Z-761d5af5/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3b12aa99c6afd9762d586b49ff03d2e253164a68062fe4dd335773485de87b0f", + "evidence_path": "evidence/20260721T151335Z-761d5af5/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fa936f99aa5fb4c8ada86b9f458a7abd5898358d733364c188be488cc2ec1616", + "evidence_path": "evidence/20260721T151335Z-761d5af5/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:da770f290fbb4f777948f403ea8a44fb2c31b938a52e359a2feece9c769f6b96", + "evidence_path": "evidence/20260721T151335Z-761d5af5/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:197f3a5d8477ae36f988916fa6291984f0999d0482091a9c39b7bff4eeda0fcc", + "evidence_path": "evidence/20260721T151335Z-761d5af5/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cfc74d4f23eb6427c0504fd5e18ad65384c7a327f0f7ae2e2e71b759c646196e", + "evidence_path": "evidence/20260721T151335Z-761d5af5/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:79f910e8172d1676f6921b6543a3a77c882884fb27563c22d33a244169f3d235", + "evidence_path": "evidence/20260721T151335Z-761d5af5/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f9939af2a451bf8ef49f6ce2a2ed74b033aa93e7f14bcd508122c03bae3b5961", + "evidence_path": "evidence/20260721T151335Z-761d5af5/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7c590f51bcc9236c41a7afa8a1b7a9ce276edfacce7e67d485d468fec7bf7964", + "evidence_path": "evidence/20260721T151335Z-761d5af5/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a55586443cee51cc249f0990dd4da6e1351bbad2de5e01e87bcdc11c0444dc79", + "evidence_path": "evidence/20260721T151335Z-761d5af5/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f5028480e5b1a3eb795a37bf9f0eaaffe38a265a4195502d7957a1eb830c51c2", + "evidence_path": "evidence/20260721T151335Z-761d5af5/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:28229f4f33a44f5d0e05c61516d796c610a815aa57bd2a194188de6afa896164", + "evidence_path": "evidence/20260721T151335Z-761d5af5/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c72ca03d5b001acfbe39ae6c6fdd86c6f76dbc6ba5e4a186921d2d3282b1cb1b", + "evidence_path": "evidence/20260721T151335Z-761d5af5/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8f1a6131bd316d86087a4042d600084e5c4cdb0a99bf2b256effc989f120263e", + "evidence_path": "evidence/20260721T151335Z-761d5af5/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8be9b23e36b80fafe0d2182eaf1fa0fe583e0ac4cc5699c48b317d9145c42e44", + "evidence_path": "evidence/20260721T151335Z-761d5af5/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "unexpected argument 'service_type'", + "evidence_hash": "sha256:3162a76e9f8de29b4f63d5b768458fd54136fd36ccde64ccb31a90d6af234cde", + "evidence_path": "evidence/20260721T151335Z-761d5af5/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fb152cebaf8e07a15bfd4fcf046edc2e045efbfbce209dcc1682b721eec3198d", + "evidence_path": "evidence/20260721T151335Z-761d5af5/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:66a25368816fdc951f0f1b0859ee1236d16ca90da0a0e7cace087d7412e7410f", + "evidence_path": "evidence/20260721T151335Z-761d5af5/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a845d3e9c3d2f44d61e6c156736759535acab53756dfad01d2f19f7259dc3dad", + "evidence_path": "evidence/20260721T151335Z-761d5af5/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:553064cb2795b6eed7b283056d8a1d7a128a7f0a35df0ffd6660c5e99c947ce2", + "evidence_path": "evidence/20260721T151335Z-761d5af5/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:676f1e464af44c57a406c5a445f0d000f521ded479fa822938f4105ff10bd489", + "evidence_path": "evidence/20260721T151335Z-761d5af5/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bdfd8a71d3e2fdedc9b1f57b8678ea259e187337c014ac0e4db398158fec02dd", + "evidence_path": "evidence/20260721T151335Z-761d5af5/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:102c77d22d396dd8d3c79827cefa6473f34d4fb06e78795af04609b0180b14d8", + "evidence_path": "evidence/20260721T151335Z-761d5af5/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:43896fa1d673f594f5d07034833c977f494006281523ab75feb7d4af7ddbbe80", + "evidence_path": "evidence/20260721T151335Z-761d5af5/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:db25b8cbfe2f8cb5ef708a91d5f86e865ef8eb106d5c8c524cdc4b2113475447", + "evidence_path": "evidence/20260721T151335Z-761d5af5/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f55d01d6a14daaf7f15d000d9518f9d8db655ce27b9a5f4862aa12f09e2605cb", + "evidence_path": "evidence/20260721T151335Z-761d5af5/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:54cb1d88a42d7bd1448a97b2bc3a38e4d4e80123ffa92429256bd9ffad318de2", + "evidence_path": "evidence/20260721T151335Z-761d5af5/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f53180c7e43db5d93ae338b98500db3ca00d949713835bf0607828685b686daa", + "evidence_path": "evidence/20260721T151335Z-761d5af5/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "wrong value for 'name': expected origin-shift, got origin_shift", + "evidence_hash": "sha256:4bca4e139cc3501a4bbbbdf35e2b622b640e0e4011afeda75c8ff465a36af2d2", + "evidence_path": "evidence/20260721T151335Z-761d5af5/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:50d72c1a371c073f7886f541a594cd050f4a89c4ac1770c6bf5b81e5c4b1ed71", + "evidence_path": "evidence/20260721T151335Z-761d5af5/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0fc1fcd536918ed0246b3c1d77638e406992102782485f25c1790d1ee3a005a3", + "evidence_path": "evidence/20260721T151335Z-761d5af5/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:79ce1261f26835595194df1014c0fbba1b5184e3e88f66665c479ad6ff9b4982", + "evidence_path": "evidence/20260721T151335Z-761d5af5/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:999ae732d5b2e227166a6ed4be8d264ed532f90c6a22b9b0cbb79fa0792cdedc", + "evidence_path": "evidence/20260721T151335Z-761d5af5/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:7fdedc1d69aac4c8f8c5e43c45e697574b66467c93afe31b62a31acd86e41e2f", + "evidence_path": "evidence/20260721T151335Z-761d5af5/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:50895487ef6ba50294a763ae33e3660a04229b19f86a9dc0a67ffea05d166ac8", + "evidence_path": "evidence/20260721T151335Z-761d5af5/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected argument 'region_name'", + "evidence_hash": "sha256:da61e2db4fe76c9b12b75d1da51c97cf47a91a4b6cfdf2290e4ac0c738342a63", + "evidence_path": "evidence/20260721T151335Z-761d5af5/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fef132d0170c81c2bf451e9e4abb763508e5bb8e575235fee884b7a42ff3d531", + "evidence_path": "evidence/20260721T151335Z-761d5af5/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e0b5c5e1b2fcdd12747e6b57bf4a0c330d29c81f7c9634b0abe7244d2642475c", + "evidence_path": "evidence/20260721T151335Z-761d5af5/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:beac260913bbdec12b2309309d2d44e754ab26ca30f6ae822598b324ba349f12", + "evidence_path": "evidence/20260721T151335Z-761d5af5/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 38, + "failed": 12, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed1.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed1.json new file mode 100644 index 0000000..b84d1b2 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed1.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151052Z-29339477", + "timestamp": "2026-07-21T15:10:52Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 1, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:88da22a758f080634612c0ff00ea3897f4cf657ef2c2e0904987d3dde2577876", + "evidence_path": "evidence/20260721T151052Z-29339477/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:97007c8f3765a7e42e3fedc97f584b16e823416b6b0780ddde483fa156259331", + "evidence_path": "evidence/20260721T151052Z-29339477/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:0353b7ad034372d4bbbcc416da7b74a0000858a3ba8ab5f0841650a5cff04559", + "evidence_path": "evidence/20260721T151052Z-29339477/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:7760c503e1a1ba9d16d7c1333405bd317b5e81e11a4a4a6c780f56843937ad30", + "evidence_path": "evidence/20260721T151052Z-29339477/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:25994fc3fe77bd3444ada9253fc036e74e44480da6a6990cb6cb19bcaca0b13b", + "evidence_path": "evidence/20260721T151052Z-29339477/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:916a75370e369f23b196cc619cf9b89c393fd32389ac7501020517599ea0f4f3", + "evidence_path": "evidence/20260721T151052Z-29339477/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'find_account'", + "evidence_hash": "sha256:3309cb916b72e7577da90b560b8df98561838a1a7740b3591240f8e4610717fc", + "evidence_path": "evidence/20260721T151052Z-29339477/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2067cdf09abf03b7d8de43ab6350d2207f5a8d2cc2bff25795ec7ba36d760161", + "evidence_path": "evidence/20260721T151052Z-29339477/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e232aff54afb7af088b7a30b7b8df1f05bf5148bf0c5ca74d8687a58ce2f1b25", + "evidence_path": "evidence/20260721T151052Z-29339477/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cb385f4c15173886b9add02058c9148c1c71d9416b4970ce2c4291dad2408d81", + "evidence_path": "evidence/20260721T151052Z-29339477/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c0b99862761dd6f1be7376aa470b72694ef427b2fe18ec4620c6910ef32941e9", + "evidence_path": "evidence/20260721T151052Z-29339477/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefg-hijklmnop-qrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789", + "evidence_hash": "sha256:1a4abc54027a6a20a6ae9c78db8301776969cdceba3cda0fc78f064274365918", + "evidence_path": "evidence/20260721T151052Z-29339477/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5162582e2819109fa73529f6a19bb1dce56a01e09bd99ee0a0227f24a1049256", + "evidence_path": "evidence/20260721T151052Z-29339477/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e1f5ae3b7c3878b448a244bf17aa5cebd2405756e26dac4a5715f99e2daf86a0", + "evidence_path": "evidence/20260721T151052Z-29339477/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "missing expected tool call: 'get_time'", + "evidence_hash": "sha256:5cc7f6e3598171bc213b6acf3eee9655d39efc9811dc5229dee7e4db104c01ab", + "evidence_path": "evidence/20260721T151052Z-29339477/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ad595d86b70f3afdcd29bbc9707c64af6de2b92a7f8473041aac0e1aaf4b7637", + "evidence_path": "evidence/20260721T151052Z-29339477/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:692f23f44773a174785c5649d10718683a3869ee88e15bac45a032d8367bfdb6", + "evidence_path": "evidence/20260721T151052Z-29339477/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dfa0366d7f6836afd21f214bb6fb488b4e11385adf80d17ccc2316efe02e004a", + "evidence_path": "evidence/20260721T151052Z-29339477/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:61bf1cb04f95bc53092d9f54bb2c882e8d81cdd5e18762d25afc103177c70acd", + "evidence_path": "evidence/20260721T151052Z-29339477/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7267d65470f52909b9b82619f82b008fb07bb1626d4001aaf286c86b916f93d5", + "evidence_path": "evidence/20260721T151052Z-29339477/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce0937f98fbc42780bfb4796c2b1fa7b15e21752322bc35b5e20e3e19a073555", + "evidence_path": "evidence/20260721T151052Z-29339477/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d39168451e454fe4ebcf49e7d6121101a8e29956819747ee46ca908f8d5c6359", + "evidence_path": "evidence/20260721T151052Z-29339477/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:62245962bf9d0e08bc25582ac94182a0295bc3979638d36f248aae457b96ac4b", + "evidence_path": "evidence/20260721T151052Z-29339477/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7172635ebccdc050dc6f91d06bcce6d7e1a2800d353973e53e1eab170d3481f9", + "evidence_path": "evidence/20260721T151052Z-29339477/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2f6aa845bbd1e05cfd00dee677017b8b642ddd0035a797d3b65d5aa5865eabdd", + "evidence_path": "evidence/20260721T151052Z-29339477/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95564de31846a90b673e470530753269bb1573e3d2dc1ec04727bc5681dcba53", + "evidence_path": "evidence/20260721T151052Z-29339477/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e805495513e13954ae2392af4e4335462cfdd16cdf5fb1c327922a123d42ffae", + "evidence_path": "evidence/20260721T151052Z-29339477/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8f89829aa1066378761e7f08472ade1b3da77b01e1a9afc2f9baa673aae2633e", + "evidence_path": "evidence/20260721T151052Z-29339477/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ca34e91ffa78ead39cb6e0d6e67d9f1a333d2071dd5812151550b86eb147d03f", + "evidence_path": "evidence/20260721T151052Z-29339477/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fcc2d8ec57c8adc6ce777dab8fe9a9f8588be90eb0c426f0a2831e5a475f41c5", + "evidence_path": "evidence/20260721T151052Z-29339477/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:48213103d701f7d5dfec1fd544ddf4f920b9c6c4532f586586a9ed278774fcb1", + "evidence_path": "evidence/20260721T151052Z-29339477/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d6ee375901afa286bf96bc59dbe4b97d651e2ba2a196bc02a105dec34a58471d", + "evidence_path": "evidence/20260721T151052Z-29339477/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "wrong value for 'name': expected road-test, got Road-Test Playlist", + "evidence_hash": "sha256:a16555928c51de0413d7a10fa0eeca789df596f1212be8011c336f3dd35a7d5c", + "evidence_path": "evidence/20260721T151052Z-29339477/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2279a945e604d3d67d1a09d8f6d533b66577e56187e961cccbc6fc47b9000caf", + "evidence_path": "evidence/20260721T151052Z-29339477/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4b2d231fe0b8e5e67d09d49c030e406ccd740e4160910048afdd14327badd58d", + "evidence_path": "evidence/20260721T151052Z-29339477/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5f549e6db99f244260d00854be8a8efcd76c9656922d92bffba0c5604e4acb38", + "evidence_path": "evidence/20260721T151052Z-29339477/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e0b4985d259ede776a71fc8e53d10e627efca8da83416607a650f904a130e5c1", + "evidence_path": "evidence/20260721T151052Z-29339477/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "unexpected argument 'service_name'", + "evidence_hash": "sha256:76381abfdfea7c23f26f7aaff9688793061aeedd5fbcf5c92eecf0e99fe00535", + "evidence_path": "evidence/20260721T151052Z-29339477/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f7f9329a05e623fcad539b273338c76e0adae18639c96f3612a3947edbbd10d7", + "evidence_path": "evidence/20260721T151052Z-29339477/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fc1c36612b719bba0fdfd47386cac6e722018d0a204ada6223f216dfa216794d", + "evidence_path": "evidence/20260721T151052Z-29339477/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4ed8bfd76486c7a4d8e61cf7e3745578c4badbc80010279bbe05b55565974cb7", + "evidence_path": "evidence/20260721T151052Z-29339477/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1c7e04c307a4b37337f5adb627ec6c772bb77f9c127b46124f8e2a7510b35c9e", + "evidence_path": "evidence/20260721T151052Z-29339477/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d07301cfcfdb2b0e73160d41bad437d16f811a8a0c41f8047019947f85ac4c6e", + "evidence_path": "evidence/20260721T151052Z-29339477/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d56cce40a6e74016e310826f5bebf6b0b1a4c11227fb083b6433c1b2bb7b5a72", + "evidence_path": "evidence/20260721T151052Z-29339477/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:577f7f6cb10a61cb35fbfe80abdfa5f958fad842b96f4500a0cd3c1d980c2899", + "evidence_path": "evidence/20260721T151052Z-29339477/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:84a3822987230a12d0f87f8719faed2144a4852f59cdb502ad0bf42354613f35", + "evidence_path": "evidence/20260721T151052Z-29339477/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:397330164d94e49b637750878158968632457700e7d69ae3409f59a5d02e4a1f", + "evidence_path": "evidence/20260721T151052Z-29339477/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d7fb7be4495c988e6cc88935f29cc5fc5cf5af24311e63bcafc11357a0f69ed6", + "evidence_path": "evidence/20260721T151052Z-29339477/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:12a03e858c409501f4200f6c00e3b9a66737864070e7962dee902d2ca09b8442", + "evidence_path": "evidence/20260721T151052Z-29339477/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ca00c054182577ea6ff622f232b10457f1ea03c16ab3e958b9459ebdd4194642", + "evidence_path": "evidence/20260721T151052Z-29339477/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 39, + "failed": 11, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed2.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed2.json new file mode 100644 index 0000000..0fd59fb --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed2.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151110Z-18fd0e6f", + "timestamp": "2026-07-21T15:11:10Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 2, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:a123cdf83ecf4a065567bfb079ec1313bbc2f5f790c9d532af89580a452ee25a", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'book_flight'", + "evidence_hash": "sha256:da67644ddcf377ec3fd86e71d715b21eeb4779ddf0bac0d14eaf37f5c2e09495", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:790251fd41240c2c0672e8d31f0bb3eeb8616116928381ad662fd0fd8ab3cc02", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:afd8fe702fa73af6d82ff6400b650501022f1e0068f98882c9b60370508dd520", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:d486cf3868d17776bb479bd94025169b670e51fddca0fd86d1d5d3b843be58d1", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:60c3e532debcd7a9e39127b5fa2eb720e35d25b5cc46ec0f2bbfb7c123fc36c0", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'find_account'", + "evidence_hash": "sha256:dda4a83da39aed193a978ef9d06f3fcc0b2fb6b20203917ee4c784896276dbc1", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:697a37fdf7a9d734c094fcc2f3cd61b72c1edac51bdac87b3c47f2544931aa9b", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dcedb3b155a45004283e0ca19a8bcab6f9042b2d0772bd43a76f2dd710a0ed3c", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:21dc054a50aa0835d956d048bcb698f1c1772036a99fa75b264ac7ae02e77201", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c218a07d3ec729310a05934bf89672fe39fd531707da704eb0599ccdda420755", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0614b5b8838252be7f69e2fe63dfb5d8e466442a831b80e4b91b99b9bef6d9c3", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:59897b1a39623d37650b615d8d571c754e81ff69ec08bf7026d6b9f2157feb4b", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4feb88a3fdeb426d1080d22ee334aa759bacfa50d5668889f6b0855ce2646fdc", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dd9260a6d2376af34e4d922f2c0f87c80ccc555840bd54bc7b70538527bf3687", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b7ce8861303512c21ffc2c01c640016f786f9fafad15c11891dc8d3c1620418b", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:28446ba0678726e4fbbe0a99316bc0afaa37e4231494e8237176e91a7475b869", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:87ca87a539aa40f31489cdd2a8ab95e8c2d4f5eff4d3267a29ef7d1aa8ee8483", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d48edff6fcaa04497dc8620d1c59ed1728d800db9aafd09aea2500e4c28e1e40", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:143c584def71a480448624229227f0e53f23f31d406b83cd171e5a9cb29b4dcc", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:70aeb3d85a0ba5c6713d517609e9a05a28bbe10ec9f35c0c8951eccd3ef1e30c", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1cd5eae7684074e45474eddadf744181405875fa00acb768fb0fd502c24c9b07", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:db5a644e157dd08f83853bfb1d51dcfc2a47bc6b123d58751270150a83890e3d", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb5bae94cb214700181750b3dc7c80be99ced707219e6d981fe8709ed28e5a18", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:11d88463063d583c747c243925047e3b5e83de5536a74c129e9a105a7fbf228f", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f8b78a8890cd106fe6d3a080794fe9ad87bc269915b082c6738dbbfeef14bc33", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c34175cb649604e054c0970657a7525181033a60b8168a7099e1a1c1d9825f1c", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5ad30fe383db5c83ad094c4e4a0da43f4f5a640fe917e9c8ae93c6098dba394e", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:196ef698ba4f798559b3980ce6258ce68cf3f0f298dff063f73a2efaa3499f2a", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:246fd471c817ef92a35be0f69aa20572cafcde59e9530e2f0052e0a07622c649", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:272e7647a9d7a891ad270b53dcf702b52e9a9a0fb6bf4ab29384334d52c7b22e", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6d349410963737f7f149625552379b8d7de2216474511f08c41f0b164cdbe576", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "wrong value for 'name': expected road-test, got Road Test", + "evidence_hash": "sha256:75c5f18fbccc7d83d338f91c89e82f7095165730e4f885fcdc93bde91ba3018a", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:610c07b33775e123bc92559241829f54331afeb0c964b960b8b60b24609b0ec2", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2dc0106e73d3a415b61d8e76aab329b229ac813b55e05e8714b873416be44029", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:640363ffe228860114f174e60dd44f6b0b22d97113dff5d7b6261ae842d383f5", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:240cacdc6402a15e3fb5af86499dfb035712e40a575b5f14584fe3c3a02c9de5", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "unexpected argument 'unit'", + "evidence_hash": "sha256:794ce58104d1e1e4b15ddbab4d3b9766faec8b45b4bb31836e666dd192d809e4", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:950426f83631ee7d604d775717d74763b39feb3b675e63dbfd68d5f2d101164b", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:17d599e353109c830c1f28953791a5ccf89c653104f3c315e4cac9131366ebe3", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2d7bd250ba74d16e86bcf3497570a53db8b9b00bf2ba7eed92acba12b0c3c88e", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7d91f5e5c99e0cb83818852dca98990d71f6fcbe00785f0d972112c36325175a", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f8e86350d600b8430ebe598973d89b5a3340633ae52eff52bcc4e45278e18087", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:42b8e1b9f7463a8586b5753f3760268714e1ca6ca4a9ba502884357284a1f540", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ba205b44fcc46ec6d337d08111d2b71440d4c54cf6893787ae43e57424aff871", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:31b3b3ebd940c182931de8b6efde654d12198c1ddcff26b5fe26e289e0dd0cf8", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5a5ea7acaf17be36ec792105bbea394e8cc6a0d406ad5925ce60597100e7a4b5", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c627c86a6911b457a37b0bf70c6748cbd11514b186adac3aeb5e4d3b796582e2", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bee7696cb9103cf92016a9daf3892b9e7c5cf98c482065264994a519add2a1d3", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f6fa9d3345d4a06fad5dbccbb177587c514c0952b2aa67b23d4a9c08e21eea32", + "evidence_path": "evidence/20260721T151110Z-18fd0e6f/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 41, + "failed": 9, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed3.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed3.json new file mode 100644 index 0000000..a5f6284 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed3.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151125Z-60d53bd3", + "timestamp": "2026-07-21T15:11:25Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 3, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:e436927c843f41a678fd61f8b848c5eb45b03f82e9f2faa82304367359d8cf33", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:8c3a46aed7b7ca63652580ed9fd0aafe0506539c6b82344893d29f119a27175a", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:6884ca3c39500b568aebfa59cd67efbcd8274f33a1956a83ed07febc0e787885", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:f765537392bcad7d590a30f8469e48f5c3e836a295c9844a2212410f2dadfb88", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:5a81b98c84c48aa2720cb3644fb9f538fb0008e0b94847b9526ce268d94a174c", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:7a41f984189db69bc294c0277a29ef30335378c8eacce12196628e0c37af0838", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'find_account'", + "evidence_hash": "sha256:22f3568c860e47f6bc8ec423a839cb497a3de18904ef140ca3faf49b3da070e7", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4ebc3588b2986cb4ba2350ceda86f6e7d583ba503fff116bd2487b5a5a701f36", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3154d2bfcd16fb31220c00af681ff78c28e7079008dd579ae6e1701c620ddfcf", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eae8c92fc5ed0f81d6b6a7119b9f928a0536612c670e346662e849ee0c2aaaba", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b00995b06693e6946678b3026a6f5a43d000f101586a6f8e75b67777c3d94024", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:7ffe81850082e4c56052475565a3de847f6505288c28a67acb68a85b0c4e0b99", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3dad8dd1aa7777905381c742892cffbfaf630d551520c4272b1ed298ee3b5dde", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:38b82041b718c06b8193910bf13db4bed2bb91c23f8dfb3382b59835bab19104", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:244ea370abe0308ac9a520588b16176ccb544bcc79e748c432d4fbda78ac3cc9", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:15d7067e3c2744155221e09cde8a0ce7bc138cc2b871ef4a9f4a2a81303b691e", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a8cd26dfaa76991e4cda765096ab0b351c2e56fe798aa0a6211d5cc82d8ab627", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5da077079584867ede54ba1595916b1572975b8072a1dc337cb56bf9bdd6355d", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e8fabd4ed35364ce13d67b517cca0fa5c48e62b172db376daed7bf45ec5d15ee", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5c48ac22fb1e9a82016d16e0c343012ef695b383bb0d445d9a6b43316cf7d873", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:91048b468fded927a4ceb3f9cf4bc8d10657bedfe9ea9ce771c8b5078b287206", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9c9e88d119de3fc28d91e2830e4c0f6abe050a9593c9ebe679ab08e612b6faf9", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:344a132bb42c0c1850b8a47f14503a4ad19ba73d89c5ee1976a6b3ddbfdf2d0f", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:773a4d34f83b993d007b666ca160b44f4fb731bb30bdf5dc3e31e595fe511f9b", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fa419a140421acae3fa15e2bb6afd50ca4d67c2cede4642b8e9fd7e1fa9d95c1", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:64319ac7515aceda9a9df928b9880ad1c17b0c2c3b469ab5c343c1074bde1586", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dec82f66dd2a2d151ba6499edb434de2f85bb367cd988d2d322774fab0580be3", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:228d6aa878c189f39f955f1b8a0b8f49ee264d18a1160e5c0aa6be0fb1e953c0", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ca1ae4071b62aee95e1e72b367df3b710c287a6ad10830eef024dba04b77318e", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aa670f595249fc7496a0151b85255eae0adc0df45ef49859ce2ec5200576544f", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cb4c133e02a93a5f2bc27337579a1d03c1c61ece32394172f361551a4d38a105", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e0a48059fd6a1326016d9ef1abed6c3a0ee687f8493aebd54b5b03644e98d0ed", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7242f1e2bb7d7e890869f405e8d3a7238fe7e49d62dc6816cb53907ab05e237a", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:32174e5fbc1b1e2a7ef3558a285ae728787eb1d461d8108f5e2c32358ece6985", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:df06b2eaf9209201616bb6bd5c7a5bc77beba7693e4afcc2db8fe4df7ab13651", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:262028c5db82e17894a4dca3ce8a8957c6ec9b948b27b62f3a0093b97902beef", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:43bf34ae4ff77843486d3d4609b98fb2f541882e6d8b0399b759f97872ab5bb9", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:00e521056401b344af5f07225ce924ca9186174a88b92d078ec4dbdc38a81ee0", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:da2f39e511a5fff445a6436f10bf1bbd3135fdad02cc49921d1789a58b6ad906", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:81e05c15f8918a83f691a6c086e024899e30d6cdf38334999df00f20f0b38801", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3b994298f067ba2140e9d1c046b3b813fd585747aca506ee121fb8243b0ff22f", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:da238001cf8d61f5664184a914aaf45c98f2f057e22f17ed6746ab69e3b5ee1d", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6e14a24d07f104f0f57516222c7efbe039d9b8b5583b58b8ec6a6c2f5b8e7257", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:32220bfdfc41eb200a35053a65f8a47dae6493f25e66ad24e9c723b0bf0df069", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c13544ffdf962b3e7171d0423d6cb7318b5ae1c4cff2d0c64d40c3edd03244fb", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8889147db036fd514a03c7bf25e0cfa6034ebb1841f8565caf5fc944bc2c5ce1", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bda6caaf9284bc2f597ea5cd20b9fb1007d1171ea90a49fce3757fd975dcc0c6", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:32bf0f89df3544e2d74dfd88b44761524c7d5573ff14805869f26e2e309664e2", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:254b31406a57d202e5f7e884ce0f28e04272994f74077afa09a74b5b4b1ea3c3", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1af63b2c89fc193763fa325b9f7949c887a1b8fb2917c710ad3a74d8aebf7f8c", + "evidence_path": "evidence/20260721T151125Z-60d53bd3/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 42, + "failed": 8, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed4.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed4.json new file mode 100644 index 0000000..5d24596 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed4.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151140Z-126bc3e7", + "timestamp": "2026-07-21T15:11:40Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 4, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:ae406958b4ce2a0293d706cdf13875ed601f5a8216c7d71b9e812fc3b818dda5", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:c8802ace39cbcde4f4f3a96a0cd90478ea87462b250a3ed5a35745403f269af8", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:259588ac993157940011b2225092fca842ffebc6b3811ec0c8e2e24bbf6393f2", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:500f4eadf9edea470a4641dca81dc71ff68c8a343471ccb438e426a864284244", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:e1e308716185d77df925bdad16ee8568f224e7b1d9c2cc0945ec18f99af838c4", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:a2396a23b535983cb06e15db23babca99a89343a96314e378c17b9f5e44e119c", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:dc42a516ab535394dd16022df5a94c0c515a4f7df3ced7496cc73b3cb299d646", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b8d24ddbf6a214b389f2b20727c1ab7f2345f373964bea7e97736870ec5d5009", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:94fa27e67aa2bfb4ca68f9d032ccd84a004f9e462d2b40051b04aea01ce3dc0b", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8d8f236fd48ca91354ccc70f4ff15dc9a72507e80eacfa594aa402dff4ba7023", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6890900ff75f08d69d156041a6721a0fdc3d58db747ee8fe4238f8385baeeb36", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefgijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:c6f94fc0f6550468dec7bdd0cf01ea06ccc7473787b94544eb74b4f8b4ec5d7b", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a5c5147f78079dd98db1711f5a87e5a27b9439658c4b55e782f7897708114426", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:51ca99a96b0839bc23edc3b66f0551ecdd636f49e3b40fe9f1ea0f0d3d7bc2a0", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6d0f36c2686855b43506c3c8d46f669c19b5516009298167aa28c4ab6fc2d80d", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:02364a6fb201bb67d642920be497c34a98f471cbcd151f542b8415a8cbea02de", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aae14edc4dc69e56fc0a6a44bf11a65f46af0d47b0e8731c73c54fe73c46d160", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:89ff12b344aa04ac4ac68ba3c580f2a14edc10a5174e4dc938b15080f88a9ba6", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:37b64720f5531dd9b8f288d16c041c05a2d131fb416b0f37b7a862d30bdd958d", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:19b1f43ceafccdd366738dec749ddbe68b1776d8699605f4f31c2a92e89a219a", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:84468a362795251a70905d2788f9654291c65c526325138b3740d38e590209d3", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fc88a02e66c4c2454997d5567e636ce98cb5f4727d4b8203443e9dc10e606677", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1ba8b394786fe438151d16601c1699028adebfd69dc6f32b8b73152022f1411e", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:965012349de5b14c1846f901f3654f1fbfdc5ff7d4354fb0ccb672bc14d0761a", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9dfb7a1bb1735e41b4c4876d37aebecfd3ff26c90ce0558a5c3ae1650571eaf5", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:58ea71a3c66950e9d6bfa33dcb66f4023a84006bf26cea42c10409f513561e43", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:79263ff5e3e5f7a40ece041bbbb8cf03de733e8a414281af58a8814d6a184e91", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b9b5b1f184d45c035e683397c119488041d1dca9f68c021ab94a7d9f0408cdb4", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b4371c58596bae299be19a3783bccd775d93a4a4ee94675a572226bdc5b2a5e4", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:825ae60a1bc575eea723b86f01cb4daff98d889beae5d7bbe1d7540de677d221", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:449959375e5d67c49b321a6a292ef6046173766b4b2c88136f62b883cddf97ed", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:20ff7c36395f7d247cc279b068a20a8452eca7e6242266ec837d4493922c8faf", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0ee67a4deaa02c2c7522e3f562e33370e514f56765ab07e044d6811604bd77ef", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dc267b5feb21dfcdf72edacda4ba23b0cdb87a61910c84e0b59ca71ef208bfaa", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:94547d64ebabc66b48e9dc9d185adedf0793d3c47d39aa22e3b667e4b76691e5", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7e1fafa9b8033df4868495a82ab84ae9b457c1e83662b63f565c2bb35713aebe", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d9256071bfd67e7772d560a57651337dbb46106da3ffbcb76a4675926c688fb9", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:18c7025fade71daf064325ba22da86f686092892c64ab16069493bc6e785018d", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:96e2f8b53f1c819e73cee71c972904959bc3afcdbccc128b271266ff6d4984d2", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:436ca1aa0c6c1fc735b4fb10b64596b47a7bbfe205ee014f6ae26ba53c06001a", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:36ab75adde3f1033770125791e6f86af1647dc9b1728581eb712fab36c9dabbd", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:90fe63af20f1281a95d07b07455c392a15ff86d8e7d11943ea0f294234091335", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:020f22d128022d10ba1352f99251324031efc9cd1514dbdee25d7f02229e7c66", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:654cbddf0d20ca4f96c1735d10285b07d03303b72b0b0859c725cf86938baa91", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "unexpected tool call emitted: 'get_temperature'", + "evidence_hash": "sha256:9896846d15d91c84c3859a2a41c96fa8186bceb689e43ff91224fa9f8866b698", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:55e69e9fc2080dba92ddaacca13bc4c7a0fe306c55e5fab567952318d20877e3", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7e23e7c810db6b7ca15d5504bd045b31e25efde20e650ef41bb4ab992a87e73f", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c2cfbf7d45ccb18de313d53db32229c76c3f8576219d952c0e698365c2e63127", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e38b980bd49d058c72f8acd3e3846754fa51aa617aebf45239bf6cbbd91ea6c3", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8d95e1ac1514b444409b7f3b581d63bd2bd76d667bc5186c302f2b648564f16d", + "evidence_path": "evidence/20260721T151140Z-126bc3e7/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 41, + "failed": 9, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed5.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed5.json new file mode 100644 index 0000000..3993027 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-1.5b-instruct-q8_0-seed5.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T151156Z-22e77c55", + "timestamp": "2026-07-21T15:11:56Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 5, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:d615fb81e2e8590c498c04acdccb1719e5c7445591012a5b54344697d7853127", + "evidence_path": "evidence/20260721T151156Z-22e77c55/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:41211a190c873b4b31b1a253abf5e0dae17c56fab1c256c23a22f83fe2fc0683", + "evidence_path": "evidence/20260721T151156Z-22e77c55/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:6ade7141e8296a4aa3ad67cab4ae2f26fde100833308aec286a8d31e43d8b1d1", + "evidence_path": "evidence/20260721T151156Z-22e77c55/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:d1fb566acdcbb5e0dcf98a9802f085866d8441ef65c0b2421288a1929bcf77fd", + "evidence_path": "evidence/20260721T151156Z-22e77c55/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'geocode', got 'get_route'", + "evidence_hash": "sha256:501ccf227d106b75df10f6b1060c59d3ff1df55be6410bfcd79873bfcf6ab09f", + "evidence_path": "evidence/20260721T151156Z-22e77c55/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:bcf5a79b316adeda5530e6a5a53cd23e70c9601a430bd20e09fce01f18208a34", + "evidence_path": "evidence/20260721T151156Z-22e77c55/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: wrong tool call: expected 'find_user', got 'find_account'", + "evidence_hash": "sha256:654e7f7ab2ac95f57ad3da7f6aa1fb49f3b1bb7715e5d5619e9712e287f2c7d4", + "evidence_path": "evidence/20260721T151156Z-22e77c55/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:329f11e797c42a3435ebc959f62748195b359c4e1873afa2c0f0d9243175c84c", + "evidence_path": "evidence/20260721T151156Z-22e77c55/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b72e597ee5f3a257660884e1e9c086e1495f4caa868d61e664a183a57d31b5a9", + "evidence_path": "evidence/20260721T151156Z-22e77c55/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7e3c2d017094203df3e850202be8c1360dd9a4e13e001467d2e8c7c8beeca490", + "evidence_path": "evidence/20260721T151156Z-22e77c55/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:76b7f82e4005f58df9a4b3e020c48568145ecb93dc79a8f131b3d78ce9c854fb", + "evidence_path": "evidence/20260721T151156Z-22e77c55/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789", + "evidence_hash": "sha256:4911a62fd055452e2eb61876610c3f5ee781809363b2ac0321ec685b4a6718ea", + "evidence_path": "evidence/20260721T151156Z-22e77c55/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cac7ed521bc90acda29b2581d261abb5e3f3612270c7bbf14202bc39e2ff6544", + "evidence_path": "evidence/20260721T151156Z-22e77c55/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a4fe8811d8352b800458490ec97e4d594b82b4a82bdb9f04600b381b90e5bf28", + "evidence_path": "evidence/20260721T151156Z-22e77c55/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0bee4d04cb4394d66506946fa29a40a1ba74a97b59f301ad646900edb35e616e", + "evidence_path": "evidence/20260721T151156Z-22e77c55/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f17dd920fd24414ab983757d009c14457bf158cf363902085cadd489a74dd319", + "evidence_path": "evidence/20260721T151156Z-22e77c55/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:151119c3c0ba7a5ad04546795133e8dfa9ccc26ac4bbc6510f6b779ecc881c04", + "evidence_path": "evidence/20260721T151156Z-22e77c55/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1475dcbcdcb274ccf38b9fd2c455e791b9cf511ad7faacb8072af17551ccd2fa", + "evidence_path": "evidence/20260721T151156Z-22e77c55/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:70a1ed7ff708c7b7843a2bd82b12370bba62bc11cf319f0406434da91d0c42f0", + "evidence_path": "evidence/20260721T151156Z-22e77c55/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5e6ac94918feea821c4dac090ca911e39f18c7e9a8eb5f942cc50d09918c2816", + "evidence_path": "evidence/20260721T151156Z-22e77c55/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d0652bd345f2ffdebdd47394537f99192c542cf7d81a8cf51709f653c2633a20", + "evidence_path": "evidence/20260721T151156Z-22e77c55/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a06fce0e5212381487bb87dc6f8d6d1d6860e9d27ca4bbe762ab8c67ba516c92", + "evidence_path": "evidence/20260721T151156Z-22e77c55/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a03537d94535756c20d8fad3395ba79da967c9a02b3dce713318c7935927ad15", + "evidence_path": "evidence/20260721T151156Z-22e77c55/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6f467ef4f06c4f704f1dedf2953135d0156cd96a486afb8b471e9c32d0b5a89e", + "evidence_path": "evidence/20260721T151156Z-22e77c55/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ee57d4472df0ccfaaef18c02cb346e324652ae2636b9e6c556f9ae11344ef346", + "evidence_path": "evidence/20260721T151156Z-22e77c55/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:742f8610176d0b88e31a32c5a3d4a383623fc61a5c91355d8f22dfdde72197c0", + "evidence_path": "evidence/20260721T151156Z-22e77c55/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:58d649eced5847344dba51c4ded5d2701751c1cbd198fdf3208176ffcf298a68", + "evidence_path": "evidence/20260721T151156Z-22e77c55/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:85b859cf8fd80fde05af0b3dee7fcb0d8e0e134f756d77dc9db6dbced5498670", + "evidence_path": "evidence/20260721T151156Z-22e77c55/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:88bf699de742a02628b967e331d580f9cebc2e1681efbbe30d2a5e72bb3b9b74", + "evidence_path": "evidence/20260721T151156Z-22e77c55/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:907e85bd96cbea69c8624f03eadc9323edf876f85af369355a8c51e62023a288", + "evidence_path": "evidence/20260721T151156Z-22e77c55/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:79022ccb6d57ed2922cd432a79c654c8066a6e1484458b19aac441f5b85a27d4", + "evidence_path": "evidence/20260721T151156Z-22e77c55/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:50bc74ca572b7c413d275e131a8e80bea12b73940b10f3b5ccf0943c7f40bde6", + "evidence_path": "evidence/20260721T151156Z-22e77c55/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6c81c01623e59e6083f5fc6c08067b0c3398d08b02a073c6bb6ca6c85679f94", + "evidence_path": "evidence/20260721T151156Z-22e77c55/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2ef547bc29dc5e945d5889d6196e21c7ea19dcdc6e1411c489ad2dd23c76fc91", + "evidence_path": "evidence/20260721T151156Z-22e77c55/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:94bd189806808a47c79fdd9fb986634106b1a5a9faaf8f8d0204509106553c4d", + "evidence_path": "evidence/20260721T151156Z-22e77c55/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3cf9b9ec6848b14be450fe17f4f54337815cd9d10dcd32c060dd042b90e946e6", + "evidence_path": "evidence/20260721T151156Z-22e77c55/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cdc044294653aeeb261fe2c685d302de23c3fbc02518fb0c0b34b0eff3049c30", + "evidence_path": "evidence/20260721T151156Z-22e77c55/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5341ca7caee37476242b7a6f9d3c74cf9934acd215911fa66a0db5087f445981", + "evidence_path": "evidence/20260721T151156Z-22e77c55/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ed10ca169f8210eb6558813d1f69bb86c598ca6f85a16f8a1bae5a4af174314a", + "evidence_path": "evidence/20260721T151156Z-22e77c55/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d704f1e07519766f55962a99534fb9165d0afc2f2ba9afc940eda94c93a30de9", + "evidence_path": "evidence/20260721T151156Z-22e77c55/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e6656b792d0beb33c6b8b2fc9f215fc80874aa35e6f5122232a717bda999694a", + "evidence_path": "evidence/20260721T151156Z-22e77c55/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95a50aae3a2ba9e93cd70c67392ff256a800a7fc982982b256dd56672d685ae4", + "evidence_path": "evidence/20260721T151156Z-22e77c55/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ca3ddcf6817ae308fb8fc6156592ad7a7b941c27de8a57df63d85e494e012334", + "evidence_path": "evidence/20260721T151156Z-22e77c55/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2154a2efef62f0249e4dae422f12e01c261e2a6f3c9e2cfd793950d927c04051", + "evidence_path": "evidence/20260721T151156Z-22e77c55/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5662712e677c9e17c23a1196d598b6052f31b99f7ad13cc7b343bb998edcdb67", + "evidence_path": "evidence/20260721T151156Z-22e77c55/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bcdff1d6ce6e31067a9ba66d5cf28f9b9e2898331c6ca6fb9af0c37b8c7501cc", + "evidence_path": "evidence/20260721T151156Z-22e77c55/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e7ab210cd14793268cee6a162c3a45d0bc32e1ce98a5917ea3d107c8ef84412", + "evidence_path": "evidence/20260721T151156Z-22e77c55/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0cddfd62d922605d4ceb6076468fa4b5237030bf328f17bd06af54b497a5b1fb", + "evidence_path": "evidence/20260721T151156Z-22e77c55/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e733675bb6a6ae13464a234bac91e25564c39f58f8fa508e6fe41deef8ea659f", + "evidence_path": "evidence/20260721T151156Z-22e77c55/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:680604b2362c29f12f06479077c060ea170a1aeb438d31450c0cb88aee41458f", + "evidence_path": "evidence/20260721T151156Z-22e77c55/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 42, + "failed": 8, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed1.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed1.json new file mode 100644 index 0000000..bc3b6b2 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed1.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T150702Z-3a29e548", + "timestamp": "2026-07-21T15:07:02Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 1, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ffc12fc743231aa1c55c20a0f5c23062c437b531b4a1917c5121a3b7d90da4be", + "evidence_path": "evidence/20260721T150702Z-3a29e548/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7fc15655e604cd0374ed8a37e3b39d0a05d1a3545ae68d1deb539736215a2582", + "evidence_path": "evidence/20260721T150702Z-3a29e548/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f1257079b7b4861763a7a54294419b53e35fcceaecf777f6b345b8ab54c2f4a8", + "evidence_path": "evidence/20260721T150702Z-3a29e548/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:a46354fba576e41e9d4098198689bda8cc712aa911c5ced126e29d2c828aa97d", + "evidence_path": "evidence/20260721T150702Z-3a29e548/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:057f9a9510355bedb27f3e530c9e87a35f21fc111f7e9b728eaf647e9892898c", + "evidence_path": "evidence/20260721T150702Z-3a29e548/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:3b89c81837a5f43d5fb9db3d2257445515ff11c40c22d2a4bc044eda47645a10", + "evidence_path": "evidence/20260721T150702Z-3a29e548/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4f677dc0ec1815e49205514cf9a9b9cc9ed29b668660e89fa56e08eb347caca4", + "evidence_path": "evidence/20260721T150702Z-3a29e548/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:393496f2dd232b68bf74e6893ca3867f5e2db2d4ba9cfb0c8b35367ebaf2c048", + "evidence_path": "evidence/20260721T150702Z-3a29e548/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:31f0a5a8c67d23d767865ac879b4f6c25bdaffb799d3149a9722ecb1caf0c591", + "evidence_path": "evidence/20260721T150702Z-3a29e548/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:320a75e5919e08a1e840d9cc64d4ad4139c5229c34d39b87c21a2646027a3808", + "evidence_path": "evidence/20260721T150702Z-3a29e548/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a2275e45066c1580f2de40c03dbbc1f9b8014ae54bd3141db85786544bbee15e", + "evidence_path": "evidence/20260721T150702Z-3a29e548/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:41ba7e2c9ee530b0429f0c93885b858a73d3a44221f36fbd8356f300d9be4b0a", + "evidence_path": "evidence/20260721T150702Z-3a29e548/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b9c2a1167f9c4732385a41d075437818f32e27cb6d357c3bb56c3c63916b71a0", + "evidence_path": "evidence/20260721T150702Z-3a29e548/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:18a5c28cb7e9c9a5110a5de0f3fbfc93f5f868ff29ae998e4435b66f392803db", + "evidence_path": "evidence/20260721T150702Z-3a29e548/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cd034bad1b388ff70d3df3ab16816dc8e6ed7b7acaafe6511a1ad36ed60a1fe7", + "evidence_path": "evidence/20260721T150702Z-3a29e548/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9c3c2e4bd12694a1476f84e499b48fbaf54ee6dc1df97f9e542ad86567cfcf54", + "evidence_path": "evidence/20260721T150702Z-3a29e548/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:912ef652e86e64e1dded67e1f9a6a3069b485c57d6e63a8e528af2c59ff10bf0", + "evidence_path": "evidence/20260721T150702Z-3a29e548/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6dc3133f11c564674704e1295309e68cb2e5b5096e08a49725067d7136061899", + "evidence_path": "evidence/20260721T150702Z-3a29e548/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:557d08d7f19060f57b3fb60f8e401422e8a89ab1f5a30d7adcd5aa685e2e5659", + "evidence_path": "evidence/20260721T150702Z-3a29e548/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e45224d78716e91d0de7ffa623a992af87919dfc97a19355b31ea81ab0acfa71", + "evidence_path": "evidence/20260721T150702Z-3a29e548/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:65d526f2694f9b4e57db1207d2548756bc05d4f121804517d8d8a8a3014d65c0", + "evidence_path": "evidence/20260721T150702Z-3a29e548/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d756cd0d34c98fb06582a1eadde20b81f2c6a52eda1ec587ed5b0e660f863107", + "evidence_path": "evidence/20260721T150702Z-3a29e548/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:90fb48f55353832a6aa0b9f57eda134f629f71b5882dc3cce16b7dc65baf2787", + "evidence_path": "evidence/20260721T150702Z-3a29e548/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b489bdc6f2114b8d37717701e089b4faf0c6a7f8266fb1a289f468d33213c98a", + "evidence_path": "evidence/20260721T150702Z-3a29e548/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fd5881eba93894b948ab7c929868f411634d9456f814c68fb4c15825e6358d7a", + "evidence_path": "evidence/20260721T150702Z-3a29e548/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f651ca472cb72fb8c41c4c4df546dcf5b503684559dd75936a8ee7e58c81081", + "evidence_path": "evidence/20260721T150702Z-3a29e548/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f2256ab84c7c39742d3a7b78f30ed5b56c665e1f662bd416a2a0d9a90334d814", + "evidence_path": "evidence/20260721T150702Z-3a29e548/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:24dfdc535c8f315f27f36db2669943fc9d61546728ea7f118f0eea2c0c6942ba", + "evidence_path": "evidence/20260721T150702Z-3a29e548/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b74e598d7c06dbbb211f53b9a2dc077206a27df9efec7fe6003fc01d55ed83a4", + "evidence_path": "evidence/20260721T150702Z-3a29e548/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8b0bcd08a80ce06a915c67b497d34ee3379d4657bbbd9cd358413fec06e22ce5", + "evidence_path": "evidence/20260721T150702Z-3a29e548/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1ff41753bcc8752dd6f33e35cafba6dc675898025341f252cfa2d09574b514de", + "evidence_path": "evidence/20260721T150702Z-3a29e548/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f8a9ab2568de8fc37898bb0e4477c5a7b4898b2d5210f2366d16da6bad35ea66", + "evidence_path": "evidence/20260721T150702Z-3a29e548/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:543e987ffa895c21e0a78f9e804bd5e757f80aef72ea0c176c3f5394c019db42", + "evidence_path": "evidence/20260721T150702Z-3a29e548/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1540787f2099272803e66ac7e4bbd3461b0b832a061ae533fed66daf509bf477", + "evidence_path": "evidence/20260721T150702Z-3a29e548/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:832bafff4e0f4bad591001fb0a7a0e51bbc3600ce3ab698904f6a5c990c4f8ed", + "evidence_path": "evidence/20260721T150702Z-3a29e548/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ee29ed6c38ac9ef77d7ffb28174aa95d273cefd2142e5d53c406e6a60f73d0a7", + "evidence_path": "evidence/20260721T150702Z-3a29e548/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:73afc3ba8b3a4d8770eb37b0c72ed1ccae5c6b6196d761020d2b3ac66367a94d", + "evidence_path": "evidence/20260721T150702Z-3a29e548/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7f9cb705af469674a67ddbcbad6b2f188cb50bf095ab70a2091ea67a9e67db03", + "evidence_path": "evidence/20260721T150702Z-3a29e548/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:68aac63037d678068faba0670c5e2e812b49f86ab37c07211a7661ab7e1e72d5", + "evidence_path": "evidence/20260721T150702Z-3a29e548/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b5ef3a0a0fae6681d860a199ae1d80acf76a1063605faa97aba3a857933e9681", + "evidence_path": "evidence/20260721T150702Z-3a29e548/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8780425af354aaa669e83efe660273d3e736596bbdb0a84e790eccba78d70f22", + "evidence_path": "evidence/20260721T150702Z-3a29e548/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9deefb875c9fd7d2fff43134ea5ad0f5a8ddfa164e889141a60f152480827aab", + "evidence_path": "evidence/20260721T150702Z-3a29e548/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eade02b0044e90ab18e8020ebcd76f16b50b01872b59a0e94cf716e965683d04", + "evidence_path": "evidence/20260721T150702Z-3a29e548/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:36002f93a2e9eea22883ee479b5d3aec47dfc0aba7197eb8e9ce6c32d5b036de", + "evidence_path": "evidence/20260721T150702Z-3a29e548/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9d225986fb3e13778b3b64537bad830b20231cb814d78ea6d8d389d527110514", + "evidence_path": "evidence/20260721T150702Z-3a29e548/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:302c60ad9da460d4b84d8d7760052318fe21f6749fc2e270dd846c9430fe864a", + "evidence_path": "evidence/20260721T150702Z-3a29e548/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95fe5adac5884d7a47fca0c6237e626386cf0eea79a07c3d38915bfe33b83153", + "evidence_path": "evidence/20260721T150702Z-3a29e548/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:89866a2dc894828253c5f7d30415fe13f36b1bc61049c7300a73d30783f5d3aa", + "evidence_path": "evidence/20260721T150702Z-3a29e548/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:10c55f756ce8416c538d4b358ebf911625ad0c822d4daf57f2367952d667265e", + "evidence_path": "evidence/20260721T150702Z-3a29e548/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:407876a466a3aa22d71992826ee2da81b9d6c11d1e29262b88618b67020c637a", + "evidence_path": "evidence/20260721T150702Z-3a29e548/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 46, + "failed": 4, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed2.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed2.json new file mode 100644 index 0000000..cd36c75 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed2.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T150751Z-9330c727", + "timestamp": "2026-07-21T15:07:51Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 2, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:b446bef30eee0d553ef1be408b2fa19e80bf43a158c73fc611857ce0b6e4f651", + "evidence_path": "evidence/20260721T150751Z-9330c727/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:09480d1051165886857e0ab01270874fc833c4ee4f82755e2f2fbc9e564a36f3", + "evidence_path": "evidence/20260721T150751Z-9330c727/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:f37bd4fe0f4b7b499b85c40d4b868f26e9502b4ca0b6e49e5abc1d68e48a8757", + "evidence_path": "evidence/20260721T150751Z-9330c727/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:dc5097acf52f4df5296080cb6b8eee5fb47fff76153799d0ccc0fddb21e40482", + "evidence_path": "evidence/20260721T150751Z-9330c727/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:1380f85403c39f72ce60ea1823702593359220446a7ad904967299005f2e52bd", + "evidence_path": "evidence/20260721T150751Z-9330c727/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:bdd30c7ea2501041bbc8393a626cf574d77e3004d6d9a4211da40ef9c1501e79", + "evidence_path": "evidence/20260721T150751Z-9330c727/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fc22ab796c21d5eeda9bdcecd7908d9d2ef53560212c19cd22cddaa8a290a666", + "evidence_path": "evidence/20260721T150751Z-9330c727/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8b97e4ff90a45c6b793a32deb1e14d6fa0b40aff875a30ef1a2ba05c82f43b26", + "evidence_path": "evidence/20260721T150751Z-9330c727/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6702b1d9729e7f24d2d0804c11821fc238148e13add9991d8f173561ff3d5f8e", + "evidence_path": "evidence/20260721T150751Z-9330c727/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7e0c4bc01f145a0cbb655b0a92a02bc4f5011bdb644510ac93952ec96c0ba562", + "evidence_path": "evidence/20260721T150751Z-9330c727/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b9c4f8f61c42afbfc4a1bea33c1127b903f1ec965e3a74881286abeecd983a2a", + "evidence_path": "evidence/20260721T150751Z-9330c727/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9d8b02946490eaefa4be342d2aa6c992bf7972dbaaa4d82ddb01de27a30b55b5", + "evidence_path": "evidence/20260721T150751Z-9330c727/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:55e92a1d9c08851d965da0f620c83818c1120a14a45b78964f3e6e71f0479522", + "evidence_path": "evidence/20260721T150751Z-9330c727/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9c2a6a9599f90a0d43ddb87fe45d1d1f5e810d623ff9a573f98353061cf98f47", + "evidence_path": "evidence/20260721T150751Z-9330c727/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:535d0a87666154951c39ac4394a6d8180f24cf3d8d467d1856c01c7c7709ea5a", + "evidence_path": "evidence/20260721T150751Z-9330c727/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5562acc8fc4d5c6b3aab64dc2886887080ff40ceb873b8ed0849bee1c4d50e3a", + "evidence_path": "evidence/20260721T150751Z-9330c727/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dbbc9a45c1707ebc1ad58f4e0a72eb57761b7fa1d31701302d0f25ea507de457", + "evidence_path": "evidence/20260721T150751Z-9330c727/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8505ee9fc5880d57d8b235b56871f5d0c36380b38b0aee9396cfe432e82656f1", + "evidence_path": "evidence/20260721T150751Z-9330c727/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e91296447e9da486b30192cebf46c0e6f1f6478631560f62d5315d73a33b1040", + "evidence_path": "evidence/20260721T150751Z-9330c727/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:35f67e7e32acc224c22192c4597c87bcfc7af4fef06f27f9238401dd36bf202e", + "evidence_path": "evidence/20260721T150751Z-9330c727/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:af79f4f992d4204852d07f32113e3902fdd35aadf7daa4b57b67e0460997066e", + "evidence_path": "evidence/20260721T150751Z-9330c727/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bc93baca6454453efe6822aa9c1b93c59bfea8ab4c5b2400475d5e095df06c6b", + "evidence_path": "evidence/20260721T150751Z-9330c727/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:def2a367e1add7ce7a74c330d15e859ae8ac6fb9bc1ca54a73292bbd39111225", + "evidence_path": "evidence/20260721T150751Z-9330c727/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a849d425f3c2936665b85b90c32d7f845415d893277bb20667b6d112769bd5eb", + "evidence_path": "evidence/20260721T150751Z-9330c727/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6b61d99942b342885c2dd89f0f7528ee0efa1b252e05d47609d35d0cc7c36fbc", + "evidence_path": "evidence/20260721T150751Z-9330c727/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d48300aa31edce0b89177707a4943294c5a48c7987649470af3161da092a36fd", + "evidence_path": "evidence/20260721T150751Z-9330c727/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95bcaaa941c49baa2926faf62cec6623bb20ab6df5d731a5a158ef127218aa73", + "evidence_path": "evidence/20260721T150751Z-9330c727/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1348193c8b49daa5319c53094db4b3c6b7283520b13cbc707f4074b5e8f39794", + "evidence_path": "evidence/20260721T150751Z-9330c727/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:39947a9946728d5e874bc3a0f714e97bc3dbfaeb4b6092ad3665d71d2593aaf6", + "evidence_path": "evidence/20260721T150751Z-9330c727/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:23479f78ac1a3475a40d387b5c5cf4cfb8c0d9d47828cd86605a8b3c29533970", + "evidence_path": "evidence/20260721T150751Z-9330c727/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a69532c215ccb4fb6870e3d94725955d82f7c501214267992dea82a4b789ae12", + "evidence_path": "evidence/20260721T150751Z-9330c727/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dbcb3c4618adfd735d722c34c5c268cc21b05c6c3da2ed375cd08788d72e9227", + "evidence_path": "evidence/20260721T150751Z-9330c727/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2fba50a064255c67fa94c98dfd0aa7e19d0e05955236afac0290c8e3ac8817e9", + "evidence_path": "evidence/20260721T150751Z-9330c727/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ecb2b68de33d9b9194aaba60377c873ce462b0254f53e3106fac438accc8b43b", + "evidence_path": "evidence/20260721T150751Z-9330c727/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f327130419e70d79fdd4f14762973ea3dc98c897eec18780b90a80b8a49d3b79", + "evidence_path": "evidence/20260721T150751Z-9330c727/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ee72f1a80f2471b4decc7d5fe3b9b682bbb3246b29d4ca8e08cb06159936ce6b", + "evidence_path": "evidence/20260721T150751Z-9330c727/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:663af6a86c099c75ed952a00b6a7eb9e67c61cb7ff2c24403036ffdf6cbcfc9a", + "evidence_path": "evidence/20260721T150751Z-9330c727/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2c7ea53c8f6b83eb537ce8f86302a1f69c827e09fdd84c255b89d80760a637ae", + "evidence_path": "evidence/20260721T150751Z-9330c727/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0f42bdf72bf232d72d391a4d9af7d4c4f1362a731fca33931163d17781faf167", + "evidence_path": "evidence/20260721T150751Z-9330c727/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:40dd6ffff960cea6b4651d0134739ea691078248a0450d5dc337da67ea02d194", + "evidence_path": "evidence/20260721T150751Z-9330c727/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:988fa7c85e593567a8fa24afa821da4c2cbfd46dcd218210b97c8524bc8d96cb", + "evidence_path": "evidence/20260721T150751Z-9330c727/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:179429ec44ad2dda3b230f46c50fae77c3ba4ba0a10f6a460ab3ebc3029ed7f8", + "evidence_path": "evidence/20260721T150751Z-9330c727/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:73c8188c8fb179360d5646120d1ed598bbfda5b2d13a0dfafb94667808024719", + "evidence_path": "evidence/20260721T150751Z-9330c727/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e0152d535ee7ae44d223961987f77f2f502775dca0ab5a5049eeb0e3b586b0b0", + "evidence_path": "evidence/20260721T150751Z-9330c727/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:672d008c3d8e83fef74d9b0309ceec3a81a1ac6316f7654915154891e7182e36", + "evidence_path": "evidence/20260721T150751Z-9330c727/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6ec1943429fa09ddb74020e5dba35b05352646df227ea6e36389e7fb19546e6a", + "evidence_path": "evidence/20260721T150751Z-9330c727/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:021dee40b240c552c89f5c06053c144660e00f0ae848753b478c732ad8875ccb", + "evidence_path": "evidence/20260721T150751Z-9330c727/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dea573ff94b2348f3a7bdbeec2719bc4b944fa4d782fbf1a75cc09e971cbaf47", + "evidence_path": "evidence/20260721T150751Z-9330c727/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2e1256b71977e4700dd8795745668a96ec95ee182362b8e0acf42f18382b7a82", + "evidence_path": "evidence/20260721T150751Z-9330c727/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b199e6bc861a2f3836c9eaea02402dc9f776a64d9f3b6e82cb63f0b4e600da06", + "evidence_path": "evidence/20260721T150751Z-9330c727/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed3.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed3.json new file mode 100644 index 0000000..667012a --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed3.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T150832Z-ff7fb599", + "timestamp": "2026-07-21T15:08:32Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 3, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6d0933118ce53a1b88f62660fb425c6ad8f5939700d17b3612ccc9311b564fc0", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b19e5302bfbafa5c4c412c8b1e2ab079053be84ab21ff93bfd7ee53a1906e873", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:6b7d8afa48dccd2c4b66c181f0da1c414adad8734f99242146ca8b697e00714e", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:598d8f7cf7dfea365e2a7cd4cd1f3d0e359fe4cbe1d63bb1441685b39a2091e1", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:cd2546fd7e14d7d68ce6abbf2aa87afdc6661e79120d93606573c94e3860174c", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:f9b5ec5ca5e969498ef57b23d19280d394b7440c30ae11a2bfe06c846d9d84ba", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d21cd11361865b2fce0f07fb1963f9d39171fb09392dbb47c5f21d378f17fdc7", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a08f05f2230a78c6e3875e9fde33591c7915ccfdabd04359f0cc84b8427f7c0f", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c8b886776c1d57217ac084fb7adf18a805d7aa13794457ce481d863587fd0c92", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8410cb408fb06df300fb91345aa08f0a72fcaa78fc86be67411dea4f520b14d3", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:831518a0b74ffe59d9e6a02a95f5ea40c960a3d97b96187ffe0c8daff3f6c069", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f122f8b46ea3d3cc854d9d48c09e901c73e238a04f4b0992625286cc9748dd3c", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cff382ac0e58200c296682ff689c1da02250b4801cf541675bafdcdd51bfc1b5", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:844de51c2e15f388ce30b4f7c77dfa9c1fae833dd91762d994c95a74d0f4bccb", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6c32d07d98064252e36f29a3e2ab1d52fe37f330e89a39ef782c0250b45bcc3", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:918c442ef32df2dd9ed8e513eccf29d6f1aec09d920394c36881f81953b97fbc", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b5bb18b789f2d990611da38f521aaf833cad0e21dc5ce8261d7d6efcf2c43089", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e0d9dba86bd0dbb1d24e5644305bbc256f52e2ccc53ff6494c936d1526204c4f", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d0613d04375b7910000ce302b0b82634013dbd6cf7604052540ef847c7d7241b", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9ea8282064e054a500023d102c2df5b79f14e4cce83334962db4984bf6a9dc20", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e9ab840609e69f3b1c9619f49b9ed96711d518dada89fdabadc450bf94f5c004", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8c7d2a293b2e376f81c4a8f5ba2f02a8a7dd507d677038100a5f477f010af2e9", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7a35b202c96d5c74015a61dc11ef2615a7fa92983513f5c2808659a8eca9e3cb", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:46ea4ec2c8d1834a85a27a0ff9396207d50487041b4428f0ac624269dbc1ad73", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1361f43d47590b9d2d875bbfd8fd11607a56e1b3bf8b89e540fac61f39907c4d", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:900dd185d48af5e3b2ad35e08d9f88752e77ed0db9edd7520d359f34bdc78e6e", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2647341ede1d3f2d5d7506b6ff3516e4d7463c8ea39cb862b927fd379ad2e571", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b9ba2d31cc0bb86f77358f02bb0b2ae3bc274f7b0ca3c94f9ecc18ab8c83be6b", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2c680a193a94da1340c58e237b24d079d907c40835116cd616df78d9cc2093ee", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a569b1e1a4b1969fadd4486ae5370d8cd728387a1e1400c46ba1e59057502a00", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ceb91084f07510b89eefdfd4c01dae840414ffaf3d168eb71bcc7e20041b0182", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ffdabf96b09b5699d91b30edf9bf58e182ded8768495ac19ad0fa9bd87e8cec9", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ea5f0d283245f008edab04c23a0e42bbb34c48a63abdf0d23ffac307e352b6a0", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ff1aa8b4f0c08a02b53beaf69b381cc846708e491ba60a4dd5351fd0c7edb864", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:af78b7511d6b560a02c4ab5b02d46cd67466d0b814191946c5c0a9f62da22dbc", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5fa4d01d781d556c95b45161bcd5baf31e82525c25732716323979e7587b7837", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9380230e848739a12138bd171d6d30b40d05f233d5d1c8bbb4a05c5a7d59976e", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dcc9998eeb2d075dd28ea3930b187e160448bf3905b4d9ca6e5dc74ec1e70717", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cc0bef459925f3093c0afbf537b3ca2aecc56e99479d699c07d363ceb78b0b92", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9ba0b345af24355c19cffc53ce95d6afdabeab95b7bfd325f2f94609c77506e3", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ae96a716ea4dc4c82a25d2ddc858e13bb5b142587473c1a00e64f024a842cb64", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a334075b932487cbb236d44d8fe415a3edb70bfe7342b2b10c7ee30cb08b1a85", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fa53680c26d1d5e0135415be9ef7048fb3d8e1d88467fd2cc2bfadf17e4db8fc", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:516881a1c5fb1e3f25e8587b00755b3872f1ed8f6634d8f43c261193767a5d44", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f4e158b4e0fdeef7d6c8634cb31b4ddee6372183af2d306a476e8ec382ebe76c", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cc1c928669b9a7a9f8a97fd63e824d858c76b27fe230a8b41d086097f5a1f480", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3cc5baf9c67e667bfb33745ac43f969c2a28361747475aa7c113c77dcff2a984", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:36f3e8493c5a9224285451b6d410a207bdd1bc1b416bf52d19bf2c12fb174d05", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:644931011ca44908c7b22658263083233418eb06a824aaa985e5fb790039a169", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c2617dc38c221060d98504489e15aa8347c1b611edc5fece64b59836f44c03c6", + "evidence_path": "evidence/20260721T150832Z-ff7fb599/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 46, + "failed": 4, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed4.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed4.json new file mode 100644 index 0000000..b31814f --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed4.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T150910Z-3e1b7154", + "timestamp": "2026-07-21T15:09:10Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 4, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2e8ee387ae051cc14bac4b92ad7d33229ed274e748eb688dba9307655a79d3fe", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:303ce2f1f14bdc8f6260b930bcda7bbcf2ccea1a70772eb1493318ad14323ada", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:467d566e2fbe865b4e77313365966cc7f0033b4305a520ebe29d91cbb487af10", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:505042892f47b5aaa39eeb2e500166c99b4ba145ab8f75c530898aa6c4a61dbf", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6f53102ed74e3e1e279e1b586a1fb15d74f420a3218c41c2c87468d44d347e6", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:b541ca39046a5d56e6ee20b0376f0c7eb6ba06af008cc5e031e9db0611e21848", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:0f7b6e50092ab12953fa5c863415f3152a73a99f898e9d2195fd4efbe9b37ea7", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9cb2494debd072db6d7fad4aed6c9fdaaf6d3236bf4481082eb6fc32b7d1690a", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fa665c4f8f4056569a855248bfc7fc248879d48120c311fdd9e6b24672221d16", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2dd7901f8aa189a981da11acd19a90df0dd1f5b5becc586b222caf7e7b3b1fb4", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d936afc91e5f7c898850a63ca9490bb503159bffcb7ac203c687bd71580870ad", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a48677750156ed3cabe14f607f1163c144bafca6843afd37a30d274ddb852e3a", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a24f9c020c6d3008585d37a35e29b882c130fa31c87022accdc99cb6ac0947a1", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7c602cda84eacfec1305835ca5af53f02bc7eb82cb23f1a6c4bfd718cfc5d711", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a73d4ee2f5338b1aeb551f55f6c9c6dfd2679e01654a30f92d72a88e15df8416", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1d570a87ded0af6dbda01d3eb32c6f7d68cbaac768c02e66e53becab567b2911", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb3a1dcd8d19eb48be9813c182b6764163045a1757bd36e77f8a1c32c9dd5fd8", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7753deaa7c35b3745a09e9312d3a38d9e6e3718ff4ec5031f596663060ddda79", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5958a7bf36e0ab78b4f6cd8c30f688e3a27acd89e02dfe5a78bb1dbb4b694b59", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f4c1633d103d141d92b90bc35982b696f99b583e28bb37c483adcd94e3f06bee", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e0383733b79b8b8e8548a44ec32fe69b023862c7458bf541c620db69c73af3b9", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8e5ac00787b9735cffe41e00f68dc363ff88ab6c29e532aa24f59a24b332c57e", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:095d539c375b86a38341e0e8fa5082a18037a84708ba28c477bed66478385dd4", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:23e622c1558bd434dcb213d570c3af475ba86f09f8a0be5e88dfc2bc768666e8", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1bb4e2bcc82dc064223bb1f3c5a1fcfec92865e2f763dafb18c68a83c32fdde9", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4488322af345f25f972d27af32170da511c3a12089a094428c4c3062262c4037", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f1c3ebe658b1282603eb7c5880c0e367fffdf5deedb15f225a5d7b4f5920172b", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bbe1bee60125cc669e46b41a8176663463e3ec42069861a79b6465d111564884", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1100b1b9e42817b36f81544a88625f8a063e5aa883ed602b2dcba684a9083395", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0e3c03818206f03c1655b85debeef248099bb05b8003b3bfc51555ccd6bf1e09", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95a0f7c6031345f9bb1ddb4f8754d6d5eef4edc0ee6eab4d8d5788b4032b24ed", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fd36df8e8fa00956cd80b7f0fd41d0cdbbd55f71a9589e72ec783851f908cb35", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e3f0368fb13ca8f7d9afedb7290b83107d5dd82c1fa8890a699fdcc4fcf5ee43", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6739f443478f3f3987af75f350d47b5dd15e70a975f3ffc4ddb8f22447e57a3c", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c0188de72446265aaf11d1a58a720160181dfcaf3565f5337eb509af3f126963", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b6d18415078c9fa78b821bd58e1453b27b9d9b50c5b6f868ca9f5a50c5d9db8e", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b458aed1e90da52c54dbb10ad8878feed6ba6ab3d7c55aedf2fbb1ad0a69bf70", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:44d9a6cb6111395610b9fac241bbe16cde9fb97cee079ed33221fed9f4fa9979", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f7a33dbbe32eb1a919353a97d5972f09d8a5a46d2c75ab6011982fc1479fc460", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cac97934ec37a75c606b72c7db559c99a25c57f81a1e2c0c127450155c34ffe2", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9e1c835e2ed1ecde8b7148b016473302cc2ac2ff827b447477fe84aa7060920a", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:87080be20c519cf6bd1014e892cfc296d56de9a568b24b600b91bedb2a76f8c6", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b7a86ed13dd0203b4418bb38447597cb48c44d55d4be07471f3ed3da95472f64", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:98affc63b09d1b2eac559b2e1e325a18c632ea0fe4b66af2ac4b3057a7fd2d3b", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ecabdb80047a4c9aefc9336aa09ae51aa536e91618e6e663c660d7b0024892ab", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:05729eb988c312c5b028936fc6f1f09edc3ab1ac38f489490e5f20bfacefd7c9", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb106f54fbe73085994e3881d5d65e2813c0648c20a2be255c1c63fa0114f531", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0bfdd70384e6c32b01464164b740e3d3eab3afbdee53b3519b2e2e753357234c", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5154bf7bc1732733dd56340b8fc4d4b967228245fb500b561723d82a76e685f6", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:04e3b62d520283a69cd43e33c7f2957b363c802b0a99da00437cf989aad77ff5", + "evidence_path": "evidence/20260721T150910Z-3e1b7154/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 47, + "failed": 3, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed5.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed5.json new file mode 100644 index 0000000..93074d7 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q3_k_m-seed5.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T150949Z-57087578", + "timestamp": "2026-07-21T15:09:49Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q3_K_M", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 5, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:669d5bc218db05cb443a3012621626f959250f5d173bede86efa6211e063d29d", + "evidence_path": "evidence/20260721T150949Z-57087578/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f93fa6572a70b5b0f2668773d654629b6239fefffe2cb947f16b7d17926d724d", + "evidence_path": "evidence/20260721T150949Z-57087578/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:44a4a9f83959d33be53595d872c1b87db5c8b29b225c9d461cbe102111639d46", + "evidence_path": "evidence/20260721T150949Z-57087578/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3237b4f39de58a860ed2988b02437288ec06f525c1c9fd3a5d8c98fd1289ae14", + "evidence_path": "evidence/20260721T150949Z-57087578/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:22b3c419e7f71c8f76777f684dad5d0c764a0a28094a6a1fc1a9a61f9ed08d86", + "evidence_path": "evidence/20260721T150949Z-57087578/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:76e38b03bd031891919090f7480b147f01d5d835763759967ce8280a0fb42f32", + "evidence_path": "evidence/20260721T150949Z-57087578/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6e4d09b196e590b358e048195a8b7cbadd498ce35ae1cc8a0c26892d305a7dfa", + "evidence_path": "evidence/20260721T150949Z-57087578/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:103a4dfbda59b5f5c51a767103c85af361b72b80f11467c7bce9e1b0824b918b", + "evidence_path": "evidence/20260721T150949Z-57087578/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4445b2b474ed314ed114c806516856166dcbc4a99de63a155f31a3a1abdcddbd", + "evidence_path": "evidence/20260721T150949Z-57087578/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:82b74a8f8dd827d5572dfac23bffd033bd29bdc1f664b6dbb97eee31ebf7158e", + "evidence_path": "evidence/20260721T150949Z-57087578/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e62fd537db9ec3a9d02d0d5af433ee03d57559549d4e0150fa92b53abe8f6bbe", + "evidence_path": "evidence/20260721T150949Z-57087578/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e8f1f087acc6731437d7759e4dc495ad80b526f5481eb647bd2d6e9cbea878b2", + "evidence_path": "evidence/20260721T150949Z-57087578/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e1ae38d432fc8274e5143460a6c765ec0d5c498a2ec5f436e282fc20e35760a5", + "evidence_path": "evidence/20260721T150949Z-57087578/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3560dc8ddcb3f2483215fd60b5862ff31e94df1b731f0b24bb6cb312faf9df44", + "evidence_path": "evidence/20260721T150949Z-57087578/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3df7978338aa8dc1cb1728bea352a830a555026646fefc68226ac374f7ed1b05", + "evidence_path": "evidence/20260721T150949Z-57087578/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f2a83523202a76ea843c5291f25da2c6e1193138fffe85dca217061554669e16", + "evidence_path": "evidence/20260721T150949Z-57087578/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c360818403518050999b73f6aed1277fbd6904d72f54a985cee82770562c3baa", + "evidence_path": "evidence/20260721T150949Z-57087578/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9025a4274e34e2d8df4130712581ef35543183c8da67825b72b5cdce4a0351b9", + "evidence_path": "evidence/20260721T150949Z-57087578/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:55b1b7e669be7d32ed0fef9f64670a99cfdf4af31e998a31ab3011680a913299", + "evidence_path": "evidence/20260721T150949Z-57087578/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dcce1b3bf1b6d3a0cd59c26fd6e6c70feee9c0213c63727e25a1dd27eb7dcffb", + "evidence_path": "evidence/20260721T150949Z-57087578/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5bf4222dd059f026bb12589c4abfa912bd5a74369617cab09e6a284096af273e", + "evidence_path": "evidence/20260721T150949Z-57087578/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:568c87db16d3fcad355c9559e96996010e3ae231185baee0ab3cff8af85bee1e", + "evidence_path": "evidence/20260721T150949Z-57087578/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:49197d80b95a2f7d94e803a59ae740d0a685e1b8155bc8bc8654e9ea3d41bc18", + "evidence_path": "evidence/20260721T150949Z-57087578/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de1b30daf1b9a7f1aa95386acdb047f8adc966dc82aa30741901c25229d7d41a", + "evidence_path": "evidence/20260721T150949Z-57087578/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f5feff280471c3c9e6c0bb9cb7379f53fd9bd13da3db921bd065c2e7009d40b4", + "evidence_path": "evidence/20260721T150949Z-57087578/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ffda4aaf0de3a3375b4ced23d11cf52db2f3bfa4e195561362ae6277bceb38c3", + "evidence_path": "evidence/20260721T150949Z-57087578/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dfb3af65bc4089beb016839108d8882d6512430b6912563c8250ace54d9c5a25", + "evidence_path": "evidence/20260721T150949Z-57087578/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d085a72fb54fb9f1b80274fd3e0bcbc4ddb42d12dc673d15f1bddfb728681106", + "evidence_path": "evidence/20260721T150949Z-57087578/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:182703ffb3b29992750afc163ae6e7fa126ff3571b879dc388f3803e0112e49b", + "evidence_path": "evidence/20260721T150949Z-57087578/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2487fa48bb36046c43ec586bce7c8947871a64fd25645ab70cdff5a286f3afc7", + "evidence_path": "evidence/20260721T150949Z-57087578/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4bd296310f16ed55e50c72cb1b6ecd43e09a4a31dd4a8a41e552892dd7df81fb", + "evidence_path": "evidence/20260721T150949Z-57087578/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:efd37fef7046dfffcb0312b81b187caaa32042f0d4a2f17875b950df3fdbfde4", + "evidence_path": "evidence/20260721T150949Z-57087578/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0306c39af8c8e9cde4df0a0398a1753e7c6842adb8c73507fa4e72bb36ae2b76", + "evidence_path": "evidence/20260721T150949Z-57087578/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:958d5eb4ef1c3ab8cd232ca11c18856dcbd1d001fbe78bec9c004b549cf96992", + "evidence_path": "evidence/20260721T150949Z-57087578/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:198d49255714f26ce9d7caf16d7a9fe613c2c8e3ac3117cf4bc88356761bfe87", + "evidence_path": "evidence/20260721T150949Z-57087578/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9aea68eabc1ae10708d0cafb0534b783470071080a565f48242d42d0b8e2c096", + "evidence_path": "evidence/20260721T150949Z-57087578/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7f11a52a67e9069aa695d5bd427436449f3f44bddebcbb47081a8dcd07f50246", + "evidence_path": "evidence/20260721T150949Z-57087578/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e27405c1498d7c666ba4dde4ee8b71d2058e1ff62ad0fd24e8f116135d0741cd", + "evidence_path": "evidence/20260721T150949Z-57087578/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0677a864e11bb25fc24e2a6a0cffc49f7d29ac8be52753c9ff76f80d0cb0069c", + "evidence_path": "evidence/20260721T150949Z-57087578/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1daef76ea964c62ee15a5635a255dfcf0f079f893adbad4c18c306ae29378cb1", + "evidence_path": "evidence/20260721T150949Z-57087578/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f1ef174b66043d1eb813bc75f866c84bafddb2e37fa9c821f7b239cad75448a9", + "evidence_path": "evidence/20260721T150949Z-57087578/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:301aaddd2f187f5c1d69156a7edfc0bed3ce93c6502e9be00d31e61dcf6bafa9", + "evidence_path": "evidence/20260721T150949Z-57087578/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:96b5c60ea2e24a3b20b3b9054c75c5a6418bc624fac779aca71eac553f2549a6", + "evidence_path": "evidence/20260721T150949Z-57087578/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:38a8bdef4e8f63d8fd110a57364543d31449c53835cfa33906d92ac9ee2cfee7", + "evidence_path": "evidence/20260721T150949Z-57087578/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3c7ad268922352b51fbdc894ded15a2e6853c67272caba8ca4af74664b78838f", + "evidence_path": "evidence/20260721T150949Z-57087578/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0f1eb29b1dfe1662a81dd92522b894648a8db572771145670f33a61ffa2fffe9", + "evidence_path": "evidence/20260721T150949Z-57087578/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7e96dc2ad86f2a81937e04d40a4ba64931c6d789cfba277cf65bd23aa6223409", + "evidence_path": "evidence/20260721T150949Z-57087578/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4d0c98dadc560a74e8613b63f74abc959b5deb0b109c80b1fc57d7d46412ad16", + "evidence_path": "evidence/20260721T150949Z-57087578/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e89540d2a6033a201177d9d37fad7af651ce920da943157af849aeb5d6f2aef5", + "evidence_path": "evidence/20260721T150949Z-57087578/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d80cb49329dfcf4b846c1f23a0bef96df68f08cd24dfea1f98259e1a5b888274", + "evidence_path": "evidence/20260721T150949Z-57087578/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 48, + "failed": 2, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed1.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed1.json new file mode 100644 index 0000000..478571b --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed1.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T150345Z-e8069135", + "timestamp": "2026-07-21T15:03:45Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 1, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:a8d69693eac0902ee36a0fb2f58b8393b77f03bde0c68869f90e224ee926b406", + "evidence_path": "evidence/20260721T150345Z-e8069135/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a6264924c3a08503d18df27fe8333b5795f710a9790cb37fc356cb3c823b4881", + "evidence_path": "evidence/20260721T150345Z-e8069135/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:28257a3eaa575fc527d9e57004f19fa63400042523d2c020b4c234dd209279ef", + "evidence_path": "evidence/20260721T150345Z-e8069135/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:c39d76673d96cee16938fdb4bc5df9fac05164bd8670a0437fb427a4352188fa", + "evidence_path": "evidence/20260721T150345Z-e8069135/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fadae7a20254c6d669db869a373bf1f36d2d3003e9b075cb3a6142d682b84d9f", + "evidence_path": "evidence/20260721T150345Z-e8069135/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: arguments not valid JSON for 'find_ticket': EOF while parsing a string at line 2 column 17", + "evidence_hash": "sha256:7c5588b79c4f5f26ba43581a955a5f739177b6045a62be81d2b78440425582f3", + "evidence_path": "evidence/20260721T150345Z-e8069135/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 3: no tool call emitted", + "evidence_hash": "sha256:be7f6cdb9d3d210ec9b831778ba3bd2019e89e6e6b9955caf9642eb33b0e0b6e", + "evidence_path": "evidence/20260721T150345Z-e8069135/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f038b08a03be63d47455aa3cfc1bea44081d9eab68c2334f78f5f7ebc1aace40", + "evidence_path": "evidence/20260721T150345Z-e8069135/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:393d60617a7dc63395c53fa80657d1f905afab4e753f2e826a4a0307832436a3", + "evidence_path": "evidence/20260721T150345Z-e8069135/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:529504d6df1d625eb6ced6dc9caa54f7bd8b89495140d66499faaa27912ce374", + "evidence_path": "evidence/20260721T150345Z-e8069135/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:71f20526fda7a1251be9d56f23b6cbe019a9e70a7be943c02824fd64119526fb", + "evidence_path": "evidence/20260721T150345Z-e8069135/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuv\nabcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:017600b841e6a8de59244a68d9bb28e2f89dcbc52f694c085a5db596b9b9cccd", + "evidence_path": "evidence/20260721T150345Z-e8069135/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fa61580d1cda175f94c3bf779e419fe11fbe669412989ee37703b7c40271b0f9", + "evidence_path": "evidence/20260721T150345Z-e8069135/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ec7cc2e1a10d1206bda653fd24accbc8a353b71fdc318a1fb23d760c61307fee", + "evidence_path": "evidence/20260721T150345Z-e8069135/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0d1ccaea0449db64f68ebe07ef89a9f3efc240fd40cb9c2ebf3b511dba2a533a", + "evidence_path": "evidence/20260721T150345Z-e8069135/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9cc31d9e05f7c23c80d9d3011b62be7d5c1d832073a72f744cda798ef3bfdf64", + "evidence_path": "evidence/20260721T150345Z-e8069135/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:34c11021035783c73eff8554805bc82390d365284d7bc7be397ea9fb89a7b282", + "evidence_path": "evidence/20260721T150345Z-e8069135/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d3054c22f5180c3a04fa3f525e2552ec9abf2222157c3860d2a418f7dfe3c15d", + "evidence_path": "evidence/20260721T150345Z-e8069135/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5592f54bb39f853f5300e4e15d6d61a482cef5af15410721f28194474ea13831", + "evidence_path": "evidence/20260721T150345Z-e8069135/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0f457d3cd5bf14ede4f9b24372b7a8b41556fdd3d4c8fe77348f83f213435b7d", + "evidence_path": "evidence/20260721T150345Z-e8069135/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:75e0fa9ffbce4a830c3d63d98862953012af96e548cf916fae3383dfe0283d2f", + "evidence_path": "evidence/20260721T150345Z-e8069135/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:437af3abee6c6502a2241d9bd07fb455ce5fd7793a1da16d2d6a73c7b1c79f2d", + "evidence_path": "evidence/20260721T150345Z-e8069135/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0a3e601cce4006920767659cdcc837506ff6d6a724b799195ca6f31de83b78db", + "evidence_path": "evidence/20260721T150345Z-e8069135/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bd2b185cbea8cebdc56905976bd977fe3637531baf699c18cebfcb6273cc5e20", + "evidence_path": "evidence/20260721T150345Z-e8069135/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:728dea4fa5f8b602c46948b0d4789e63cb7ba6c80acd4dae3dcb98724c4b9044", + "evidence_path": "evidence/20260721T150345Z-e8069135/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3ed94bd103937b726379c2a541e4ff716a0833347bb16f4c4ae1d57adae07f53", + "evidence_path": "evidence/20260721T150345Z-e8069135/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:921add1ffc2a1f6399f82b92fa652478d3558659b105d198622ab263844bbf87", + "evidence_path": "evidence/20260721T150345Z-e8069135/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e9d459ab01f0e0ec2df5e4c973c1e5595557ce74b36c6c98c3af8db26e2851e1", + "evidence_path": "evidence/20260721T150345Z-e8069135/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3c014d127457781c7984b3c44012d126c280e3950bb0c79d680b5cbd127498de", + "evidence_path": "evidence/20260721T150345Z-e8069135/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4669da8895810e9915ac31511d124faf2cfff0f6b66651745229655c5f0551fe", + "evidence_path": "evidence/20260721T150345Z-e8069135/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d652b2559588972c76678703e145411c4acd94b0de28fce2488ed7648ce9d5cb", + "evidence_path": "evidence/20260721T150345Z-e8069135/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f1adce3edeedf2172797e17c5cd3ed8800b73d4caf56f0bf0d71d82b9f4def9e", + "evidence_path": "evidence/20260721T150345Z-e8069135/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8474162a430ad429192f110eae27b11bad8de91cc9ee458ee2bb815c64fb1ed3", + "evidence_path": "evidence/20260721T150345Z-e8069135/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:35413375dc7de694a9a8d2a39ff4a9a6709c41632f23c188bdad7dcb4719729e", + "evidence_path": "evidence/20260721T150345Z-e8069135/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:44f94323c7120a554b84bd2821590c81d21094f415adced701af376c90ef4025", + "evidence_path": "evidence/20260721T150345Z-e8069135/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5baaa5e91af87d57aa18d3e2f2be4d67db3c4726df6b113b06ad62693c60d25a", + "evidence_path": "evidence/20260721T150345Z-e8069135/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ab3e065e8e87ef3656d8011f2d6cd19e8e164f05e7364866cf9f3ad876cd6be3", + "evidence_path": "evidence/20260721T150345Z-e8069135/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d594dea45de9e339922119b8bbcfff0ec3fb8d7ad2c0eb00938e6442e7cb74ca", + "evidence_path": "evidence/20260721T150345Z-e8069135/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:29d14ab7e3450023bc1871a2842daceedfba6f545e6f3706b85c8b6a6e97a39e", + "evidence_path": "evidence/20260721T150345Z-e8069135/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:12ad2dba7fbcbac4c71c885214de890861eff51ae5db3c6d6446a210c1ed6cab", + "evidence_path": "evidence/20260721T150345Z-e8069135/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4f572048c7b2c72af093cb1824984e628e8bec8a282e44d1d886b56e6207a1f8", + "evidence_path": "evidence/20260721T150345Z-e8069135/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3fe370cb68e4da8ba7369c06814ec67745641f6b1c848277d3c2945f2e287ee5", + "evidence_path": "evidence/20260721T150345Z-e8069135/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c8eeaae9f7308d46956de6bacb923ec088d5399523aef017c218d30497e95f93", + "evidence_path": "evidence/20260721T150345Z-e8069135/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f63de683d578c7f0d9948ce8fdef9d2a3827dc70a7b0d410cf021d0511882c4", + "evidence_path": "evidence/20260721T150345Z-e8069135/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9811ae95ab1b0d07279ff30e0b95dc6cdd2d57e643fe469a58ae4035407279d8", + "evidence_path": "evidence/20260721T150345Z-e8069135/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b443372f3a080997cfc39e0b4dc76e0903bdc3932de54b0e930e59b37428d22e", + "evidence_path": "evidence/20260721T150345Z-e8069135/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:57a207d3635f068b31d9437315456634151116cd26bcbdf73b10a450a6adf81d", + "evidence_path": "evidence/20260721T150345Z-e8069135/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:705829d024c435a6aba3a49bd48f3a28d137377cf4d059585fa49d94995ad450", + "evidence_path": "evidence/20260721T150345Z-e8069135/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1315622b8063055fc454a35a4bc96ebb4ff81a77e6d87b39caedfdff0b3c21c0", + "evidence_path": "evidence/20260721T150345Z-e8069135/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:03324d142b5a9caaddeab33b0dd6af4a06ecf512837383a33077ae4306db9ec6", + "evidence_path": "evidence/20260721T150345Z-e8069135/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed2.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed2.json new file mode 100644 index 0000000..d921d59 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed2.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T150433Z-6a48e42b", + "timestamp": "2026-07-21T15:04:33Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 2, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:28d1c0557d9b0c9504632082ad52b1647a1fb7c06305713b43b3cd5d22004f7a", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5e3960c8ccc1da251963a859e94757eee428f08d01ce83c75b6c9f47fbdd362c", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:800a66e17468aa934ecf69a5558569bacabae13b610bb012097e32749a1e936d", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:030afceb78400a8421855aacc8571b0a5c3d9201e00fb905996912052d5d9216", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ec6ea805f5e5541f78112c695464a71dea6ca0e6201825533d5eec7114f176f2", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:585615d05a6fba9ba25a0392df40e4f10105c2165ae0523c296befb2b2df78e1", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:84dbbac3df130a1941cda254c243ecb6cf0463a6f1f507ea386caa0a0050ed37", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ba64483033fe0c5d581beb268e2ab60b0cb899a885929011754563d551206916", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:980428afe9c4c2450e8047e247f9aa976e3ee6f61c7b0cfe353d56512c14ce77", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f0c1e59ce31d21047b410119b6511269f08cd564121ca06578bbac5710b8e3ab", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:efeeb321bb39a0d0c22d46da104deec5b9f6cd0d3d980d509953c2527cbed9b7", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:547d7186dc7cb49cd90e6ef27e7d1a4b3fa4b3695d8f1ba85838808c5bc82eee", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6370f441922d19efb25a8b88c6f4fc022cf2049b8efa5818808a09dce292a5e3", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6548a0b504ef1c270cbe6e3fca731e5965788e5e4ae5170a49508885695fbf10", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:96d1cdf3c0f6df4c8a01f6e27c0a54ef47afc4256cf750a62cbc680ceff84796", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3f156274a1e3472ff31a838421fbd6c56a5c9575b20177d8b593001b78152cb2", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:18995b60538f4e8a74333c943df04e44fb26e4756c58b28196dcf4a585848cc8", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5f8f04a5ac6be980ffab56a9bee340b8e83a8ddc4e85ef4ac616239a63796da1", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c2b5dcd274b035958362347c2c77e23317ff594962e7d699080f2a54ddac470e", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:99e88e8b4408643743d5cdbd7317a7d3b7c513d83200ceaa606db6ee2321fc58", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95e16c6441092d6e7cc6d0a18d08360b98851c3a03940ed78e1abca17f67838f", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c1cfd4732906a2912a573ab6332723466dfd8c71bf42550463742f750ab9e9d5", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7bd80384ec703285dfba86ee7eb871613e0a7ec127774702a989d45747401541", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:638149118da66c32e7c6bbf3cdb09337ce10b4deb99234a98358d13b43c15dd5", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:21451baa362d73ab700fc08af8df6153a27a6eda8c4a9f793adfbe11108232c0", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:46e9010e99792fe5c6bba5c90c21e6a42a728d09db1a9a5b92e9dda1323a6a5d", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f088cb853265db022a41eeec8f0f24ea2e86281f8c8d92f9d3d655da8b857590", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:da24f51b59b2f33a61eca8f8eddc33123ab0fce2d31bc014f5aefefeb4d5a096", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d02fbcbe08e20bf8f4d6fa6e913feb2c68ddf31fef45d9b7cbe7eb7f916f370e", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7441feef782b16aca2cd3fe77ce30ab712e2d916c33648d4d483d08357ae3a13", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7dd7ce332a8098cc8a5d1cbb77786944fdad27a90886019fca22700ae16739da", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5040a26155d646d97ab92a5f3ef5394211a99be01736f0407cbc79a4b693881a", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fad111f25e60ef744f6a39663b0938af6594e08bce3939cef46fdf44ed343772", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:54c0fee7d5805d72094e430b5da17401e8ae87cf6b4345b9c06d7061314adaba", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7dfa8ae67e0648c763f898e61dbd108df9e385cb4aa8dd211273e8ff2fda8959", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:23cd6f0e0a86be4a6b670115a158e71e20759e658a193d5ec1a0f66b9c8078bd", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:addb223d552ba795b07ac4194ace5c4c903139e0f8936db422bc841a75a4ab19", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:62e925e33e5b795d2152624153afdfe10e79a4eb7ee9f5df4132853ae301c617", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9c7cb5aee1f4697cb5d0f40efb3088badfcb9103bd2fe69b2c58723eaf9445c3", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:58dc793df0c82c2cf4abebb3b05d59ff6426196e85c5a4450b3f90f7c5eaefc0", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5a85a97db2cbb5d9d3ceb4eff1c6f4b0669ffac1c17d0ec6f9dad66345427fe1", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a9f722b1d895791d66d044754a7cbee99eb370e338ef1f2afb94872e59a80d38", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:75c74a8c84c6a43312f12ef3df4c8d749fb0a18a8c33fadd004fe24bbdd900ac", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1d0c8b26f3f980dfe5866bc7abe2a534e8de8ab9d4765d4247c366c966aee983", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b76d4ee8387bc37e1f747b666ab570b094b829ff1df0f6bb1240db7ec8fcfb73", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c8690a0e5cf56ddb86d8bdee8d52dd7c77cb7babce80fc9218f1c157943ec5ac", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0cabc2cd6a46328d221d65fbd1159e4722c1e091b10ae74d3ee58fc3c13b69fb", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:db59f634e0725f7e44345d77da4fcb993fd07974c3a7bd12ea3e17037c0ea92b", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7fc84c7df7bbfb3caba438c209c765e9f02622873fb9558142dfc9851835c210", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:43216445f0aded45a7ba47df86c6f00b5fbbcab5d46f6dd5d6cdca15bcec10cb", + "evidence_path": "evidence/20260721T150433Z-6a48e42b/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed3.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed3.json new file mode 100644 index 0000000..46b7be0 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed3.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T150505Z-1a58f077", + "timestamp": "2026-07-21T15:05:05Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 3, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:3e85c22f180712041edd6ba3172fc770b51f091d6a2c116e9c0ad429f0df7f2c", + "evidence_path": "evidence/20260721T150505Z-1a58f077/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:15e3233d555cc5e6842509005616803ff9277e9c863eef0850e8f48240ec0b42", + "evidence_path": "evidence/20260721T150505Z-1a58f077/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:148824572b3203dd962ddb57f1b1eebe924c29a34fa3c265a3b14484e57b1097", + "evidence_path": "evidence/20260721T150505Z-1a58f077/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:6b5c2d8e75f89237f12c0c5883612fb4dfa39f36e7d6b91cadb4a3f146bec945", + "evidence_path": "evidence/20260721T150505Z-1a58f077/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5089ae057755a5e732523edbe23f7b9c3fba0af1934072ff2268fdef8b9bd336", + "evidence_path": "evidence/20260721T150505Z-1a58f077/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:af4e4aa8c3254feceda4639bb283b3bb066a66de1c7fda4c2ad3c5b3fd13b361", + "evidence_path": "evidence/20260721T150505Z-1a58f077/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:f0a44304cdcc46fa01c1774f80cd3fc3e504f5f961707e70fd94a6cf02f98d10", + "evidence_path": "evidence/20260721T150505Z-1a58f077/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:83aa85be92f35de0a5b2745d78b222af2a39d102ba182a14dc04db24da72c874", + "evidence_path": "evidence/20260721T150505Z-1a58f077/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3fb8173ecee7ddee92847126dfd6498ccece2db26db21f7dfa7ba5617fc55982", + "evidence_path": "evidence/20260721T150505Z-1a58f077/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:72ecc91f3e7664795f56f6198504dcf72f66f05deeecd16d95526feea14ed3d7", + "evidence_path": "evidence/20260721T150505Z-1a58f077/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:92550677e35d52249e2d07b6da2429f59dca48d789c5491429898eed8683fb34", + "evidence_path": "evidence/20260721T150505Z-1a58f077/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:81a6c682b5bafcfaa1217d9b6cb6145af6b52823587f94d4b691454984f41208", + "evidence_path": "evidence/20260721T150505Z-1a58f077/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5c04e5be037592af2cd6b928f3f8e808434cea525e8bfbb0481fe1cd7cded9f0", + "evidence_path": "evidence/20260721T150505Z-1a58f077/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a523ab7ff1606b28079134b406caba0677f3542b5b6634788b6f0c3420209530", + "evidence_path": "evidence/20260721T150505Z-1a58f077/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a08349da0edb7c064b57115c84de857a7e404d14aeb0cd07ae16d0a6c5b9ad92", + "evidence_path": "evidence/20260721T150505Z-1a58f077/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8114f99b0d2609a20f12c10aaf884070bdcc1a8c0439659732f359809959d7f3", + "evidence_path": "evidence/20260721T150505Z-1a58f077/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1b5bc01b1a7c04c77944ac7d68b16b892df37b50f62fa10fe0f12dbd28653040", + "evidence_path": "evidence/20260721T150505Z-1a58f077/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5b82ddfc8529f43bb728e6f6bf42eab3805770d39f4bae525b108f597715f2c4", + "evidence_path": "evidence/20260721T150505Z-1a58f077/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:49686d0bce0dba440833b27c9854963394a8a65b588464b8a08ee1761cbc0db9", + "evidence_path": "evidence/20260721T150505Z-1a58f077/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6cf1604239f3a4a63acc690d5979a47b980f1723adb295f366c398b5322be2f8", + "evidence_path": "evidence/20260721T150505Z-1a58f077/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:02dac415befb9ed6a8e06f0c50b545fad006c13f052ab879073df420d21f05df", + "evidence_path": "evidence/20260721T150505Z-1a58f077/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1a3941bc854934138807993763ad04755363d4829a000d519c68158ab63ad1a2", + "evidence_path": "evidence/20260721T150505Z-1a58f077/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0ab56227cba28596a1388f90b3c189a700ce4cedeac48b11b710f88cdfab1460", + "evidence_path": "evidence/20260721T150505Z-1a58f077/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1a911e30b105893fa3dbf538b4df2649c220c90fe7f6351084fe6cd495f9ad06", + "evidence_path": "evidence/20260721T150505Z-1a58f077/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb6347002c267a6cb2e500d14f30c01a673bdf46143b6cc9c63ac09bbb8e6b56", + "evidence_path": "evidence/20260721T150505Z-1a58f077/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7358610139ec5a9d334d9386e8b4af41923a7e72b4edc71f6835d6fcd4ab3fb3", + "evidence_path": "evidence/20260721T150505Z-1a58f077/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f66d5ca49526612a88e71e0adb95b12e697e48c3746f51ee68d2d5c3ad69413b", + "evidence_path": "evidence/20260721T150505Z-1a58f077/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c7c2396bfd4b6eb16f4675447620a3a8d3ffb8419bc6291341ae4ea6d012fea7", + "evidence_path": "evidence/20260721T150505Z-1a58f077/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4663ec0358ef8d3527d7e4e784d9591259c4e79a7c44142b73d270fc835a97ab", + "evidence_path": "evidence/20260721T150505Z-1a58f077/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7111846caeb812212479252172645b9bd2b0373db794cd006b237eb5c0e237e3", + "evidence_path": "evidence/20260721T150505Z-1a58f077/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1b39d413bde483c9c001b0684c67d03f0b280ceb8d417b1d21ef080f6d9014f4", + "evidence_path": "evidence/20260721T150505Z-1a58f077/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb77bd5fb2e1cce11a66dd52e9477d514788dc0971625a9ef9733f556112cb3e", + "evidence_path": "evidence/20260721T150505Z-1a58f077/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f932faee66221e19b48b404330c7477b257fa0a9db1566212bbac4771ee43c29", + "evidence_path": "evidence/20260721T150505Z-1a58f077/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aabcceb981d4aa6cada4ad7259ac645f52b3ffe2ac1b30b5bfde709a4cbad4c1", + "evidence_path": "evidence/20260721T150505Z-1a58f077/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d0d58a98fb61c58a299637520ab6e4a1f42e3f677a12483975c44096837a56ac", + "evidence_path": "evidence/20260721T150505Z-1a58f077/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:258d11bb9b3408eacc95e7426aafb9b36e2ff3ee31bfbd72bac91c38441b32f6", + "evidence_path": "evidence/20260721T150505Z-1a58f077/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fc5a16fd0bf8a5f462e0eb7d63f761ddc35a8ea444719649e5577d4ec72da770", + "evidence_path": "evidence/20260721T150505Z-1a58f077/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f2da0f83e80d1d0e01737c7ce704d93e7f16a16ce46c6cf4c664c969b38414b9", + "evidence_path": "evidence/20260721T150505Z-1a58f077/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:74f8bed9d3086cb78bdf095f98da77989a72c939f3a12499f9cb9d038c40309c", + "evidence_path": "evidence/20260721T150505Z-1a58f077/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5157a339bb600d9bf7eae5d54f6926d4c978061f916cbfe08a1fe363a733b2e8", + "evidence_path": "evidence/20260721T150505Z-1a58f077/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:59bf6e9efbb346c5791a9e1f8a3143d08460bac8a400d92f38a4711e1f865900", + "evidence_path": "evidence/20260721T150505Z-1a58f077/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e33daafe35e4b521abe125cd683523049a8f34d9b1b53b40b349d0c81f54eb82", + "evidence_path": "evidence/20260721T150505Z-1a58f077/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:874408fe89e6286c23e8b753902e527ccd6de387e0510c600fc6dad47f8d885e", + "evidence_path": "evidence/20260721T150505Z-1a58f077/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:571604ca42a91f26448d2ca6baf4972009d974c529b5e6b7267b44a706dcc9e8", + "evidence_path": "evidence/20260721T150505Z-1a58f077/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:577b647f522a44dcfc5bb2cba0f5260bdc6053534d3fdc817b194a018fd0b22b", + "evidence_path": "evidence/20260721T150505Z-1a58f077/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:23d18f5d554da069373fb16bb37a9c20d806f7a80a5ff88772b75741529afb85", + "evidence_path": "evidence/20260721T150505Z-1a58f077/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d2125f007b76c32998bfda852cb417c40f12d5f5e25f615b437d87435870ed31", + "evidence_path": "evidence/20260721T150505Z-1a58f077/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4c81f3cd199c7e583c11cee75653ef44258f082dc456e3cca78a4a14f77e98cc", + "evidence_path": "evidence/20260721T150505Z-1a58f077/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:27d53c2e16aee7678b07ec6c965ca70c897782767fea6447cc6e9cd09f4dfd35", + "evidence_path": "evidence/20260721T150505Z-1a58f077/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:307bedc81adfa4d66ba4a538f487eb5a29c26687cee70970ea1e8c43fc64ad33", + "evidence_path": "evidence/20260721T150505Z-1a58f077/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed4.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed4.json new file mode 100644 index 0000000..293b185 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed4.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T150535Z-9773f6be", + "timestamp": "2026-07-21T15:05:35Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 4, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:c2d361207d065af9cfc26f20781ae2f369852aa9f8dcb7f102aec2cd390471b4", + "evidence_path": "evidence/20260721T150535Z-9773f6be/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0d9be49db9fa648f96bc1e81c4b9345ce1b8ef7fd1fb683aa8862e0f69662073", + "evidence_path": "evidence/20260721T150535Z-9773f6be/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6b31b538a28ce55e4d20a0c671289f3782f7109a6a486abef05396d2dd1dc9ea", + "evidence_path": "evidence/20260721T150535Z-9773f6be/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:742de67e00ec31a35b9544530a071c48d8ba02f1943a81245c5a3e395f458b41", + "evidence_path": "evidence/20260721T150535Z-9773f6be/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:43423cfe32e21e6157c342cc9d90fdd893ca41ae92d22eb1a36e04dfbf5205ad", + "evidence_path": "evidence/20260721T150535Z-9773f6be/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:ed9124a4a95d4ab9b136ec18878433ae3a683e478789a16b5506a3c112b7e634", + "evidence_path": "evidence/20260721T150535Z-9773f6be/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6f22c41dfdd007de4ab5fe1953af5376ef7ea53e678f6b1ff8758d41becff1da", + "evidence_path": "evidence/20260721T150535Z-9773f6be/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3b1dcadb88d643673e606f34f9f3e48fa006a8b6e399bcadd17a55b1b16bd44f", + "evidence_path": "evidence/20260721T150535Z-9773f6be/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b20a1385ea2370ed1b6768064ca3a8e79f1b1f891c897ad0bd4d1a68aa32d37d", + "evidence_path": "evidence/20260721T150535Z-9773f6be/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:48d6349dd7fbfa91f3ca7635e273419e1697d0cda156086aba421e1033270f61", + "evidence_path": "evidence/20260721T150535Z-9773f6be/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a846a4649689660f2737c5f736b217b9187c899f4ac8ed055edc2b484ddb7f1a", + "evidence_path": "evidence/20260721T150535Z-9773f6be/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e5dac019ea04b45b9577ac27f0296d90f7bd537dbcaa698448b75bdce3ae7c21", + "evidence_path": "evidence/20260721T150535Z-9773f6be/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f74edfaead235173f4954e643cf913d11d3a50fdb89b4cb6ca91cb500adcccdf", + "evidence_path": "evidence/20260721T150535Z-9773f6be/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:743cf61a4b903351ab33c9c26b5730741f2999dd917a61aa482ad03bd609c36d", + "evidence_path": "evidence/20260721T150535Z-9773f6be/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8c9c77abd7ff0ec11376cc94ddf5741f5bd3d9af4578a8fc64a919bcdb7e7d3c", + "evidence_path": "evidence/20260721T150535Z-9773f6be/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e27be36b87377e41da8ae051852f221f0f3e36663b00f26939b1761febb5cc81", + "evidence_path": "evidence/20260721T150535Z-9773f6be/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cad418192948882a25d3ba92428fd4b05e5a18a1c1e099312ab8c1a66814b2db", + "evidence_path": "evidence/20260721T150535Z-9773f6be/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f15b85e61d068b01a47017e12cdeea95d7722fb8db78c910d5aca23018137b1d", + "evidence_path": "evidence/20260721T150535Z-9773f6be/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b29aa3aaf2a078256306e24548111163264665839ddbaca5be286c458251ed55", + "evidence_path": "evidence/20260721T150535Z-9773f6be/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8fc850a20931a13a86ac54ff178d42d508f192d579739a2c7aad49b23350338a", + "evidence_path": "evidence/20260721T150535Z-9773f6be/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ac2e3c421ce61c48884b1b2443de2eeb374c4c1d69a4118464bcc34075167fa4", + "evidence_path": "evidence/20260721T150535Z-9773f6be/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1fbad6760b8c1f409a8b7eafe325ade3cd312555fd29ac452980fc58e2df008a", + "evidence_path": "evidence/20260721T150535Z-9773f6be/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f36e9461e200459445cf41759a7113cdf00e067345a1f6ee3a50d170ba334da1", + "evidence_path": "evidence/20260721T150535Z-9773f6be/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6738849ea5a37b47135ceb1f73bef1335de3eafe8fad9211a2d949aef0929f01", + "evidence_path": "evidence/20260721T150535Z-9773f6be/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2baf4502fa9cd709066b4ba4547d1074f13b58232ddddbf145b6d76f13d556e7", + "evidence_path": "evidence/20260721T150535Z-9773f6be/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d9001a685e50acbed92f1e7380a78b222ace369faa63d505e50de015f096c8a3", + "evidence_path": "evidence/20260721T150535Z-9773f6be/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1bf63ca41655f9cd5957302f7c4784d74825161801ac911e5cffb4a192fc4c76", + "evidence_path": "evidence/20260721T150535Z-9773f6be/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0e0d3ba0c8dfd371c3ca3901254bb7c4180be8060d8521b36c0b1244d664bebb", + "evidence_path": "evidence/20260721T150535Z-9773f6be/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7b5f5a85f8faaa62c964a3d5ad9eb316f178c85de6ca6cdac24d85ae38fa6fdb", + "evidence_path": "evidence/20260721T150535Z-9773f6be/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:15fad3f1dc8b88c8812cbb47c4075ce47184b2fa388e8d71aebd0b8da775a5ad", + "evidence_path": "evidence/20260721T150535Z-9773f6be/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8c83ebc3f62dd330352e96aec4062c28d62ba0a36c1f2c549a926ed44fe8884f", + "evidence_path": "evidence/20260721T150535Z-9773f6be/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:179c212b7d3ba78d65099535c9ee32be8ddd07334fdcb7591b803646158d6b03", + "evidence_path": "evidence/20260721T150535Z-9773f6be/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:25ecdce9f0a3f160bbf241835939ddd5abd8c63d9acef2f255625d79092e783c", + "evidence_path": "evidence/20260721T150535Z-9773f6be/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:34951636faa75292f25735edd1679991a14aa71129212970efcd5a7a1a40e4aa", + "evidence_path": "evidence/20260721T150535Z-9773f6be/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9a3204cec6f3d34f8c9af22d402c442f14774069114435c6b6e870486cdd918d", + "evidence_path": "evidence/20260721T150535Z-9773f6be/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:18c7ff626721ae5b062e1e78da20ba88c552b52e0ef1ba5ccfc2e8d68fa57d41", + "evidence_path": "evidence/20260721T150535Z-9773f6be/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8ac269867d452a133d4c1b743cd40aecc2af2511afafaa5e78490d5f20891221", + "evidence_path": "evidence/20260721T150535Z-9773f6be/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:65b9062319e1976d5fde2aa46ca9075bafab109506b5bfc64a8889c462e01a00", + "evidence_path": "evidence/20260721T150535Z-9773f6be/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1fd6fcaddc31f9d90ee151aeb96caaaf1a7621200a541fab60ce4d54292c87d2", + "evidence_path": "evidence/20260721T150535Z-9773f6be/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7ae5392142af22f904f776d4ac9ff844c9eea0b5da32c91c689c3e8e5fc85029", + "evidence_path": "evidence/20260721T150535Z-9773f6be/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f2302889f5dec61df4db791a31d13c21b78b00cbd39fd8edfd035232579cac00", + "evidence_path": "evidence/20260721T150535Z-9773f6be/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:45c9e0c178ff240858cc7642ff8f50a7899d9eafc338ec916161dd9979dd7be7", + "evidence_path": "evidence/20260721T150535Z-9773f6be/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f83534d647327c444c137cb2d98d5764e52e75e26b0a469823e7ec0f9e21d08d", + "evidence_path": "evidence/20260721T150535Z-9773f6be/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:076965d7720da2c7ce12ddd8ec8b26f93ae56757d9241398970267e2a548f844", + "evidence_path": "evidence/20260721T150535Z-9773f6be/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a3e207f5df25000a61eee0c392814396b27e5e37b5518b16cc1ba528cf4a3b4a", + "evidence_path": "evidence/20260721T150535Z-9773f6be/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:045047e399570a8748a1e60aa148ee63689ca91fe5b7ea5c12e6439720214ac7", + "evidence_path": "evidence/20260721T150535Z-9773f6be/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6398e08bef74281387fe622c3d1f26720fef745de3ac18780b263096b2fa2f1", + "evidence_path": "evidence/20260721T150535Z-9773f6be/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d8576e920078198067c50a7c64d5563ff568b7b8e02aaaa435c6a8522af15bb7", + "evidence_path": "evidence/20260721T150535Z-9773f6be/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:67a513abba104890a1738cbea171fdb24cf4e40934ae104be095827cdc6f4c70", + "evidence_path": "evidence/20260721T150535Z-9773f6be/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ee48c3392f2e4d6e023c581ac5d434b087c9aa1b1087c51e5fd4b288d6da917c", + "evidence_path": "evidence/20260721T150535Z-9773f6be/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 47, + "failed": 3, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed5.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed5.json new file mode 100644 index 0000000..129d44e --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q4_k_m-seed5.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T150606Z-d7e331f9", + "timestamp": "2026-07-21T15:06:06Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "declared_quant": "Q4_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 5, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:0b625ec1e58009f985cc304a8b58e549a489454ffa93797b019cfb7a2a77ec72", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:29f24fcf194039e5acd83b00b16d9290fb188ccce4050a618ec119e9157884a2", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:3de6331871a8b51e3092e4246424991942c179c50452eb993d632b9df330253a", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:19113477dcf3a7a262823ef4c4c159a0dfb33b5505aa510387e2b299f663eceb", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5657a023deeffa2a92746c0cb6f4d33ece6c776972c0b49e0cdaa05ddbe13d17", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:650bf0683677fa16534c123bbf9b397da1ab76d1d9e458037ac2bb967387451a", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'find_account'", + "evidence_hash": "sha256:e6b3d4a9040409905887b2a191308238c4d31c28823f83f2bfd99b4e5b416a8e", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5eaf088c76e47f86b41d92a8cca9b028db011554693ab0ab2189ea97b3b7ded5", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6e56f03194632c9c01eb61610cef76d0205868537cbdb05f528b57974eceb3e7", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ff37608ecc22d32daaa39bd5879671bce81dccdac95d31d89ccd1dc17ddd7cd4", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3aebbf7209d07107d95ac11bbac10057b7a37d36dd1517ad4a02e847d63d83eb", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1e3b668756da64b7e2dd2b0993ffa7a506d0d768014686eae18b626eefcdd3af", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cc0d8792b5520479c4bf181e86016cf86417eb0f28a9b225228addf37026fa92", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4226c1f75111dc76f7ea1a00407275523663cce68041a446273d9729b02c6aa5", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d11484bd139943d8824f5b72f1b4029d74829e2d9bba7d897a3fe77bd1ea768b", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6377c9730831f5c0c7a4b6dfa9681277e29123dcc45d8738081730d24684774", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1708870cf19f95be52712d3794547015ad541e128f4acf38e2ab0297093c04ec", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6507ada3babef33c1e18d51faf67c9883912eff9f86cf65d5e1eac76915ac5d5", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e8740dfb511a8497afd4092b5953b82968b88939906dc3583ecec59a15df85e6", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6511f1ce161da29dd0b5820f42de8c0fcfdea941689f5b1e3a37d25cabbe179e", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d63bdeb1ff450f48fe63526a887f3a1a3ef1f6b2f1104acbe13dbddcbea7e114", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7cd1c483836827a7540194db56b6e73af2b707a6c172fad88548079f9b71b073", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9e1b70ba42a3ee0ea9679b428d4d09d7da0ae925837d0fdf7823bdbe0fb5f88a", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:45337d913b539219e4af041b55fa9f247a885a49d9ae1e46c89247b8cb47264f", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c226fc15ef78ac8239edb7c1b826f756ca5c6674bf37af973da057dc93971877", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:044499196f1d150e11d4f4bff90c47c8605ed0c0a342a9c4b7fca47249260b60", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5996f23255f4c8e7f1b4248b3e84767c3f1812bb1a6f43b02a07a4e5c9c2b717", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9f3f0ed9d3ad568ae373ef4ba8e1be2c9b30a16d7d9cd22ec07bb02432796639", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:98b9732fd88b8309abc3471c8a53a63cf35ba980770490eda11ec73ee11192ac", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5d3b81bf4d6b5aeb6571b03cffaa8c1a37ebc4c46eb4e9c7dddcb0880ca74c51", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:45427f89c39e1b9e4c3a0f5cd7840d84a503d9b69d099ae017982ca4cfda8fd6", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cfcee44001d7c2a1302116858e3761b312f1635e9d517c9b3f939dbe58d6ce26", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e90d56039040ddf6466e4866945769d90c7bac622a6ec5da7e7bdcb03ea70747", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b4deb2c421236edf5ec1956857744f529dbfe6c9bd02808f30019cb9d5624116", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:02d226d09682307dfc5c5dbd6ad5a107747c742a1e38b4a6968d2eb67c6436e5", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dcc69c34e9a5f4e4493a00445d2803c0398068654ede8e61e4b52603425561ea", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:88faa138c1a09fff38b69fe1a6368169e25399206bcb1ca4a6f634f4270140ca", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a6b44ef196f83a9357faa61a8650aeda0202d083fd7061a7cc70be7918ed8749", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f99128018cad2c62c90e6d99cb2ed507c30893236be0be2357040409611f8343", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3f7ae9863794f9f738c6e6991d912df59600c1a6438642e89a1599dbf4554250", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:895fd664a29d5a4c615ac889ca3a599381b81d3b890a699041808073def66dbe", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:27a5a3e743f0e6e1745d1b22c3ed416f5a5314793f9891422a6f452e92daca3d", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dc0159ddf1ca7c43c63560f8bbe6f04e201ec54728f02e1394834fc39d5347fe", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1f6e51468cda7a52aaf80a53f674ac347acf742d1301e4e41c08fae4564f0455", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7aea8984b83e3b699426350d7f02367ece1c0a618b28d4943a4a117dc3c785aa", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9eccd19e57e7e3509cdd6795d2b987a7af71fef30240d3a0fe2303c98efb7e24", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:02ae79325e62d2648a877478a2ba0b37e5c05419013139b6010fb4ab0ff9f14c", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:97f76e1dbc8c972b77df53b5494e76d0bd9e521d7ea92913c18412482c17dce0", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:229c13da45ba58dcd4b75273cb3c0fa633c388b340e8d458cfd64ff4e50bf608", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:90fc694ab19b6563f08a47b27faf556a9ee45cbb9b470546043d32d87d2ad251", + "evidence_path": "evidence/20260721T150606Z-d7e331f9/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 45, + "failed": 5, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed1.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed1.json new file mode 100644 index 0000000..ac8af29 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed1.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T145931Z-27eea50a", + "timestamp": "2026-07-21T14:59:31Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 1, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:540ecf4706bfa726e50ba953bc539a14d96e107e2d2debc1a43edc8694216991", + "evidence_path": "evidence/20260721T145931Z-27eea50a/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4f3394128113d9567888b9cbccab40e58b62f4df480f94bed5a6d2c0e5ad71e0", + "evidence_path": "evidence/20260721T145931Z-27eea50a/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:07738e625d7d8f5000eb7f87c02d1455598567f4adfae48704e219c872bb6993", + "evidence_path": "evidence/20260721T145931Z-27eea50a/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:6fb2b9d834d3e91dbe99e6282803f8cf30f76338c96a856dbcf18d0da47d2306", + "evidence_path": "evidence/20260721T145931Z-27eea50a/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2260b75de2c9265bb6f7035f2709ff298cb94e3c5476466d868d7ac90a4705f0", + "evidence_path": "evidence/20260721T145931Z-27eea50a/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:9c2a8431120a5e017f71787eefaac999350090050d8498ae8e775f2adafbbcd8", + "evidence_path": "evidence/20260721T145931Z-27eea50a/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d6db3e9fc953fd1137449857d9206fdd694f8c38104832e0d48b4d6498a127ca", + "evidence_path": "evidence/20260721T145931Z-27eea50a/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0818a544e9ebf6d7c1f86374421d718af1a9a5d3f66d69e446a29e70cf21b2c2", + "evidence_path": "evidence/20260721T145931Z-27eea50a/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:680aaad901ef8f6f651e9c18102232d46517b1ff1d6c9697db60f82047115f1c", + "evidence_path": "evidence/20260721T145931Z-27eea50a/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6b411002ea50e4f9a421c9ae98e6247bf3eee943b41542ef2ea59f93a892b069", + "evidence_path": "evidence/20260721T145931Z-27eea50a/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3035505c8a4c3dd1c502d995609b629024e88de00cd0e13adcf47fe5a149e2c8", + "evidence_path": "evidence/20260721T145931Z-27eea50a/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxy123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "evidence_hash": "sha256:aa4571b9f525babfcf7bd0ee9fccad12a6171df6277a5a0c2831206678674074", + "evidence_path": "evidence/20260721T145931Z-27eea50a/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1e34834c75ca1db1230cc2fdd3e6858bc5cca038af135e1aa107c8d212fdcccc", + "evidence_path": "evidence/20260721T145931Z-27eea50a/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:533e3001df598065558a2feacc443dca96a7a1da2e164e8bd030a9390a9ac786", + "evidence_path": "evidence/20260721T145931Z-27eea50a/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b136ce96e9ed978a5a3189880b8a6bd207eb8407ac8db3716ae49af2361125ca", + "evidence_path": "evidence/20260721T145931Z-27eea50a/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ab25b596f8e03f2f9dc0b892860c9f80fce540b1368750186504b8257e0dcf42", + "evidence_path": "evidence/20260721T145931Z-27eea50a/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5ae8dd0194a8cced3cdec8b5edf735d20ab8bffa99337ed2caa2a572b10926fc", + "evidence_path": "evidence/20260721T145931Z-27eea50a/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bc08636bb45262460ce37ba63051b6853450b604cd625c62f2fb206dccf4b5be", + "evidence_path": "evidence/20260721T145931Z-27eea50a/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5b22b3989dff6405e6f034513841c543c31596b51c8d04cd095dd51be6542ca9", + "evidence_path": "evidence/20260721T145931Z-27eea50a/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3f9078f120d26c5b718f9dce56c0e73e7d6ad151903bf2da4793bcddceee6790", + "evidence_path": "evidence/20260721T145931Z-27eea50a/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0f0463c7cb96f26856e08106bb11ede48cffa83cf5d3e163dfb58080e06ea612", + "evidence_path": "evidence/20260721T145931Z-27eea50a/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5c8fc8d47e32aa23de582d9e177559f1ba7e8508ec91959fef6e81de634ea2be", + "evidence_path": "evidence/20260721T145931Z-27eea50a/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:53da3856194361f491efe5158fab94a1fc7cfe2ecc17cd5cc52af3402c87e6b8", + "evidence_path": "evidence/20260721T145931Z-27eea50a/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e3a033a20f5d31a3dedb0b34d79eabd53ab50a82c06a9d736ac1194335156f05", + "evidence_path": "evidence/20260721T145931Z-27eea50a/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fd543050409eb9beef38987ce4cdc64c9fe2134fb36cca69e1247219a6396795", + "evidence_path": "evidence/20260721T145931Z-27eea50a/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8021377be517b4c826c118029734989bbf90c5f33951d363f0752ebf0778d268", + "evidence_path": "evidence/20260721T145931Z-27eea50a/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5fb8731d3fcfca65a3f4f77cf354ed256fbc11ce07b589c86abe8da75ba32241", + "evidence_path": "evidence/20260721T145931Z-27eea50a/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f28f73ee5129a1f032bab63532df3d40749001ff5fea25358ed83c68f4c5a7de", + "evidence_path": "evidence/20260721T145931Z-27eea50a/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eab6f13751d9ee32dbace4d0b0cd4648a46ca8a688a50e00058995575675f8a4", + "evidence_path": "evidence/20260721T145931Z-27eea50a/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:55f7e87249e30cf6a9da9da5dda60eea10a5848ed9f3365d28d816d63d642846", + "evidence_path": "evidence/20260721T145931Z-27eea50a/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ff69afe4092c4812dc8d0a2b166f2436ee4447fd52ac41fafb4d991189767bbd", + "evidence_path": "evidence/20260721T145931Z-27eea50a/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:30672fc0277cff69ca5a3ac8ee43d2230ccf679ed1b8cdec8ab631f189879ff1", + "evidence_path": "evidence/20260721T145931Z-27eea50a/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0c3d489963385846d401de535c497b68a993be134e716f48502187e0afa8cfae", + "evidence_path": "evidence/20260721T145931Z-27eea50a/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:496393bcdbf3a71d827c02102ff097bab6278e6fd18c56afbe0e05a6c3c032a4", + "evidence_path": "evidence/20260721T145931Z-27eea50a/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0ee79cd7753da15597d33286d7b94bb4afdc14e8cee9df32a23590b23eb83286", + "evidence_path": "evidence/20260721T145931Z-27eea50a/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:06a395ac18b9a8e10b0fe5602fdaa7c4dbd03d1143165a5da1bcd90bc2e93b51", + "evidence_path": "evidence/20260721T145931Z-27eea50a/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e26dcc192e8f4e8b608861eea8588a0685b4f9251d008d55e6a97257ae92d2cd", + "evidence_path": "evidence/20260721T145931Z-27eea50a/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d3062885490575ae916523823bb60142540c4b2137b64bfc8174aa484440792a", + "evidence_path": "evidence/20260721T145931Z-27eea50a/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:273627bab8019cc28f08b2924d009129761c709bd7a29996542a185a39ba7ef3", + "evidence_path": "evidence/20260721T145931Z-27eea50a/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:19f9a04f2ee0e681cdaca515c01c33a9fce4c6d34085682fbae64d286e88695f", + "evidence_path": "evidence/20260721T145931Z-27eea50a/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a3964fb487c47687e241cdfd7234864c03d725a960e9ab802a7f747358abf5fa", + "evidence_path": "evidence/20260721T145931Z-27eea50a/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d92ec963d440bbf861da62a7bde0d359bb942c46d7165f5e33024c860657dc98", + "evidence_path": "evidence/20260721T145931Z-27eea50a/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dceee7b31ed4d944890ed8cce5bc386c2a93b3e1bb3078936688533dc236470e", + "evidence_path": "evidence/20260721T145931Z-27eea50a/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:159208414f5e568a84848fca838a3f84025126db9b49085ba185ec25d35619d1", + "evidence_path": "evidence/20260721T145931Z-27eea50a/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2e6dc8ca515977530bddd50ba92d886f0f4c23b66cd107e1a9495b56cbe18c80", + "evidence_path": "evidence/20260721T145931Z-27eea50a/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:068922ad02ea93cff37af7d53cb55c22c30a1a7e09355f87b38d8d7c7f187790", + "evidence_path": "evidence/20260721T145931Z-27eea50a/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:33cd21b9eea5d8fb8aa1ab9eac91604dfdf441e8bf19b9febcf60c827ff6987a", + "evidence_path": "evidence/20260721T145931Z-27eea50a/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:64fa753531a2958c37e5611cefd18b3391c8225f4c8f234624454542ecfae3df", + "evidence_path": "evidence/20260721T145931Z-27eea50a/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fb689dfe5e61e8944863da47cea586ac059fce374725242e196d5c5c63d72f6e", + "evidence_path": "evidence/20260721T145931Z-27eea50a/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8be724fa717e20fe832adedc11595ec2ca6d40d9e5eb8b5ff264294e428d6e2f", + "evidence_path": "evidence/20260721T145931Z-27eea50a/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 47, + "failed": 3, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed2.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed2.json new file mode 100644 index 0000000..cd577d2 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed2.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T150029Z-b3cc380b", + "timestamp": "2026-07-21T15:00:29Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 2, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f24e2adf0fda1f3d1037a2e796eff8440d526f6fdb32205c4da1ba78f9c9dfdb", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1d0c0532bfd17ad48b4e8af2c393216c21b637b3adaf0517c5a6e7721785708f", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:53aa5f87a82e5c889ac884657ed3c3774abc8cd78ef1edf6927f5483032eff47", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:00f4acaa590f96661d4af766a6360a56544b2c755966843208a2e5e29c90f9c8", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c002bf5ec965762a1572033e5b99793210da7f10b84e3e09de6d0b07adf76ad8", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:6fa5ba2c3a3321949353f102c21ff7bf2ab43a5abdaa3d1f11ca05b938406624", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2e89a53ceb11cd5dfbb2383e854a10784a69a57042ef71a1d429f252ba40b3a8", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1c9771ff7bf742e9b5895766c86e5194c41633b8ee29e8eb739194fafca9d31b", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bcd9b7290e5b77589ba1cbe2fd1bf182fcd65c6dbc3c8fbcc63fa8554fc39886", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9b82e25feb30f0f7e4ecb52d611287307eadb06c69ec06bea191aa09ffda3ef3", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:39a1899bc5825b26e45593fa69b17b305556983360b49da17fe03a5f8d3b28a7", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-", + "evidence_hash": "sha256:4c6dcc64a3022b397fb36ac5388070228f7f44a1dc29792bb68a30a3d7007f56", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5aeca8d246fde2f74f905dfa39e5e4fd6f3f276566262abb3de7c4aef5532ead", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9b33c0de1ce55eee5e7eb05a2776b5d6c08a8127487dc9d4f20b87a4949ea09a", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9fe17f8b21b1f2fe2012847a29db6792871c675484798ee20b5cddf26cbebc47", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b9ff8fcf9d6117237f125c92685f4706c63f12059e19c99ba9ba711045156119", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:321c8b1e30943a9b80a192c4e53dfb1378da51a2e48e4375097ccf08f8011e15", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:beca50d77883e850122fb3494757ebd7d65c3e10c175ac5fd6361cc73ad3d2bd", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:79edfcffcb497eca92b5b612401a511aaa3ae2801ef37f791cf8b91d181c16e0", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:224d454740cb4685441acbab4040d1c45c90ddd44121ec9a463917e38b741aea", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2525b4e5eec87bdd58b7584724a5c4ad197aced9fc2415da04e9c1b80f99cde9", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d561a07a29b83e31fd679bbdb5cdbf8ca340e3682996bf87579563846e6fef58", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9b6dbc9b232db96289456abc308d95ecd4851c934813789776d0149dbf545075", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f19e4651d1db0e6d6af0b1f1773a6bb2f2573bb44bfa702c1bd9a43fc1eddea6", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:96277237b80f3fc0ae32075b1300aa6cdd12baa2b32e3520c0696ff7e1399aaa", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:43ea4bae4a215facbbc5828bd85eb536caecfe392fb85a143fe5092311fc22ad", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:deddc980c2b761537b392b1df9358d0fe7d2fe5c658a6b2e58b003342e6d2e8e", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:44ea44467aed08d0371204c129b632d263725c8a77e31d3352a87af213324173", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:166c1bdca021028e44d73cf0bb7fe33dbcd88302e926ea1e64e758cda9c814ba", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:51527a095baf8927a8a1a3b52e8d1a6ccec3fd59865d89929d044ef2ec39c083", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0a714fd8bf75df8dcecf58b448dee5d22a0680314f1e91af2cdfbade869e4fab", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:387bd8c800a51fa43e27017047bec5bbf64ba94b49b83d1d1cbb9e220ca60ce5", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6cc484ccc475dc05c1b8843e129e6f1dfbce6b3d402a6fd0bf4df259c601aec", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0caf80d8a8aff565fd4fd8b3946e4a70a30cd065dd2b7d4880d411f0ae4c8d4e", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8e3c2140c8b245b4fa99a3943c99c56b2c806a21463f75aa76e84451e936293d", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:26a190e1fe240c4921de33ad86bdd7eddd23bdb1b63f8640279a7643bad2e96a", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e9faeeb6e52581daca4bb1e06e5cd5b63c256eecfe7555d6480c587f51d2a3d", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e269801d0067db5fdd62c55a7e0cde20885d412b225233f7c9eb8464846d8683", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bcee38a25bc8fee95f91f2c24a466d2b80929852ce5b8b59006011c5c8387824", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b183b12e7b0ca22152d9376241b4481b98f50138e6a085dabec7ebe567d8b35c", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:99ad4a42a94b8a78290a4397c64e96594d8f44856f04b0561b19210bcbb94c18", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:41395fd2bbadcfc66e2895d87f1d8c3e114865092b64202c4664e1f10d8e9566", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f4d2468464493389cb5685026afd4bb6ec8a08cf4a7a9affea8926b4d6d0012f", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3c732b24f2f30fbb48e6b863434d81023fd761cb08678a19dc576a098ac6e396", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7b2673b3e8c8a370447ede0f56b5d2a46dd84e33becaff16373d63fbe7b5c369", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9fa31fe03b81006eca38a53b365a69dcb6ad9cd1b8299c35667714f0b5763a46", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9e0d8e981e10388f3661232bd276de61e975d59e3a02c108f2420b3e474eecc2", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ae073baac8a065eb9b00e285a15e5c9da98cdb46bb05ea44215aec48d69ffb5e", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a0c7b828dff05ad6214572b7fe9b9d6f35a2c30e672fb52b2f02c6bd48e56f6b", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b2e12d8c0999e742d76ecb7dff23f0900aed01cb2bbdf69565144af85ea1890e", + "evidence_path": "evidence/20260721T150029Z-b3cc380b/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 47, + "failed": 3, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed3.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed3.json new file mode 100644 index 0000000..ef3f32c --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed3.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T150113Z-a75e8eba", + "timestamp": "2026-07-21T15:01:13Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 3, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:16b40b6d43959d636c0f36ba0b335bb31c1e8a83c0a0cdd004d47d24c61830f9", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:24450e3e8c32cea1d504bffb9c465db41d16fa94c692db5ee470e152740adf73", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:c0689f7b7ccf34a626abb9496e5ac7c7e648e1447f09e68c3ce64eeabd4cfada", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4876001d491c2d4eecac1ab6ba74f75d5579d715228a67dfc7e742e55685620c", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0c4c678f92cbc74c8ea0f1528c6048a7283c57a6c7a347e1771b8dac8e9122a7", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:c3b66e7f1a8fdcd317c0479ed9783003cde9154024591c9dcda6b8c8a1057ad1", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9da119e2159566ebd7689da8148b52ffab7cd0d5362bb99fa3be5cb5640adba2", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:35e79c2915c4c5b9e8801d1c4caf300f2925ed4e05afcc4beb39dd3aabcdd1de", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ea897db1b36adada47371fb01a88410a423479796518c993fdab69611bebed2d", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:775a7f54f043a3b7ce778a07d20215d7f087d74d53bce5520d7e3f190f3d7335", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c85ef9cfba6c768ce39aba20e920154bd363b867cda358b888bb0b277ae9c468", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-", + "evidence_hash": "sha256:ce40b3b264715ac14c70fe05b0b9dcddfe986775b2be55d080f39df8102bfd7e", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:adb9164a98fff089e34b7e2f88ddd7093535b2800c8ccf2d462338e5815e21da", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ef4624cb83def34a05b3896df9e0f7f86300a9bd04abd9b29c6f22b65ee0e5d2", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:88927a7dd27acad23fa3cb87ee78881116e8d64cd7c0f8deaefde114a7d8e5e6", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:102e3ebba5e08999193741e20491d025f51354d23c5b6b40849c0757ba578b23", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:676b2ea1ad51e09ba6406cd139d115728537cfdcc582fb6a1e6a1d1a0eeddbcc", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5823779755f66e548bf99673bf322f1bea2f8f84be75180033d081cb1aac5414", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:764f3c723dae87db7e6f5fb3278aa37ec618e05fbe060eaae01abd7dc567a847", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1d81d9144fdc4080ec2be2ec438f7ee4f56c20d134f1b49f42e1f774afe9e10b", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9548a90b08e784a53372da3db2a92e123f01c72a616443f3949e907386d84e1f", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a3a39ffa63f13b14a610aec108b465b7af7f69d57b69841e77224ee2ba131cd9", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:03322ba6fcabc3510ed3ea8af1dd064faba2794ae8a44721222d0ce4e8fa0b6d", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ae212d26446b02e2f7d1c19fe982dfef089db903885ba573fa029a65c4eb0db6", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7bc9d2c1a434d4826fe039e0ca54bd072444aee7b1e2dcd2555f205f9e2a274e", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:25ebc6b7b75bf5f02d37c441a8541da52a0e295dd446f48566d83ad9128cadcf", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:28df832cba0432d440a04374f094170f38ca43801608ebc139d2f6d7c0028bb0", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bcb7060e7793ba89d71cfcca662957f3d7bdb573f414438bf94b26cc9a0ea5e4", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f91c88bd9b95c75217be35fade6189340e23a8f2130dbada12dcc46faa3fc35c", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:57c3e642dd1595419b1507e721653218bf52e7f7e02794b832664816684c08b7", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:78dd0b91e79ad3e4c7a277da776f968740d5b62c2002421fde2d29b47fd1980c", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:08aa26de6887bab2a5eaaf7e7e272e2674ed66929c307e0b9b8e39999f2d4d5a", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:49ad393ef713f6d82146a4da01cf7ffa710d5defe0baa43f8f40acd521c3426f", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7db1303130053027d374da5275f5b46bb027ceac6a6e493029ba3515fdd9708c", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5114aa03ecc778a48b8f383f44597b4e11db32b17365ca77209ace23405fc997", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b41b87af394db5ef06c4a0272f4239e3b71554d641c8b5962e69c95021427b4a", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:04d8396aba5934b0ada71b8c7d37b2e5d8bf351b45e5a18fee6d9b5a6dc15a73", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b7ea4be01a4609128f38a325bba879ecc5e0a7c607970ea6709a36fe8c97160f", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:61ff7310fcc32e6414a1a9b669977578a2fdc9ea91167252789a9be6cb7edb67", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5e38ccc27b069c594a6e2a5d7261162b63e69a943549b2d7e9b660193ec7b5f4", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:811aed1a3db8c5573f1f193cb9a072142822b332f79f824655c3b77941fa7afa", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eac44ec9d9b1f882dbca91d1e55fdafedcea134bc559c56cad915dc2d5f4dc69", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c66255d9329fcd52a942a7a5302eb61dc54affaf5bfcda7c62aa8f1a0aeab3f9", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6dad1246266d009644c6de4e6aba78dadcb6a72a476084069894db9e30b4721f", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c8ac4c73c551efdc214cabca2fa88d4807fce29473f892ce261fadaf4f803c71", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1390e4b362730997b4d213859bf508e80d811f181b62e43fe0dd931df3b188d5", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6a9153815ed3d505c36110ce4b26134e32edebfb47c787dfb16f0152356308b3", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:62629a2f23375c8a5599d08e17901b2f9e917a67060b844abf1028e1dd5f55ea", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1e3023c1e51d87e21ec3d13138d118ce53e02d28e5ebd8f32dd1ad18621bc643", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:704779ba783fd1c07749a192ff27c6462b552eccd1f6a63f5c30eaced4445912", + "evidence_path": "evidence/20260721T150113Z-a75e8eba/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 47, + "failed": 3, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed4.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed4.json new file mode 100644 index 0000000..32296f7 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed4.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T150156Z-42d85956", + "timestamp": "2026-07-21T15:01:56Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 4, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_event'", + "evidence_hash": "sha256:ebe190e5913180254397bdbe6fe840fe35d07198d617dbc33b34f781d1f428bc", + "evidence_path": "evidence/20260721T150156Z-42d85956/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:11292f053276908fd69333c7da87ab17f8629887b8f503a95fbe0b0024f52c5c", + "evidence_path": "evidence/20260721T150156Z-42d85956/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fa357598bda820235b8b42c66b034f6c8bb362ee1ec324cc5ae74e53a6740a17", + "evidence_path": "evidence/20260721T150156Z-42d85956/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:03df37b0f50efce65960b7a7fa9ae409681b3b631674044d7a157e5b049666f7", + "evidence_path": "evidence/20260721T150156Z-42d85956/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:abc39828c9bf6905fa0a7ce44ebeb8b40d50c50bff9c3d1a363d1a3d30ea8504", + "evidence_path": "evidence/20260721T150156Z-42d85956/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:6361af4775f250c8a683cc6effbb300e6e4794d124de68a0a870cab145add0d8", + "evidence_path": "evidence/20260721T150156Z-42d85956/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:98f508611ec39c932d6bb21e3c4667da16242ec59b5b1536a19a4679d545281f", + "evidence_path": "evidence/20260721T150156Z-42d85956/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5a362a2266b61aa3ae44258a4fedb2a6331ce2e7df4ac852cd8d1bf6e5bc29de", + "evidence_path": "evidence/20260721T150156Z-42d85956/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:58ea3748d279007d8eb90a42c2dfd6ef1d92a81ef2db9df4d99e4e6ff852ab6b", + "evidence_path": "evidence/20260721T150156Z-42d85956/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:87ce68d9ba5a43a1cad21ccb3c799577e16a38711f3623cc02db0c1c753bae7b", + "evidence_path": "evidence/20260721T150156Z-42d85956/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:baede9ca9f0e431a132f318869a1edb9e69b92116828d88787a0d8ef4330e2bb", + "evidence_path": "evidence/20260721T150156Z-42d85956/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:073013f7dafb1dab02ea667341c885a5d353d78f434e0066a719da09ea6c2765", + "evidence_path": "evidence/20260721T150156Z-42d85956/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e1e644f910cb8d009046756515a63de98a0ead88f90f1e6511b363f1ac4bb5af", + "evidence_path": "evidence/20260721T150156Z-42d85956/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8d5406518ea3c68d2f6c0e1b6a86ff57ac43de92368b43f240f6dcead599de63", + "evidence_path": "evidence/20260721T150156Z-42d85956/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a305606c6c2aae752236bff4c1c0e94fbaade55b843aa1bfe69bbf5acb5a381b", + "evidence_path": "evidence/20260721T150156Z-42d85956/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:202bbf0e5ea75a60d1a1102f92d56a953b4b1b85111b32d637cdaabc076875e5", + "evidence_path": "evidence/20260721T150156Z-42d85956/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95a1494799f39e70d8e3a51923aa3de1483c1a9b067df11ed845c4ba1a1ee39a", + "evidence_path": "evidence/20260721T150156Z-42d85956/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f72135245312ae58c6259a24d6f8e473cb7239035c09136197fa45ae417af8c7", + "evidence_path": "evidence/20260721T150156Z-42d85956/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:efdea975248404a1338539065dbf632675cc3f02cb70b3fe64119961aed56ee6", + "evidence_path": "evidence/20260721T150156Z-42d85956/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8c55a038fd8e161c6f8e3b9ab207641f564ddd19442dcef65acf3f5f6dc1bf7a", + "evidence_path": "evidence/20260721T150156Z-42d85956/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:80a0be0937c38a59bcae50711f6f0092fbddb3c36e4b5bbfa022beffa70d11f5", + "evidence_path": "evidence/20260721T150156Z-42d85956/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dfe657d658470e413a68cb698d5c1241d3298c8fe2a4bc50e21a0daaf89d2ff0", + "evidence_path": "evidence/20260721T150156Z-42d85956/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5b400de000b3227e8dda2854109b50018a528435a256b84e2a2c273a657483e0", + "evidence_path": "evidence/20260721T150156Z-42d85956/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:661427dc0b778a4e4280c2dff26992f5a1079906c71d63e701d33696ae533ef4", + "evidence_path": "evidence/20260721T150156Z-42d85956/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:197acc13204195dfebea306da77bf3123a95e7fbe9b809d29400db4e17815504", + "evidence_path": "evidence/20260721T150156Z-42d85956/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4ffc5101a2c5090b46beb167a6db50692f80017d4e09c76979ddf9b38f3f4436", + "evidence_path": "evidence/20260721T150156Z-42d85956/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f89a5eee4b82f2ae2d57d29357f7f4bcc97094fad933220414027ec67ab55f3e", + "evidence_path": "evidence/20260721T150156Z-42d85956/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:47a185a62e78211bc0ee3103a8183477647a5ae60d84736bc42e8f2a8dce8b43", + "evidence_path": "evidence/20260721T150156Z-42d85956/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b3dd52c9ffa0e9a5649fd723c212ee5a477c1b0744c35fd7850885c1c985f0a6", + "evidence_path": "evidence/20260721T150156Z-42d85956/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:36d959523c554b8a12200f9ea0f64d28e66f0aac1765320d7749c45b46622e03", + "evidence_path": "evidence/20260721T150156Z-42d85956/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:35464ed3bbd4ab27e646a3ff247ee951636722f318ca90727222de0c29f131f5", + "evidence_path": "evidence/20260721T150156Z-42d85956/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2c50219bab3aab74f4ebcdcfe9176d798450234485005c26a414cf97916cc780", + "evidence_path": "evidence/20260721T150156Z-42d85956/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3dbd7c3e2a09713e61a74a8ce73b8414bf4051f280334c46eeb852030665044e", + "evidence_path": "evidence/20260721T150156Z-42d85956/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb831ba313713d52cbe0ac6d58545684cdda21e6b46d483703b6a7834d2639cd", + "evidence_path": "evidence/20260721T150156Z-42d85956/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:25713a4be3b4abf953f0d4178ec150a5ace53d8b892bb97b6c7c1a26f5fdf380", + "evidence_path": "evidence/20260721T150156Z-42d85956/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2b9ff9b35c0397996b6e5662d525d024332c0b4392af033bf9fb18a2e18d10bf", + "evidence_path": "evidence/20260721T150156Z-42d85956/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0a7eaafa6cd35b6a500e1883040faa15d4d6e72b32395d18cc69bd376f10bcc2", + "evidence_path": "evidence/20260721T150156Z-42d85956/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5c6a5d61fb7fad6255d5b5a9b185c1c1dca81d6cb88dc87094ae0aeee96ec5d7", + "evidence_path": "evidence/20260721T150156Z-42d85956/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a435704a1b581c68f83d5960eb38fb85e198c5e2ff3bf5ea41e79e30b16c72ef", + "evidence_path": "evidence/20260721T150156Z-42d85956/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b5684536b2bafd16fe30eb7d723cc2089aa846818fff9df47a6fef9cdaaf9a25", + "evidence_path": "evidence/20260721T150156Z-42d85956/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fdcde49523752d6765aa55aa965458206939b7289fc00ed71b9f8d6032bda8ac", + "evidence_path": "evidence/20260721T150156Z-42d85956/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:437fbf0edf7fa753e93042fb62b1439fd7527edc405beafbc4c256833cc85653", + "evidence_path": "evidence/20260721T150156Z-42d85956/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b6ce11250af13251b45bc8285ad74c293392e9101741626ea7b8d958305c1f30", + "evidence_path": "evidence/20260721T150156Z-42d85956/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:41d2c42399e6d03d1357fa758789bf90cd53ab1c9035e442b21d2db1d63b3ab9", + "evidence_path": "evidence/20260721T150156Z-42d85956/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:55f590cba802aa2421bd20b48525da1af558a140b33dcc168b0388d12c356720", + "evidence_path": "evidence/20260721T150156Z-42d85956/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:84bdbb14c96ba82e78a61d1d39f28e8c34c45704e8be741c9065119e5c30115b", + "evidence_path": "evidence/20260721T150156Z-42d85956/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dc574c142c592a0da0e72bc2e6dff539c10ae900172af1885696bce68a6c3dbf", + "evidence_path": "evidence/20260721T150156Z-42d85956/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:89a3813edb0c123f39b2a4346d03d9f20eff01351feab2a62a5d5105aea3e535", + "evidence_path": "evidence/20260721T150156Z-42d85956/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b49a436d4673a8355623e1bd3e0a11152889c35d90e0695d53db76d6526b6808", + "evidence_path": "evidence/20260721T150156Z-42d85956/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e0736b0391d9858b7fe966f10b22cd136684b05d5501fac0d93e8d37705c126f", + "evidence_path": "evidence/20260721T150156Z-42d85956/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 47, + "failed": 3, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed5.json b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed5.json new file mode 100644 index 0000000..8872b43 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-qwen2.5-7b-instruct-q8_0-seed5.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T150237Z-08dd2021", + "timestamp": "2026-07-21T15:02:37Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q8_0", + "declared_quant": "Q8_0", + "server": { + "preset_name": "llamacpp", + "reported_version": "b10050-b15ca938a", + "quirk_flags": [ + "grammar_constrained_decoding" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.7, + "top_p": 1.0, + "seed": 5, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3fb9522a300bf9f4a5a0caa167d989dc8de11760efdcfc18f826c6e24d40fe7e", + "evidence_path": "evidence/20260721T150237Z-08dd2021/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:595261f7af15e9e554015c3e05e2df480abf818f5937bc178c36e5675ad57ef3", + "evidence_path": "evidence/20260721T150237Z-08dd2021/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'create_order'", + "evidence_hash": "sha256:299a32c10e445822c60a4519b254bcea0ba553493d6829c8678fce7fa5704488", + "evidence_path": "evidence/20260721T150237Z-08dd2021/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:62f17718f65c1f97dfad38e2c34e888b724e43fe6992d40f979e51ce5fd1629f", + "evidence_path": "evidence/20260721T150237Z-08dd2021/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e95e1989fbe61cb009f07235e05fe53e6c7ec8fdefe2b008e2a81fda1134db4e", + "evidence_path": "evidence/20260721T150237Z-08dd2021/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:1e6d0dbcd3f9efedb313809f579d375a9d1e1683eaa627ab21c0c511533b764c", + "evidence_path": "evidence/20260721T150237Z-08dd2021/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7af3e93b7240875fc91fe7fb2500c0c59f2c7d434210e352fae206e84764d608", + "evidence_path": "evidence/20260721T150237Z-08dd2021/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d7ce331b3d2a998b134d31341fb4e4dcdd7563d970f952e3e6cf49affb7d8f00", + "evidence_path": "evidence/20260721T150237Z-08dd2021/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dcd05442d1be94da96208aa158ac4cb164761e447db17cf7cf3dbb36fec8c04e", + "evidence_path": "evidence/20260721T150237Z-08dd2021/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6be79e94a0152025380ee1f8c183262740546d29c141be2037b45ea996e314dc", + "evidence_path": "evidence/20260721T150237Z-08dd2021/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c4db9da69aeb423bb6bc8955906667c8b5c9a15c7667d9551bd54d76bbe84d75", + "evidence_path": "evidence/20260721T150237Z-08dd2021/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-", + "evidence_hash": "sha256:858e30237427733bc5e739e10f3768fd6b6fb946e950e1084a5034570b459f42", + "evidence_path": "evidence/20260721T150237Z-08dd2021/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:198c80e27ee98ce53056c06251439f86e715099c64c284364ecc97451ef6608e", + "evidence_path": "evidence/20260721T150237Z-08dd2021/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:141c35442e296f46f3e7df4edef33e3d35fd0508905317dca1d156dd22d06928", + "evidence_path": "evidence/20260721T150237Z-08dd2021/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:caac4f90b8a56db86077fe65ccdf71f4bde7d5dc546f32438a219632404a7fdf", + "evidence_path": "evidence/20260721T150237Z-08dd2021/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:07c209a72d610f1cb11b4023e97be08be6665cfa5322cb2ccd6c11d74af38002", + "evidence_path": "evidence/20260721T150237Z-08dd2021/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:569a010200058eff1059055df81048b5873bc64b717c3684546970c971bdfe92", + "evidence_path": "evidence/20260721T150237Z-08dd2021/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:545b5ca56bfc017874d1b4336e7409ba279c445face25031389169875f8609f1", + "evidence_path": "evidence/20260721T150237Z-08dd2021/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a763451bb0e26e0714718c8ddbe60c12928be7f557fd7194b7fca38ad8aaae72", + "evidence_path": "evidence/20260721T150237Z-08dd2021/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9a26f21b9cce4d80f7cec75ae1cc1cb4f661925a2899019ea32a07b17019a18e", + "evidence_path": "evidence/20260721T150237Z-08dd2021/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c6b3d688b177964fd8127a88a6c912fcce6a85a31a5273411c567d28903ebddd", + "evidence_path": "evidence/20260721T150237Z-08dd2021/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:917b24e94b2c05dca11779e4161a5ae03f06b844a16bdde941cd07babc678159", + "evidence_path": "evidence/20260721T150237Z-08dd2021/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:85ea8c03d2914a02709b09fde6e99d396b2dc2fb678ed41741e6132fe8a399b0", + "evidence_path": "evidence/20260721T150237Z-08dd2021/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:261ab1a580ab1581da057a2a9c84335a49cccccfdf089ba0f2dc209eb8cb8309", + "evidence_path": "evidence/20260721T150237Z-08dd2021/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:28bcbf0f8e534d92cd4ed7caa3c3390552306cf2fbeb6606fbbfeb508de912c8", + "evidence_path": "evidence/20260721T150237Z-08dd2021/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:55b63acb617e7722e1ed34e168ce94ef47d521563098de73ff7ea61209b45ca0", + "evidence_path": "evidence/20260721T150237Z-08dd2021/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d172ca6b10d5fe02d4de3aca35198b583787cfb60634ecf09e5fa4a4c565e97f", + "evidence_path": "evidence/20260721T150237Z-08dd2021/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5554a61cf2bef9dfa135acbf56cbdfb11e8b6cd37e3a3be26b00c579bab98d86", + "evidence_path": "evidence/20260721T150237Z-08dd2021/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:905935a8f9c0bc1772d5b6082d49f89b77333ccc1a6395be785d372c6df8a63d", + "evidence_path": "evidence/20260721T150237Z-08dd2021/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a22aa5089ad55659e36edb608cd155c44dea5cdc46bcda5df7985be58a916ef6", + "evidence_path": "evidence/20260721T150237Z-08dd2021/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:23f5760ceefd87e1e364014c6af1b8d49448d977d4f2b8a927812b17b948fa95", + "evidence_path": "evidence/20260721T150237Z-08dd2021/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ef38a0d8e9c2bf603553c7df1e9b194ebb4337756c347c964b4df7ebece4f138", + "evidence_path": "evidence/20260721T150237Z-08dd2021/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:444b99208a8e1ba8f4b6ff8dd9cbced8f2d9b7b6c109329456209e5d61a39264", + "evidence_path": "evidence/20260721T150237Z-08dd2021/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e81088e97d7e44c0bf90163a1a606998bd2e0ed297bbfda8880468762e6739ae", + "evidence_path": "evidence/20260721T150237Z-08dd2021/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0196991b0e63767b09ef06f0d3d0509268e41fd54719fd02230212635ea4751e", + "evidence_path": "evidence/20260721T150237Z-08dd2021/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:27888ea0c874e68233f0675453d7bd048ce71ab511e79dfc49d5b978de99c94a", + "evidence_path": "evidence/20260721T150237Z-08dd2021/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6c644a5ab725e89bace39a107ad7f89a0e77fd67506950a3d35e62e9919acd6c", + "evidence_path": "evidence/20260721T150237Z-08dd2021/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c72966dbdc1db092e90595d75b8888d321915cd6b82a9dbc77ce3b5049e5c853", + "evidence_path": "evidence/20260721T150237Z-08dd2021/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f74a52f70ec3d0397e4c9fe30e6360137285a9a1e71a759749dd7e92a84d2fc1", + "evidence_path": "evidence/20260721T150237Z-08dd2021/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:98665743af5a34c15f3fab8b2b8c096981d5d91b73fd37a10f63de9ca2f5b17b", + "evidence_path": "evidence/20260721T150237Z-08dd2021/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f032bf2f6d3362a2da7fbfd4a44c0bb73fadfefa7e5b9fab9dfa010148d0efb0", + "evidence_path": "evidence/20260721T150237Z-08dd2021/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8cccb0db710e098bf0aabe1c65b98e4cdf4e559e30b33efd5f29ccc0d0c6e554", + "evidence_path": "evidence/20260721T150237Z-08dd2021/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9164824c0cd55ed3cb5f57f66f248e9aab4bae87947081b18383b5a929e5c04e", + "evidence_path": "evidence/20260721T150237Z-08dd2021/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a14b1c740a24c658c47b9f6cf6fa5441078fc0c37a6ae7f88810e716bc4c3af0", + "evidence_path": "evidence/20260721T150237Z-08dd2021/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b97b9d8787485d773df874fb70453088ada43317fbf21e64616a1b9bc8858ee2", + "evidence_path": "evidence/20260721T150237Z-08dd2021/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:08be857378f24ef61f4cf1798aeae45aab8b9b9fc728bb1396ccb800ef1dc1dc", + "evidence_path": "evidence/20260721T150237Z-08dd2021/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:88da171a381b076567570ec029aa7904c966b2810213b7f4251f451a3ba54599", + "evidence_path": "evidence/20260721T150237Z-08dd2021/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a12671fc41b4fba7d50605b25af29a6f75f949951aa554b5f0ac5fc95c7a61c8", + "evidence_path": "evidence/20260721T150237Z-08dd2021/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:41f95b96231ff67b5bff3ef33df7aa233a89316044e2675b14987bf4ee58f3cf", + "evidence_path": "evidence/20260721T150237Z-08dd2021/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dcde89a36d09bf8db4fdee4f36c27e31ff93ec2f0db545e7b810944c4ab9158f", + "evidence_path": "evidence/20260721T150237Z-08dd2021/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 46, + "failed": 4, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-armS/llamacpp-version.txt b/docs/case-studies/evidence/replication/m6-armS/llamacpp-version.txt new file mode 100644 index 0000000..9a47462 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-armS/llamacpp-version.txt @@ -0,0 +1,2 @@ +version: 10050 (b15ca938a) +built with AppleClang 21.0.0.21000099 for Darwin arm64 diff --git a/docs/case-studies/evidence/replication/m6-mlx-repl/mlx-lm-version.txt b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx-lm-version.txt new file mode 100644 index 0000000..cf6f7e6 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx-lm-version.txt @@ -0,0 +1 @@ +Version: 0.31.3 diff --git a/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run1.json b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run1.json new file mode 100644 index 0000000..ab3b4ba --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run1.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T160825Z-a75b8e55", + "timestamp": "2026-07-21T16:08:25Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8081/v1", + "model_id": "mlx-community/Qwen2.5-7B-Instruct-4bit", + "declared_quant": "4bit", + "server": { + "preset_name": "mlx_lm", + "reported_version": null, + "quirk_flags": [ + "unconstrained_post_hoc_parse" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7008e84fab503f9864677cbbd21883bc6c1cb5d7c073e3ad697be61f22f4f6fe", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ed65031feda51b515fddf60e6347eb0c5939565afb2ab7fac17d1d046ec5dd0d", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cd44372a29dbe384806e5cd6a002458e0cb61e9bea6f19097cbf68e46a34149a", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:77dce3d46c45bc85bee8c8af4b2117ba87da4a2a078fa24cc899fcd1f9afb17e", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:fc38cc721d3a1a34c99dfaf3ba9b4cc87798f8630d99157410dd185be76db225", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:9ee58aa3553546b4cf60b7b7973151d975b68b8a648ff2d6a86a2a989d8c29a2", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0e1aed61d20ce83b312bf1e3d5347801bdbc86627ced8695a8016be56b2f351f", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e7f691202890d4676431354dff4119dbee478e7f00e862cdd66af76825e901a2", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:96d0f11105802b9ac7ed801807e091c0cb61f597a2c65e2bc3d6dfa258b29ab6", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c58e90129fd596a9a646caace59004fc01c2e72a10e19af59ca0be79ff0e40ff", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:02acce78c1fc6d8963df0c4a5edfbacf775d73a5b9b8e36c75bcea1f6e40d142", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-", + "evidence_hash": "sha256:5bce431828255995d186dccd97fdac8e16e1f28f730b7e2b436095fa1384ff6f", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:68e6e24185dbb890909989cf81235c7a00572b4de05e6e45aefa2cf449c07c75", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:38d40c9f8f6d09bfea447553305ad0c450b49bb15e98db20a23d58fecae55076", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:52d29054e8f9cd15c5a83d18c8c7d683a75ae933d78257676dddb2fcf52e1250", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:37d434323c4a19f51ce21f6d344d07afd7aa0921999cf271cc5027dc1324f19e", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f69c2197202ea85900f6ba64591f1e77ccec07e0783a4f4cf573effa33473e19", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2a2809e85bad0bbe99ac5bcb3950f6ad08e2c6a70272c8a207583ec1e2f8c3a6", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:27ce2fd680ef181beb4e80ecf2fa4edbda73419a0d9cd67e5ceabc438a454ee8", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0edcc1e00faa61edb428be06b6aedd64efcd5011afc1427c56ed3e51cc6260e5", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:137993dfb2f7e63315fca790f994148477a10b40b685e32c8b62daa516458b80", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:750fe527e0dd55fee9f0acb16bacaaa432e300395354f6f119481afd516b831a", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7df0eb68b97d909c4a0fd6e03c6891dc50701958b04cd809c4968748feef660f", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9faaf6a689ac0df7d8d03b9b6260c41d805987b19b27f41b493ba089991afcde", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:392c45f622a2c89ac0c3a7182ad1eca0b56b8f65026616d58137eed2f2a8042d", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d3505dbacf4e1f33e611480ef4dd4dddea20ab3bca8c731411bbecbc3ddbc6fa", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c03e9ec4c310891345728b6f5e617f50b3146834b0e17abc425cf6586f187a05", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:197f7e16639d04c4a6db0a014b5f9fbe88e482a46c6a92a4d6d41146c908a78d", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0f1d4ec184c35b7e531a88226c54cfc31c89522409eabc8d637917ffd37cfef6", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5e3859c73a857858e31a6d4989caaf07633a24eb032f5e4653c9f3a6a0548785", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c74ef86e8b885946dc59ba19b20e89fc8cb61ed49715fbda0aca0797e86955dc", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5f4fb725ef3c1ba01691ff236087f3cee8657158afe7b5304a0992097f82afb7", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:afa225f032b82fb0c7c23715a4558c587c9c99dff57ace5bd1ce9e6041b4f28c", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f8cb523ee7deeaa1265f8b5a1cbfecb467de898154ae5bc31691b3a4a203ea38", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e26d88a2aefb5680b6d5d7d655088197f81648037e557b01727cebea83da3ac3", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:eb4a34f25449dae4dc3de7fc0bf94035cb116bea0a83b564bc2fc152c35215cf", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8021f516d88b2d9538be39af5aa4619d51dc24e35c17a682160bee2bb09d9a52", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a568a73d1f5875ea0fd9aa7d876c5e5c2e5fc1245937a46fee7189c9c5528cc3", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fdbb0b55d46e02ba2ea4bef7c4145997aaea005ea880dce905c8d12837cf314c", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7d900d29854836ea0027da1e159bfa4c27a60020ce3e3c3efa77343dba93a5db", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:48da643005fac14f91dfa872f31ed9f179ce41d1c12c69cd79e0d12f9a930ed1", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:47d195ad34827a5395c758ace06adb26cde6f74341e874e2708b33b59f05ead8", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:630141acff6dc4b5dff84ffe58e18fb42fb858768b0d640e070b32dff81a5cb8", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d2f0de8e3544956d8e131ff965f7db5a3be3c6335edaa3ca04db835c0196ae7e", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a369f75cf35f6107a0991ebca9ea934ffa67ffcc5ec269f1d134c191c53482f0", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a0c314e24afc21bdf61c5bf5ef19ba56bf9bc41da686880181fdc9b7ed885328", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e2553760af64717807202a8928bec00c5a727829bf5c98c91e6df85a038502ca", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5a79d16640411856f3ae1733543fe17b7376683f091caf093071a11c5a6b2f56", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3146f52cadf6e9a256c8fbd705bd3bbad4bbb0fb3d8b34d982ac688e228c260a", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bbebb4a5dba3d36dd87f274829a8dd24455838d140e5285ba93a38aa6f9f5e85", + "evidence_path": "evidence/20260721T160825Z-a75b8e55/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 46, + "failed": 4, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run2.json b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run2.json new file mode 100644 index 0000000..d2fac42 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run2.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T160900Z-834d2ba3", + "timestamp": "2026-07-21T16:09:00Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8081/v1", + "model_id": "mlx-community/Qwen2.5-7B-Instruct-4bit", + "declared_quant": "4bit", + "server": { + "preset_name": "mlx_lm", + "reported_version": null, + "quirk_flags": [ + "unconstrained_post_hoc_parse" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6c82eca269492a4a23e5d666232b347177d349658c423762ee971671a73f6d22", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cee36232d3a2e12ca44d41cf512f74324895007ba7509407169009cb7e518d1a", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f178640c6b048f2f7041c86fa9e615f9acc30473163b0473d207d0c0f8f87aa1", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:5711b6c992ff0a9931344bbac74e00be51b0258cdddce0d510287b81877b9334", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:3465ec35c6d6b37db4b022bc63fa7ead917fcc9d1f466b218be3ee2ddef95e65", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:89cbf2b73eb1bde69a07b5935c860b1ec178e72091b3c478f23c20fc78f58855", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:dd4f8f9a6ea99f6ca39077d3fd053ba55ca40b8348828486557d1a5f8b1b6c0c", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6470a4cc53a1b6dc06b5cc422a0d94d4dca15614d98a954f06909cc6bc35fe78", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0a9c78c2f81cb5990745543b6b7951db772d54953831ab965c18c7d4d9224594", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8d94fb933550f945f3fa74822686d05806ddbbe5d7ad1007a6e588b7f792b6d2", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c1d87df01d7c3a7f02b30b32de4b4b0d2ee242aca8656ff0de083db8e8701d25", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-", + "evidence_hash": "sha256:70d189a715e39eafec25e230113fc5f07cde8cf74bd913be21335e4ba9c315bc", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:942d7adfceb22f94b27171b0d1c0d2b3b39ee16207350b34a49f164fe488c567", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4902d453e578eb23712c0346008fe64c7c30f8a1b3ae8043a92454d35c6acf65", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3bb41b0ebb5f3401d226b0b34c5db2930f88d13189a77fbb5c366621a9ca8c6e", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:70ffe3d889e40b82ff379951570a4d4bcba73cff81d4f8c701c6a067f76483de", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:76445d80755f34f82ab76cf1fba59c550c59c680253038e20c028cf1fe75c018", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5205d2edeb2e98dca141c86ba5e6706c6028d708b3e2644cbdb26d27941ddafa", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:593aeaec60bb9f9791572f5cda1651bed743c7413637196a4b0d523bd43d8fd2", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:42d1bc4b9bb479d77378f0fe6a79d0a8adcd6b7c3d1d4d69bf85ef5bbc91d7a1", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bf4e112c7b5ccab9a3bda514eae32e250f4f1a3603afc3053418c9623fbf0024", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d69d555df4354442feb89f529face09ccf00d257c44454258b266e1fea979b37", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f3b7c7bb4e1f7094b908c374953548974b739f33aac1fb28f0a3518d19370bfd", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:870d15068fe7c988e652ec5db00f93a219c79cc263557f586fcd68cb4edd786d", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6cd79d60278976b23d7d9cedc4e71cb5e71635d550501e931a25f52e124ed1c3", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95cfcd9217ee7ac2d12963e85d3cd0123dda59ed54276977ceacf2341afdec41", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ba330e2fe3e7008e4505467a5e15ad81e516e5cc6bf902e82255cdaaab06d6ba", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a8c44c532d8ac9df0aa9f244cd3a91643db299a658d68e80b00f374f63fcb946", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3f040e0d536ca269ceee686b4b0c11466c5c18fa3bb67a9b83cc5ffc5d9f922b", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cb13805b1af29dad0c471d1ad891a3b90fa128c31587a1cef5c2ead0c4897d35", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9974dbdabcf95f61f6a7eda1fdf0ff275c884eba4bd7fe3690d4b299ba58d167", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:00e9935a960786a184e7b217c2ff8c20a647a7b290cf6c6d1ef4f148d1e95249", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:89e2d75c9de2c0bb81441864c6b6b5cf5aa2152e5d212f7044a511edca594201", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:261baff085fbb7ea515ac73bdf7b3ad7c3c24bd4d2ad0829ab52443654d41d32", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9aebe720ab206cf404379f9986c6841b82db13463b363aa95083f75ac5bceb5a", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f9df48d78276ee4b7433dfb65a6180755493c51932ec0f707e109844adb72cbd", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e3841f8544fd66ece5f60b95131efb781d51a39ce51d291828eae06701b3220f", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7ad2a1c2ae60de5a97afc599729ab072df3e0fb6511b5c515d6c20bf2a1c7979", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:31476ab6575a875c19a0839acf74550d23dccb5627bfeb370be6308414e39084", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f3573f9e32d844ce79cb6d06e8f53cdf9ee13d157f701b208374eaee2350fb34", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de22dd5d55cbcf63924122b9d8ff7d175f8b56f1acfd2508e22985f7bae1fb1a", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e99cfff9dc89d9825d020ae82a486c8bd0655a8ff5edb85653a5f69ed90a31b4", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:95e29c4d74c1c3b141c02454dd9ab75b35c42a517b311f3f8c117190da327f6d", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8c8cbd9b18b1ddc28a7da6f5bc96ae7eed87ead4e2156c5a51dcc4b30e00b7bc", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9a1db4159d7f11b44bfa5be2392781b67b8ea9ddc071756a002f2c24cf2fe664", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d2713cc4dcd5eb540f7e0407d66d714ffa3305e4a0e41b67b523cc03e1eff3cb", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ba99b4e31d81864a389660c675edd3bcb7f857af4ee7c13def42018d3638dc09", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f99ca48609ae6d945ff526c8894aeea0ba6b221a4e136aa9126f0c48fe63b9ec", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4fef44e019c6d77cd6bc568db59b0ea042193d2338a72065f00adee7fc50376f", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7353eb1ea478e628719ae01919e10810d7b172d6daefc387e7494bd5cb41e674", + "evidence_path": "evidence/20260721T160900Z-834d2ba3/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 46, + "failed": 4, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run3.json b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run3.json new file mode 100644 index 0000000..820d860 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run3.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T160935Z-ef5c2b40", + "timestamp": "2026-07-21T16:09:35Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8081/v1", + "model_id": "mlx-community/Qwen2.5-7B-Instruct-4bit", + "declared_quant": "4bit", + "server": { + "preset_name": "mlx_lm", + "reported_version": null, + "quirk_flags": [ + "unconstrained_post_hoc_parse" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c1976fd55bdff195b2c9ef57a71bd2f8b09d916db47750c71f521b012c584281", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8e0e32c87bc75730f0390b78f488d8558ee63103032d2458978411bb37366473", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2aaabdf7cc3e512c411fce16cc17b2504063aca1a91225d88693fc735d52738d", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:8b759e6b3106fc3a6c0b4eb2708f6f20a2dd07423aa9a0cde5626305ef423f39", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:a3e51f00d5e0bb7e6fa66b9cafe39f7f6b6e9223a670a2b8af731f17556e95c7", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:a86d251b7ef0b077d70c3433b1d084dd951d06b56120a21a3c19dde065e5d164", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d54359ffde44b75ceb4f6950f7f94b795a1e2b8eca8c405d885aaab156f4d97e", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6a877edcaa1a03a547f8971d8261031cad4fc6eed60f1ab3d9c91e8f3ea0b44b", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:028984c51a30b3f554c15f6298ac87dbf64802104371edef96d2578ff4fad108", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:03f89d35566e91194eddd8d010b578c4214977314afd481fe06b93efa07f2103", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8cf61be4efbb4540b95b15f24f770e5993844c63d95d864cdaed7da7cc55548e", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-", + "evidence_hash": "sha256:a58002e4fbfb4b3dfa2e6652279989f9d1975d1489dbfc54793e81fb98196552", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7d1cb2b817726cfbecb6106ceaa4136e95a88653631e1c9bf762259be9b592da", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2026e3590c2fbccb28b404fea66371c40f807128ca1ebd9fe75a0162d1cfe8b3", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e93720f2577aaab56f6644a8426f45dcf36c9f28471f795185138b0b33b208a3", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:45f50e322bac6e53b4f5708622f7a98faef2c60135523e1bbafcd62262cfb686", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b9690204a0bab678d598941bc3593e65acd1e6214450b4d0abb0222e649a9acd", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4a80c29197572d1a55ca7fbea706f79617e1df5f55aa9185db18caf2f5486d81", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:48c45a3809dfa9027f1f16ebe0d3cf14c0b16680ed4962dba8129c83bec6ccf6", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4d45411ee32869983f9ad0c7e96a8c08dda5e2b99c73fd3842f737dd0afac950", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:78478e5fe245d1afd3bc3a3d4829931a77653112eb98cde5fa0eb2c8ff92b38d", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:03b625670553421c4b4564b6e81bd448ab688f76a7f5480f5b1533ddeff2ff83", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0576fb37b69f74bc5741b41b0c0656531b69d8c7f266c86ecdf1d68e9c54a109", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7b7d179ed4be8a82613403e83113f0fecc4fdd39e3fa0e3a5e648288e557bf39", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f78dddf985d04206e4bdee5d43f772f1bd6be44bb776c769041b865fc70dd9e6", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:921c888a8eb24633e1cad5b1e6db3936b701a74af345b8fcaafc4e97ab4e88a9", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:942b50b4bc2fd0c470d8c9d477c21f565debdbbdc6a9c65bc2848d2bfcfd6cdd", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d4e05b9f55d665ca346e3a954753cef0c36f6b512e964edb0f2a5e40210772f9", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8eac1609e2d60c02f193419757b6c2659f8161bb25d083e6291afc1733f23b71", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d467fa951785e4c89f5ff6bc72151247113617908b49f604816891116966b598", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0bef4f22d9d7197ce96bc8818fbe56370262dc00d436d701bde8065bb79bba60", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de93b8faf2f3a80e62ea7c73f89bf63eb0e8426c6277f7d5f1fc44fa68ce639c", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d509e0ca934d5af2124f78e27602010e21ed98c954a92825f19702848075ad06", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3cf71ed2ae42266b8bc0a1ad00398e0f00d8d3741f5f191168ae45a6fc412380", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:31e5b53b1f87f950743057d4ddb7cb90eb51cce88540e8ee3e69b6de6d82bf62", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7c1b5436490343df8ad78f7ce480aa79143259dd3b4c457c2b1aed05339cd56b", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ed6b578c60fa2348b48e29985117090d7ca89d8992cb1097e5085bd2994dc6b9", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:509ba75445eb7ae2c19c1992670a8f0a129b94d92cd707071ce1338698d14cb0", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6117d6bc2fba3d970174418e453434344c177106d75545b38cc692a33c964d94", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f66ab3b7e6055538debd427d56eb7102a51621e63cb2e2c7d9b333da2d220b7e", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ffc89d8f3bb9b8dd540fee97292d00cd6cb27ff619cd1887a968dc8c95f44642", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a8729c897b1308d7475350bc3777791b90b638159a8deacfeca697d012aa7b0a", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a87cd46cfd32316f76298fa7b9b34fc2df217156543d10cd0b585278b433d1f5", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b72114dc8d0c666ae9e5621987bde89ca24dbb65d667513bbdc05d1d0b664af3", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4efae974b16fe0ada3741b4a5b0eaeae43bcabda0fb40dbee7702c03f3770930", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ad24d42c574960d12a62edf8fd9b70975f16f545039201c7f5788267135974c6", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:616251255932734fa2127a815fec6b6bf986cc6b6b272f54445bb41f3e57e0c2", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bc06a7a4202cdebcc0a1d28cd2839b857cc3b12e19b7139f0c80bd3369cd0f17", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b6e989425f57c8c65e2404034e354a89d714deea2e7e537636fac9d89495d2ad", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:af21f3feeb03da862d6a283b398b179f308f47d17cbf23eef5a433f74ec9e254", + "evidence_path": "evidence/20260721T160935Z-ef5c2b40/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 46, + "failed": 4, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run4.json b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run4.json new file mode 100644 index 0000000..3578ebc --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run4.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T161009Z-6a644b28", + "timestamp": "2026-07-21T16:10:09Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8081/v1", + "model_id": "mlx-community/Qwen2.5-7B-Instruct-4bit", + "declared_quant": "4bit", + "server": { + "preset_name": "mlx_lm", + "reported_version": null, + "quirk_flags": [ + "unconstrained_post_hoc_parse" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ae36a80398d39d45b1cf968834e7a70f2d2e75a1830a792b24b0dbcb56e2139b", + "evidence_path": "evidence/20260721T161009Z-6a644b28/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:745015644ee06deea32c640872d25fcfcafda8227dbf5b6aacf1938d088d3427", + "evidence_path": "evidence/20260721T161009Z-6a644b28/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7f1a59b86adaa41b5d3c0c35be9e182dd8c2663a0b416d74f14d8362ad7921cb", + "evidence_path": "evidence/20260721T161009Z-6a644b28/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:9020c137f159b59d0545c7132aa8947deb90290460d6a57f392dbf021499cfdb", + "evidence_path": "evidence/20260721T161009Z-6a644b28/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:8a0bb1b547b2aa4337c47accb9eaf217d9aa59f5987b35de240729d92fe9ef61", + "evidence_path": "evidence/20260721T161009Z-6a644b28/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:6fa0c7a002196f446de20c02292fab40eee47b3c08a74a4a363a7135c7bfbe01", + "evidence_path": "evidence/20260721T161009Z-6a644b28/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:795a1d10a906a603a506832b83c2786aab0a4fedf864ac4a90c7f24bf9b65d9a", + "evidence_path": "evidence/20260721T161009Z-6a644b28/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7940a6ad0ee508eae0e156bd4b344f7cc886f3dca135055e59b5af2f64e89a34", + "evidence_path": "evidence/20260721T161009Z-6a644b28/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:5e02314eba47ee375adce9c444cf2e9e0ad1097cac53febe5a469f9fe5672083", + "evidence_path": "evidence/20260721T161009Z-6a644b28/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:14d782a15be8be008f4dc17cec56a8edeec24e7bdebc884cc39d08f1ceccf3b6", + "evidence_path": "evidence/20260721T161009Z-6a644b28/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2ceaa4e16163fecff52610d9f1de543f832a8f6fd8352e5fe3f8bc9b7dbe35f3", + "evidence_path": "evidence/20260721T161009Z-6a644b28/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-", + "evidence_hash": "sha256:a553ad0aad24a7713654fc8b04a501a59f5934e1d901067727a6c1a2484ad014", + "evidence_path": "evidence/20260721T161009Z-6a644b28/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6730c49f83c16dcb2857f3e606321a6970da6ff91f57757cdb016fadb8a0c889", + "evidence_path": "evidence/20260721T161009Z-6a644b28/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1eb5ec6b3dd76e54fdec660fdfbe83144314c7d924b3f4b0c24173051978abea", + "evidence_path": "evidence/20260721T161009Z-6a644b28/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:68dd3d8c0d873150e9246db9b9ab48f3c2742fe28ffef7cb7b32a76a3cab13c5", + "evidence_path": "evidence/20260721T161009Z-6a644b28/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1b3ded65f25c2998d59c99090508319f567ede970ae43053a7c926c67fb58372", + "evidence_path": "evidence/20260721T161009Z-6a644b28/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ed3bbe24be54f207cec77a77f4756b075855d859878261f285f721fdc04cf252", + "evidence_path": "evidence/20260721T161009Z-6a644b28/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:07fecce219276ff67507f3374716850c1381f31355efd49d3253f01cb7c84e46", + "evidence_path": "evidence/20260721T161009Z-6a644b28/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2630c46cbe35b37adc725cd1cc10ba375337dd9fe9770640cd7a3094d691d253", + "evidence_path": "evidence/20260721T161009Z-6a644b28/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3258a49b6d7d0c78ff12c41ab5de434c0765aba6d4fb67161e43740a744e3e6e", + "evidence_path": "evidence/20260721T161009Z-6a644b28/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6a1e4a75d75f2353a7bd52451c7df1dd83e72dace5ada46fa7a6b3419b05b5bf", + "evidence_path": "evidence/20260721T161009Z-6a644b28/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ccc3aae3dde5d397c75e6b83e6b3aed566ba1a1b6d0d619561aa6369b59ee59a", + "evidence_path": "evidence/20260721T161009Z-6a644b28/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb8da0c9e29a271f80b5f8f4998430309ff47034d55663243e64338118f8cc74", + "evidence_path": "evidence/20260721T161009Z-6a644b28/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:26c625c038c8850ab65afe34fdd19267348c0931a4f87578266867987f4e56de", + "evidence_path": "evidence/20260721T161009Z-6a644b28/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1649fd537007054b11479f237ad4705f4cb1f21edc27edd64915a66daaf6269c", + "evidence_path": "evidence/20260721T161009Z-6a644b28/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce67dbd95e92c5f15239e781203562ef374a35086f50cb1a9d6cff1488150acf", + "evidence_path": "evidence/20260721T161009Z-6a644b28/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3244a66d8b7de6a49f9d5bab944853bd7583e0d3cec3713d31f8e5786ccfec55", + "evidence_path": "evidence/20260721T161009Z-6a644b28/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b6a9c53cd58c3f3480feba775d65d6fadfb2d6d7da0d2f011beded509bcb407a", + "evidence_path": "evidence/20260721T161009Z-6a644b28/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ce8b6297bc88b564d5acf288ae2106642677bc0a2ef65a5a1076d1e4c02db5ab", + "evidence_path": "evidence/20260721T161009Z-6a644b28/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0c7ffd86b2f00908b1d69ca57085adfab527a965c2d05018838255ac9889ced1", + "evidence_path": "evidence/20260721T161009Z-6a644b28/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:05c996cb7996271e0428bb857dbd673e4458ce401d1cbca55d9c87f7c82a3deb", + "evidence_path": "evidence/20260721T161009Z-6a644b28/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3c1b9c77c0a920efbaf58e0490bcb31eb07cd2d1d99873d137061bddc2309bf3", + "evidence_path": "evidence/20260721T161009Z-6a644b28/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8b4cac6eba3eb9b1b0b4a376a9761e59f90ad3517df7fa5deae23809f79d13cf", + "evidence_path": "evidence/20260721T161009Z-6a644b28/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:65c3e93bea69cc59b79247ad8806c163ba6167d777cc822504c064fd50ace647", + "evidence_path": "evidence/20260721T161009Z-6a644b28/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8169eba62626c65f1d3d3ed57fa65bd197f1a42f9d192fb5240fbe781411374f", + "evidence_path": "evidence/20260721T161009Z-6a644b28/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e7ecae3c5a278ef3de3ee9e169750fe84e5c6e4dd0fab585bed1c7498d4b3515", + "evidence_path": "evidence/20260721T161009Z-6a644b28/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d5a9a896d10497e003a5b4569b63e7e40239e5c877b96263783a682b8d06b3af", + "evidence_path": "evidence/20260721T161009Z-6a644b28/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e9833700a10ebe1bbc8df0518ead763997b112b6cd6b7ae51a99fc02e550bc07", + "evidence_path": "evidence/20260721T161009Z-6a644b28/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:68d28926eef49f10b892617c93b2df2d64dd85fbf3662a14ebb4f426d13b8d00", + "evidence_path": "evidence/20260721T161009Z-6a644b28/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:86244b1c75d96d10ff02c0b2e58e6d59ccdae89d6c9e85a9a7b5ffe050c49690", + "evidence_path": "evidence/20260721T161009Z-6a644b28/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bdfea30966e78d7af900230bfcea218bcfcb5ff7516b5f8bb8ffb941ad9b9dac", + "evidence_path": "evidence/20260721T161009Z-6a644b28/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7d9a48a04039d9c5629b680155aee05a27b3187232732eb6822b2a88697e7c8b", + "evidence_path": "evidence/20260721T161009Z-6a644b28/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:19d627d9033d932832eb759481553a208757243a44350af8cdee9a8cf11c7535", + "evidence_path": "evidence/20260721T161009Z-6a644b28/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6a8ff68f95194b2521deb07b6ad7ead04b4b6c1692d8ada8a39848dc5ac5fa88", + "evidence_path": "evidence/20260721T161009Z-6a644b28/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:407d94d594dc642a59ea8e2a35202b98ea209b0c0a1dde3d77b42d94b8f234bc", + "evidence_path": "evidence/20260721T161009Z-6a644b28/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:005d6063059b4f8a6aa08340d76c275d95127545bb281a63aae3d7363a29495f", + "evidence_path": "evidence/20260721T161009Z-6a644b28/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cdfffb644e54d1eb2b742a43549e46840f94b17e7d7b9b7e2f79b51979f3124a", + "evidence_path": "evidence/20260721T161009Z-6a644b28/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ca0aa5ca382e4a48d61a57a3f8fe03f6c9e2725843f48880e9272726adc44931", + "evidence_path": "evidence/20260721T161009Z-6a644b28/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:52782c25d9668accdd94ad5bafcc80339b4ab30b88cd431d2b3b2a2343ef6b66", + "evidence_path": "evidence/20260721T161009Z-6a644b28/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fb9dc5044f3131abf6eabf4c346f79a0ac789ed344f87a765973fa08a8d5512d", + "evidence_path": "evidence/20260721T161009Z-6a644b28/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 46, + "failed": 4, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run5.json b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run5.json new file mode 100644 index 0000000..b03ea01 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-4bit-run5.json @@ -0,0 +1,487 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T161044Z-744fdd7d", + "timestamp": "2026-07-21T16:10:44Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8081/v1", + "model_id": "mlx-community/Qwen2.5-7B-Instruct-4bit", + "declared_quant": "4bit", + "server": { + "preset_name": "mlx_lm", + "reported_version": null, + "quirk_flags": [ + "unconstrained_post_hoc_parse" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a421a0052fc4a7f2e52fbd0ba2cdde81a2f314896b5b21ec72f5b5c43401d82d", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3ef6be5348ba98b305230ef9c2bb61286fafe740da6cd9a98ebfd1766173e21d", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:21b32f0877865592fc6e2317697f7e64251169aa2f3ef5ee832f3e465b09f759", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'compare_temperatures'", + "evidence_hash": "sha256:b9396b9d282a4e95abbbf77e2632c3896b5031a71fd827018e34984565446d6a", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 2: no tool call emitted", + "evidence_hash": "sha256:43d95beea59c603abc43af217d554eb65aa9a273dbf255e20b265a163da38ca5", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: unexpected extra tool call: 'set_ticket_priority'", + "evidence_hash": "sha256:e8a2f9e385782d0d7af9bc5477de5a1a2d931a6973c3827d91965c38d54f5806", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1041527573c4b7a018e1759b752eaaf8b7ae1a350eabc035d73f54703d8ce2da", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:62460d95a8093c9aa02bbf536795dd9daba351b8db15f0d97944dd47f57c440f", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e07b3237a62fc142216cffcb10051f00bcc8176de94a870d772a8d66aacc5203", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b2590ec0e02f5ffeeca44040f2df7eadd7c78a0327aa3ec05fa2abc89f631fad", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d3a17ad04b61c3ddbcbf4e9a7d3f923a7b95cc8b454429b3072a5a553a41ecdd", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-", + "evidence_hash": "sha256:01b80d0b9c783a5b1625121d9d0bdbfd4f2787ac9ee9a3a2d7dcb3b8d6a90aad", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:751307cf160968539e315588776c9a0b0b2f5b02afe8de528067640db40e6f41", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:65fc42326cbe3b4860e81871123af4d0023589d58e21907567dc8db20d712c8c", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ddd58e16516e256d4724621ae1ee273059fe4f3725fdeb5aa68aa8fecb2737c8", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:42c5527ab5fa5144c905be56989f1fef9303072ebcfc3b6f6e437490cbe47b9a", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0d488cdf1fca8f429d7a15b2103b8aae82a8f43516ee48a1b8cb0b134cdf7975", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:61e4b820130f96585e13e7045a24b26e8f0f31307c38e414b162072c9454360f", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cada4faa6d622cb0beee28c8293e3821a11bfdd6b66223cb78939cb9242171df", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f60776a6313ccaaabdf82d05aa24959c6de6a3ce639072725ec8e9563e68d5ee", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:285db3e05964cc88e58bdf3dee64e9d667d0ab05f824c39bb195ddbc56d3b103", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e1ee340caa133b5d96bf6dc21e0d5edf6116b6b7adedf594bb7e1935bd12e5a", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c09743f3c88ac11c00c574fd6eda646298e387aa2eb3b28cfbf20f68f7988db8", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8d033575ac74b8d73468fad05eab18c440a675e3f82c32cb4d1ed6c68931615e", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:97ba68a6611322a340260464dae26d58a5a48379511e9127729594a0f37ff591", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:0142dfbfaa108edd82a69c5a535cbd65a63d22688e61cb4ae057a47462b30d1a", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a34ad4d82cebdd7cdcf9654cb08d3ab78041be79d888cb1ae6060acaf55d2ce0", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7e0c96ddfbbfbbfe05936963994832aaa2184810d8263392379fb7f65c185aac", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:f3f3593138336cfa018f35fa4533135560246e009bedfdc2eda5917a0793ed68", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:72d6210cdf144efcd561ae3335273ce508caed8ab36632f51a523985844ef469", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:412c37d2a37ef846cfe828dfede6e1efb5bdec1785a07a6c461be2c74badba77", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3ce6204a164bae3e5af330a309bc20e0c19facaa988263567762e43a10a311b7", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8b535f1bf03888f261ee89bbba94751a4fe24ffc0dfeab0d108279beac9f2de9", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:871ddbc97457c1f75a9102fb974d1fea9ab97aada40c3125b9f83dfb71d529e0", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:34ac224f662f4e4ebacc10534b0a8a86e0fdfeff0993c839576c41625cee8c99", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:6f5bf8f231587ee5c78feafede5fc0294b3f73eeb57702e6f030ed6fa3589de5", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:23ac229494f1cc035bccb059990e677b1fabc9aaf2a6a12786e618b2237938cb", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d37c2e9adc49aa05abeb7feba89a0db8928b781f3fcc97a7e8bdab00350417c7", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:d86978b0918b7b55c468599bc50eb0f3da8e67151f9e7c33f11a9aa28a20f8bb", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1262847a70ed13b0114d8ed3ccc1b48a8612c74988f2a328aec324707439f885", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:09e047af74d443623ad43c114ec17bd4472b7e99626dbef741e529fb068d8b12", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c7d2879cacd57117c49c59a7c9f3927643ac94231560eaaf7a3329071c64867e", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8008cbf6b13618a7acb71d01d9368de69b6c1104c9047ac1df23db2224751fe5", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:cbc114027568b088d9484e3e581361e110af73ab817b53e8c0233e65104622bb", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:aad6ece38141ac4ae2d5e3234cc74fb4febc1aa104405c791796325ba696fc62", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:852f1a998ea6b5eaebb6d0d3586e1fcb5324df4f0ec3658313fb294995d2f089", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bb365a5f9f19aeb85320ae7c9fee31774f4534620aaca6f26a9277d8c5883f90", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:809b3b43122b2bbc709cdf2380864334fcb0484c533191cd8b82e19620f54074", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bdd33fe2f6a5989926c26858478551a6050bc57a718c113ba65a4046e11218f5", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7630006e26a944e5b7a69d27c0cb2d74f54ac0aa947ca9b6afa7dc696c4d9f3d", + "evidence_path": "evidence/20260721T161044Z-744fdd7d/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 46, + "failed": 4, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run1.json b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run1.json new file mode 100644 index 0000000..ba2213e --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run1.json @@ -0,0 +1,517 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T161139Z-63347520", + "timestamp": "2026-07-21T16:11:39Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8081/v1", + "model_id": "mlx-community/Qwen2.5-7B-Instruct-8bit", + "declared_quant": "8bit", + "server": { + "preset_name": "mlx_lm", + "reported_version": null, + "quirk_flags": [ + "unconstrained_post_hoc_parse" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:4823ad9326a3e7c4eeebe0235e17fcec18a2436724210c5edd2eb8f9a91d2765", + "evidence_path": "evidence/20260721T161139Z-63347520/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:096db8a60f60c02e69ebd65ab54aa6c37a773f4c0371eb8f37beafb65f37f4cf", + "evidence_path": "evidence/20260721T161139Z-63347520/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:f124e40d96816a3dff9665d21758da9efd461fbf1b6b1e54e027eb67b0e739c9", + "evidence_path": "evidence/20260721T161139Z-63347520/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:dfe23106c69bc29f9e92605dec50e301865f2cf9658d396a30129a6484c6194f", + "evidence_path": "evidence/20260721T161139Z-63347520/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:db1d7b3f03af6d984e8ee0fc04e698678ed13152b884677232cbb95cfa46301b", + "evidence_path": "evidence/20260721T161139Z-63347520/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:f11a7b373d31c8a9e61830a04603ce3659b76b9fa04d335efbd51b0a5a0f6f98", + "evidence_path": "evidence/20260721T161139Z-63347520/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:d57239fa8a715e4ffef30eed2a61493c25abe08456b7cb4117a0a85d34b0a216", + "evidence_path": "evidence/20260721T161139Z-63347520/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e605ffb828c96b203c6025b5584ce91d375da257c8b838f8d86d5e271432e332", + "evidence_path": "evidence/20260721T161139Z-63347520/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3e9aa2a93be177b7a0749cf6838dfd46d8cf19e346ed5992764889078eca37c0", + "evidence_path": "evidence/20260721T161139Z-63347520/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:88d9a31bce28fc5aa45a0215cb91e7fd3cea0ca0d13d706ca9220d0d6a358736", + "evidence_path": "evidence/20260721T161139Z-63347520/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:abb93466d287806c8d71b988b434259773a22ad0cb7cbd1664aab4583bc2b0bb", + "evidence_path": "evidence/20260721T161139Z-63347520/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:08af7dedb11c1bdbbffa529c55ca9f791d5d2ca81d585c000f94df836724d385", + "evidence_path": "evidence/20260721T161139Z-63347520/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:41e03601860a62e0992ba6d7fbeb1517e2750c35f50f10e45a51e5691588909e", + "evidence_path": "evidence/20260721T161139Z-63347520/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:b76d90bc0ab5c5c14c8c428759963dafb6a2ea73fea22619824989c2b99cc74f", + "evidence_path": "evidence/20260721T161139Z-63347520/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:7e180669369326fe8522dd50f91bd0d535b5d8efba0a71d2da1f4e40e42c86fe", + "evidence_path": "evidence/20260721T161139Z-63347520/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:fefa1563a54fe237930354fd465855e3c924be952612ad48a8b154bc023434cb", + "evidence_path": "evidence/20260721T161139Z-63347520/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:44283a7247da7909b51935b3367c0bce89e90c1901d5963850fcea89c40a0f14", + "evidence_path": "evidence/20260721T161139Z-63347520/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:36208ac9ce6d6e30e62725498b037fb439bb58b0872784d59c311fa85440fc3b", + "evidence_path": "evidence/20260721T161139Z-63347520/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:43c7af7dc8397f4ea9b807995ca7bb2401f5a8a7e0a9d4acdfb583ae1ce6b6ed", + "evidence_path": "evidence/20260721T161139Z-63347520/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:a68f992db46fd0c8a18c116c0cd5c753215a8eb22922be85cdd28c885d9d735d", + "evidence_path": "evidence/20260721T161139Z-63347520/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:403e803ee36dc6d4b30d416a15ff2d053f34a24fb60ec9de34478b50e776a949", + "evidence_path": "evidence/20260721T161139Z-63347520/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:d8969ad53f1a255570446fc0408e559eeb1fb350f45d766adff1f5620185cd54", + "evidence_path": "evidence/20260721T161139Z-63347520/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:5827115456033d70d2c56f6a9a658ec10d578d3340daa48bc47336e59f3cf0a0", + "evidence_path": "evidence/20260721T161139Z-63347520/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:7a722054f01452e70edbeb54e779301533ad56f6b952850f1cfaa89d4c2c1d10", + "evidence_path": "evidence/20260721T161139Z-63347520/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:9ba17286ac2a974b58e3981f1b4dc0afcef2801aabb63d4339dc9599b026ac06", + "evidence_path": "evidence/20260721T161139Z-63347520/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:7ecefedec5638c9011156d41011d3048cea811b088321ec196b6f1ee97800433", + "evidence_path": "evidence/20260721T161139Z-63347520/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:057e2f446063f8a0a1393a9259c682f0cbde9f98ac64af2eacef161dbf2d8e07", + "evidence_path": "evidence/20260721T161139Z-63347520/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:e3c1bb2fe1366b5cbd2ec65d655cd2a888730cec7dbed2ca0606099767270b74", + "evidence_path": "evidence/20260721T161139Z-63347520/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:bcb68ce7a0bd9b4f3d93e60312f914488ac119ac133bd2c741b4b704b768c737", + "evidence_path": "evidence/20260721T161139Z-63347520/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:8379a818e404bffb9f35c4603bfa654bfe24041f6eb52e8104d549269c154097", + "evidence_path": "evidence/20260721T161139Z-63347520/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:41b5423fef5eebc8d147d2a790e969eb6b1608ce613a4fd9cc7eaafc84f58a9f", + "evidence_path": "evidence/20260721T161139Z-63347520/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:779edaa0e7f4cd003e8c7475e14226e9e4dd6b8c9739ed587b446b01c3fd1d1d", + "evidence_path": "evidence/20260721T161139Z-63347520/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:ffc29486d28dcfcd0f782d79af0e9f8de02cfd3063ffbfa0c7d99f94fe3fe372", + "evidence_path": "evidence/20260721T161139Z-63347520/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:1c7a64538afaabca3fdb8099dee2c559c53beb099b318be76b7b2344b3eaee89", + "evidence_path": "evidence/20260721T161139Z-63347520/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:20cb85547365a56c7870ce5e8e204ab30b5d3c8291db14745150b87ea5c571ac", + "evidence_path": "evidence/20260721T161139Z-63347520/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:bdd41b1e588b08c7bf847479c631188b8b2d63593133b36fa681472a34f53c86", + "evidence_path": "evidence/20260721T161139Z-63347520/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:5719e15a26b8363daa4cd41547dcebd82e5bd617e5f92b1276b69330bb012b52", + "evidence_path": "evidence/20260721T161139Z-63347520/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:61e0a67c58238869b07f01d20d93a17940fdb5b91a5eeb86ed78a72fe6416b91", + "evidence_path": "evidence/20260721T161139Z-63347520/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:bce85e6ca90b67b609e67732630b1138b9751b705c1b8fdbaf1baf5490d8f0bc", + "evidence_path": "evidence/20260721T161139Z-63347520/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:237fc88391a32d3e638ab63a1088fa43af52d2a43f1d4064338d65f80ea173e7", + "evidence_path": "evidence/20260721T161139Z-63347520/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:b6702a65a4973e60505594d7e0198b009890dd75d506ca54940008a2cc61802a", + "evidence_path": "evidence/20260721T161139Z-63347520/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:d5f3fbc614070dfe8c604a009663e505344959a88698e5c1a20c975a26fa342c", + "evidence_path": "evidence/20260721T161139Z-63347520/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:1fecb9d382c84937f9169c4acb0b648a38eaf0ecb434fe9db4a9b39c9a0bdf8a", + "evidence_path": "evidence/20260721T161139Z-63347520/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:ed0fc9f7485f30862de69df19323cbec6c02f43a7235dd5b0ca265cafdc80575", + "evidence_path": "evidence/20260721T161139Z-63347520/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:4c774f3d86ce42c02dd5bdb2327bdfdac488fcd7aa1cdab38207ea9437428f73", + "evidence_path": "evidence/20260721T161139Z-63347520/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:4b6a72a601074c94f770f178326bc110a60daf935bc6641002363ced4434d063", + "evidence_path": "evidence/20260721T161139Z-63347520/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:1e6729a9e54c02a50262e2b19b7a198fe3c1b616921a936d9628337ae1a330fa", + "evidence_path": "evidence/20260721T161139Z-63347520/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:b1dea52f31f7ed5566b4b2d859f1e30dd5a3887eacff87530dc739ce0f8ea0a2", + "evidence_path": "evidence/20260721T161139Z-63347520/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:833e35aa810e028dd77493f78bf905b133a5e9d30465d0f34a2ce31cbe6ef45d", + "evidence_path": "evidence/20260721T161139Z-63347520/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:7ac7dafaf86d9e48e526089ed200ad17eecaed45ba107652e3b80367a52dc5e7", + "evidence_path": "evidence/20260721T161139Z-63347520/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run2.json b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run2.json new file mode 100644 index 0000000..301a2eb --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run2.json @@ -0,0 +1,517 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T161240Z-e1196347", + "timestamp": "2026-07-21T16:12:40Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8081/v1", + "model_id": "mlx-community/Qwen2.5-7B-Instruct-8bit", + "declared_quant": "8bit", + "server": { + "preset_name": "mlx_lm", + "reported_version": null, + "quirk_flags": [ + "unconstrained_post_hoc_parse" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:25d74616a4b773493a9c4d02542ce094382f39996311aef55ceeb71facc0f0c8", + "evidence_path": "evidence/20260721T161240Z-e1196347/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:a5875bb753d3e384b39de09517b1dd16ef7b16aafa7747fc8e72b256283fdc3e", + "evidence_path": "evidence/20260721T161240Z-e1196347/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:f2d4bd3b310e7a98925cab08188439b0e22b7eacc9e0a056e37d737656cd8597", + "evidence_path": "evidence/20260721T161240Z-e1196347/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:ad4ebe209c1a4e7ffa7428bcba9c18aeee28af405b252b6387fa3b902d146ba8", + "evidence_path": "evidence/20260721T161240Z-e1196347/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:4aeb5f55d6f41186ba32123a5f742478e2558b23ff3bae98c0ffabbe1f63ffbb", + "evidence_path": "evidence/20260721T161240Z-e1196347/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:72e900b540e7e843d0ab8f6421faf795014939d6550d15342ce1172a9ee2a328", + "evidence_path": "evidence/20260721T161240Z-e1196347/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:6796190e8a12b13063049ce1bb2e8541b45e9fc813c412b3fe78b223770224a8", + "evidence_path": "evidence/20260721T161240Z-e1196347/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:37b7750571ae5967e07170c57251a860f7223fefbbab13f71339c451f1b43644", + "evidence_path": "evidence/20260721T161240Z-e1196347/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fc9f10bd40537e8b2ce13cefa2bd98e44c1c02dc9144e3e2844ec34df5a2646d", + "evidence_path": "evidence/20260721T161240Z-e1196347/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:853ec4654d554056fb80041cb438b21e3ab087c473def415817cb06de0dc383a", + "evidence_path": "evidence/20260721T161240Z-e1196347/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:98fdb466e600fb5597a7c0893a4e311039b93b5e34a71bd0f2170a99102d8257", + "evidence_path": "evidence/20260721T161240Z-e1196347/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:de402b1c5336f1d8ea05f22d0fd4fb7ea00d95de28fe766d04371eb12a36ef62", + "evidence_path": "evidence/20260721T161240Z-e1196347/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3f7c860b9a3f481dbf64d1081b9ff1ad0647a039bcda767a4d22132b4e2e377a", + "evidence_path": "evidence/20260721T161240Z-e1196347/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:fc7ef0cc8e019e32eaaa5be724bb74c1905b9ed605345a03afc15b92be1a0ebc", + "evidence_path": "evidence/20260721T161240Z-e1196347/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:b7c98f8188f4bdf5093630e9233c557447330d03d478cc36fa707b2ac5192069", + "evidence_path": "evidence/20260721T161240Z-e1196347/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:efa9f148df1da0cabe17221d49182df5afc14fca028331c343945a98a233df89", + "evidence_path": "evidence/20260721T161240Z-e1196347/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:95c9ca965a53b257802e6ef96c534b03fde86bb6cd1a2a70fbd424664adec4f6", + "evidence_path": "evidence/20260721T161240Z-e1196347/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:d38c1aaf3bdae4d278856b0c8b28f53542607b1afd729f747875c135a5407cc6", + "evidence_path": "evidence/20260721T161240Z-e1196347/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:26a5c18d3016f966fd0ebb9a44156226df68a5b4bbf0a26fea9367edbeeb267d", + "evidence_path": "evidence/20260721T161240Z-e1196347/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:cc15d3c641c60832b6772b36bd4b605fedb7ef399559dbc27bebf08d05cdabc8", + "evidence_path": "evidence/20260721T161240Z-e1196347/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:048b8790ff977fe44fd98d55bb648eff65b8b6b01f90b9494bef718e785f598d", + "evidence_path": "evidence/20260721T161240Z-e1196347/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:2268ad2a667665c51d039a9d04bc2d99c9dc03b6817c972b70116f9c40ec4b2f", + "evidence_path": "evidence/20260721T161240Z-e1196347/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:cdeb0cbef12cd143b931e982dc2626f9d0118a848f1934152b20b09a3765d277", + "evidence_path": "evidence/20260721T161240Z-e1196347/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:5062f80d2cad68110b666d3d767b989bb8dca0e5edccc2ae63d66f7133196e95", + "evidence_path": "evidence/20260721T161240Z-e1196347/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:762d78d54604739bd470a4b1302fd966201f1f9f5f7a4de5e3d75220dcf3528a", + "evidence_path": "evidence/20260721T161240Z-e1196347/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:8d4429310697a07b72000d8dc39b37a5a4750dc423c98c8d45e1b9d48b83eed4", + "evidence_path": "evidence/20260721T161240Z-e1196347/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:1b75d1b88fbba5914187f62a9c733597001abe878062f8fadf5c11e4d7f51064", + "evidence_path": "evidence/20260721T161240Z-e1196347/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:45ca2b062b4aec28c333c91f43e9207508b1ec48e5c8d8af16a17cdd4d2cc4c6", + "evidence_path": "evidence/20260721T161240Z-e1196347/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:5553409a498d8579db83f3c7eb6db031d71c89a06e300a7512d02caea66101ea", + "evidence_path": "evidence/20260721T161240Z-e1196347/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:6b0163ebcbd5bd459ecc952d021820ee2785dbd78659c739900fe0d5e8e2003a", + "evidence_path": "evidence/20260721T161240Z-e1196347/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:6b3bc044832701d1dba96bc17f16b06ad4af98a30c18271d19aa62b9433e8053", + "evidence_path": "evidence/20260721T161240Z-e1196347/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:e58804a9e1aa72463e02b506b950ea0f21ee493540452aac5df6fdccdd0dba09", + "evidence_path": "evidence/20260721T161240Z-e1196347/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:39def4edec348e77b79afd63f9bf559d98b5dee8ec7405b6b0f6938dac81eeb5", + "evidence_path": "evidence/20260721T161240Z-e1196347/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:ebafb4a4fda18d936c2a84c9c9de89f45cd0ec114808bd2d45613db30a07f366", + "evidence_path": "evidence/20260721T161240Z-e1196347/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:1cc9a6ed9d95a87bf287bc401f81af3aaf2c0dec2f939c702357a7b5d93a037a", + "evidence_path": "evidence/20260721T161240Z-e1196347/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:bd0a63f633a6293116751e2430a72cc1f979fe66903ddd4f7ca93681465cdda8", + "evidence_path": "evidence/20260721T161240Z-e1196347/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:83baf528d249c1cd0bef4660a9b5a0df208aa791872127cee90bc51f53354289", + "evidence_path": "evidence/20260721T161240Z-e1196347/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:239122e08c1cd2e2437911dcde10703dbc949d946aaa03880eb1844cec2861f4", + "evidence_path": "evidence/20260721T161240Z-e1196347/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:02638dcda62e43205b9a597f5cda851f744f3643515eff1e232bf81b6fb5c587", + "evidence_path": "evidence/20260721T161240Z-e1196347/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:c40ab67051047855e591f4010fbc18e88cbf382c2aafec5c46fa01d9eb5e2c9e", + "evidence_path": "evidence/20260721T161240Z-e1196347/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:9695ac0aad81bf0fe1c8db4e7c2f50529a4377c55dd203d3da96084f9b4493a3", + "evidence_path": "evidence/20260721T161240Z-e1196347/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:a9d0c896143089e6028668f3887397634857337b87aad86ec7d24f97e4c21c3e", + "evidence_path": "evidence/20260721T161240Z-e1196347/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:2399a7acbfbaca60363fa2781354892f7490ab7929d98156f343aea08bc86fa0", + "evidence_path": "evidence/20260721T161240Z-e1196347/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:c1f1a9ea15ebd28432b0efd70c985f8c230495f362b3a1c15ebd982cca968184", + "evidence_path": "evidence/20260721T161240Z-e1196347/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3001298e5bbb59ddb2a6c4a313246cbd589e4ed6241d8b18de970a974e7f1080", + "evidence_path": "evidence/20260721T161240Z-e1196347/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:63ea125e9e7c51e2a6502c8dd26b9f72d572d0de223965e64d0bc31e3adb0b3a", + "evidence_path": "evidence/20260721T161240Z-e1196347/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:bfd70cd2823ce90165edbf7f44daece6b54d5fcc33427a5573a51ada9f25791f", + "evidence_path": "evidence/20260721T161240Z-e1196347/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fc8fdb9b3dada608671186cd433c645a90376c056a1cc2f389fbc3a1ca6ff52c", + "evidence_path": "evidence/20260721T161240Z-e1196347/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:4bd8ff80ca81235164748e4cf9f8b106ee668d906b87a7f7cd68fead320a511e", + "evidence_path": "evidence/20260721T161240Z-e1196347/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:2f90e39377d16a5dbeeddc7d5a4e7c88799ea30f4d95a43328cf2c598ecea430", + "evidence_path": "evidence/20260721T161240Z-e1196347/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run3.json b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run3.json new file mode 100644 index 0000000..a1e0e38 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run3.json @@ -0,0 +1,517 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T161342Z-c3fec484", + "timestamp": "2026-07-21T16:13:42Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8081/v1", + "model_id": "mlx-community/Qwen2.5-7B-Instruct-8bit", + "declared_quant": "8bit", + "server": { + "preset_name": "mlx_lm", + "reported_version": null, + "quirk_flags": [ + "unconstrained_post_hoc_parse" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:33cd2940b2ca37c81591404000b567caf1d1e57b227c8ab31859705e7f75ba6a", + "evidence_path": "evidence/20260721T161342Z-c3fec484/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:771db9e5cfcea8b1c3a3ee5dce43a50b4a6a253832fc9e692ef58be2ae4e89a6", + "evidence_path": "evidence/20260721T161342Z-c3fec484/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:1f566d616a2f3388746940ed825a3583f67474075ebfcf790fe56943b10a9f1c", + "evidence_path": "evidence/20260721T161342Z-c3fec484/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:96a9c5ceaf8b98baad8cef7c7b6987836b4d58c8e7abbf5ca6708a96f4635eb5", + "evidence_path": "evidence/20260721T161342Z-c3fec484/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:7c0a6e35aef94cf6674cf2f76aebc6952a3b8a9bb720cfc5ad170ff2e03f6207", + "evidence_path": "evidence/20260721T161342Z-c3fec484/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:b18010ae6449336b0218c67fd9e92a648876a5ac418bc276255caa5fa65557fa", + "evidence_path": "evidence/20260721T161342Z-c3fec484/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:6698e2d9209028cb3d2c02e1baede693df40d311ba5a414dc2f8650b8a8378b0", + "evidence_path": "evidence/20260721T161342Z-c3fec484/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:1aac49cf6d8371995c0ef7bb79e508fa020cbb00f788cbe4ea81ae96859967e1", + "evidence_path": "evidence/20260721T161342Z-c3fec484/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:3b308f39a59bf0f79a459894f382ca6b0d3ab1884a991c76d2a02adcae197c13", + "evidence_path": "evidence/20260721T161342Z-c3fec484/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9015437e403fd8f93161246785070b0c7cb9da0252cc7329e2eb4ab095be6419", + "evidence_path": "evidence/20260721T161342Z-c3fec484/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8df0954354a2ddc9378fe354b37744b4ddee2e8818b21dd65eaf8a534ca63c3c", + "evidence_path": "evidence/20260721T161342Z-c3fec484/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:c2c2c6b0d4b4ecb6445834b5ad056a49bd9936ec16483ad870d00c939a5ddc9e", + "evidence_path": "evidence/20260721T161342Z-c3fec484/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:a1d32688f6f6733f2843b06f6729cba780b8b3663dd06f2ada372a6f83bf8583", + "evidence_path": "evidence/20260721T161342Z-c3fec484/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:a4fcee2155d763127a9bca21a99d12df7352f40904993d501aa2edc2a7fc5159", + "evidence_path": "evidence/20260721T161342Z-c3fec484/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:2c22f092c36b0c3f749c80168b5fb340a48bd95a6c60e40adfc69bcb6cc02042", + "evidence_path": "evidence/20260721T161342Z-c3fec484/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:7244bb7b9d9bb5ef3eddd8ef0501002f9d61c16df440d599ffea7eb0d5c4f40a", + "evidence_path": "evidence/20260721T161342Z-c3fec484/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:78d46d6d5541bc7add06f4235bf68201fc02607d7d560a5f4bf63195c6751e07", + "evidence_path": "evidence/20260721T161342Z-c3fec484/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:266845b72969bcb92ce2d4e1627b1b7a6bc7462bc85a15a47310a718dafe9d0b", + "evidence_path": "evidence/20260721T161342Z-c3fec484/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:b6bc69b82f4c99299e219562470e557e8e7b10eb2382ec19997b62f55f5b0222", + "evidence_path": "evidence/20260721T161342Z-c3fec484/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:ca5ff9a535672becfac8ee9eff9a11fd1f96fb0b3bbfb20716f68ec52388f16a", + "evidence_path": "evidence/20260721T161342Z-c3fec484/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:997cba597ad2578edde927324d314fb7b8bc8d01899949e647efdc069ec71057", + "evidence_path": "evidence/20260721T161342Z-c3fec484/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:2bd4f65279d0748af545aa33c9a712774200593c2000bb7a8374e9bec2ead03c", + "evidence_path": "evidence/20260721T161342Z-c3fec484/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:d6a9d10ed502e4874f0f20b0f41000d7bfd4c748b991820ba3f87c01377f3115", + "evidence_path": "evidence/20260721T161342Z-c3fec484/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:a3e4472c4f659924f61b4ebccaff283b2a06900684eb72d087f04494117dacac", + "evidence_path": "evidence/20260721T161342Z-c3fec484/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:95adabb9c9cfa36f40ea518e2ed34795f9a0cfc83d14499ab679130fbac186d9", + "evidence_path": "evidence/20260721T161342Z-c3fec484/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:2a2f2287bd75915e624b5cb6452b551fee1e6b53517a8129892b7d69c0be14df", + "evidence_path": "evidence/20260721T161342Z-c3fec484/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:b3eb2eff4a93a1c3e1c93bd2d21cbadd978e9cb85d461aa367b52c33a0505cbb", + "evidence_path": "evidence/20260721T161342Z-c3fec484/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:50ffe56ae42646fee535b2adf65c1eb814a1085476427197f78303321db42008", + "evidence_path": "evidence/20260721T161342Z-c3fec484/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:da490472d59603bdeaa9f3d579ec5c0479cc39e0b74dd1991df924a417d9b931", + "evidence_path": "evidence/20260721T161342Z-c3fec484/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:ec506e92b7bac377e142b797a267200459dec092b374a10061f92c6bad4b4fae", + "evidence_path": "evidence/20260721T161342Z-c3fec484/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:b7a023bffe12494dd0561453a923d5f6dace0074bf05847a8cc7279cf34e2173", + "evidence_path": "evidence/20260721T161342Z-c3fec484/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:d2fef2a00e499fc1a8afe66910019b6e7644e1eb5b67086a9558013aa1fd06a6", + "evidence_path": "evidence/20260721T161342Z-c3fec484/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:7e458da8eec31cdf16bfff6a4b9d4c60432301e08686340c1492d00b2221093e", + "evidence_path": "evidence/20260721T161342Z-c3fec484/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:414cf3c8f4170364d24fbeddf44537d2cbf181ae5a4ae103afb29dadbbe87fc3", + "evidence_path": "evidence/20260721T161342Z-c3fec484/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:d89c1aaf0c62ee0d5f8947635c134801234b21ab393c6481a989d5aaee79ee36", + "evidence_path": "evidence/20260721T161342Z-c3fec484/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:e9f9b9f985834fcceea6fa4b05c17f0ef9fd2d24735dbdc1c603aa6e2fd8d8e3", + "evidence_path": "evidence/20260721T161342Z-c3fec484/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:925ea99bbe8d7f45180fae99a5c22a6f4c192956238af61a6e723f9660da3bea", + "evidence_path": "evidence/20260721T161342Z-c3fec484/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:7ff2e8a09a2fa577d5176562d08ed0ad685c5e3ee673a88929ecfa3dc9a16ccc", + "evidence_path": "evidence/20260721T161342Z-c3fec484/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:55ad95f099b88b114fc03c41f081cba48710fd6c48155b4c27f8104150b6e188", + "evidence_path": "evidence/20260721T161342Z-c3fec484/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:d7c094f9a9d89defd53a40e1279fe3fb5001deed057e8244f927dc3068ce3b43", + "evidence_path": "evidence/20260721T161342Z-c3fec484/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:964ac45e0bd75288b7c80858b6cab9a7f358650e986e13f307cce72cff8d4787", + "evidence_path": "evidence/20260721T161342Z-c3fec484/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:47f9bc0430ad1cd5560b24c492ca970f42cd7ffc11bfebc66f91f628d3598ce4", + "evidence_path": "evidence/20260721T161342Z-c3fec484/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:2fb1210395bd29d0ae2b389f1360df5dc3fb6965fa000cd71874a29ddd238315", + "evidence_path": "evidence/20260721T161342Z-c3fec484/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:bbd73e78646b60fdfd3b03039136b93ab94985d6931630bd43b1c8457131e28b", + "evidence_path": "evidence/20260721T161342Z-c3fec484/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:ec7bee1a7a8eec056f95104b68d291478338c79ad11e018a8e3d8e9ccdf8d029", + "evidence_path": "evidence/20260721T161342Z-c3fec484/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:7d814bf6f5fb04fad69cf0f2c571d6ec02d689fbefa10bf337c29ca33a6efe2b", + "evidence_path": "evidence/20260721T161342Z-c3fec484/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:30aacf97742e36d7e6a81e78ad8940ba24a8c60148a54e38b6a16d34b059e882", + "evidence_path": "evidence/20260721T161342Z-c3fec484/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:32e560171e71400f4b51b139547e093cee0a30b448412dfb76d166d08ba2ced6", + "evidence_path": "evidence/20260721T161342Z-c3fec484/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:e41a7fdbc6629b2d231650fbb9d9c6f62983fccb15a7e3dd3103964816da6db4", + "evidence_path": "evidence/20260721T161342Z-c3fec484/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:ec41e4284d22dd2fd82bf3e3deca748fc644553b39208ce36e728ac06b61b18e", + "evidence_path": "evidence/20260721T161342Z-c3fec484/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run4.json b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run4.json new file mode 100644 index 0000000..d940d87 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run4.json @@ -0,0 +1,517 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T161443Z-6be8388c", + "timestamp": "2026-07-21T16:14:43Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8081/v1", + "model_id": "mlx-community/Qwen2.5-7B-Instruct-8bit", + "declared_quant": "8bit", + "server": { + "preset_name": "mlx_lm", + "reported_version": null, + "quirk_flags": [ + "unconstrained_post_hoc_parse" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:22176a04a02f4eaaa5be59df7842ec1b4479d019b45ff77f8a538241e9da91e1", + "evidence_path": "evidence/20260721T161443Z-6be8388c/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:97c6a41911345d49ca0f462de73418398dfa70709ccf9ef22af108d8b8071004", + "evidence_path": "evidence/20260721T161443Z-6be8388c/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:a49f8894d587f2370eb56ae275357c04e10d55bf75860cc7c31076b43d2b6813", + "evidence_path": "evidence/20260721T161443Z-6be8388c/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:3226911bd3b2b50f66007d52f2a2fc8ebf8de50f4d2530c8e03a90ae89d2724b", + "evidence_path": "evidence/20260721T161443Z-6be8388c/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:9d1223b6a6fbf983a746698d81d10f348e784b2ba063ed6959e184c8cf9d3905", + "evidence_path": "evidence/20260721T161443Z-6be8388c/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:6adffd2cc150b9549ffeafc8999820e62d025fda1a52c132458f3452e6ab1df0", + "evidence_path": "evidence/20260721T161443Z-6be8388c/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:055a64de317a11e83d7e5ede69beca7274b92afaeb25aae3b353f255c7f623ed", + "evidence_path": "evidence/20260721T161443Z-6be8388c/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fd70eff3127a49c1df8b854d73344acac9a3c6db0f72877c5ec1a6167caeb987", + "evidence_path": "evidence/20260721T161443Z-6be8388c/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:fd8574caef7dd81eaaa1efd067a1c86d9dcb0be25032f2ca6b092c4681bc3d36", + "evidence_path": "evidence/20260721T161443Z-6be8388c/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:7ffd61e0b217bb75623b077173e513abef97ae2c5cb06cd55627eab3df78a6e6", + "evidence_path": "evidence/20260721T161443Z-6be8388c/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:c323e364e35f28283c8ffd37aa2488f6ce2fd54d7952bb44192e46ff689a9ce3", + "evidence_path": "evidence/20260721T161443Z-6be8388c/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:7e9fdac2dbad8569cffc2ef87192f2596e89c824810d96a6a8682e7c0726012b", + "evidence_path": "evidence/20260721T161443Z-6be8388c/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:2221bba6fd8373646ec494e6bfbb98e0b5fdc246cd4370ff0de1d0fe6794c7fa", + "evidence_path": "evidence/20260721T161443Z-6be8388c/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:e142e6d191af922d5f54737e2c18a7d28a0f7664799d4f2e39fcd136d2969c43", + "evidence_path": "evidence/20260721T161443Z-6be8388c/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:07a38ab718c9ae7570bc97b9b811e10a3cef48690237fcf058021bd2a98fd24e", + "evidence_path": "evidence/20260721T161443Z-6be8388c/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:1288fc4aeae02186b4201526111156c4394561ffbac991dabbdc331e2ac6a93c", + "evidence_path": "evidence/20260721T161443Z-6be8388c/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:1d7f812c2da0a657e943bfa99273d96fe010048c191e90370810816155642fdb", + "evidence_path": "evidence/20260721T161443Z-6be8388c/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:cffa8e67bf967d98782248f060b4acfd4f585e3c45c5c8e09d8dcb958e2ac3d6", + "evidence_path": "evidence/20260721T161443Z-6be8388c/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:473cb6ccad7ca4d923d11e5dcaed3975fe69c57970d4a090ef98fe219bd2bd55", + "evidence_path": "evidence/20260721T161443Z-6be8388c/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:179c08ff23aa054093938d866ba33f6e530cded51fbc03b5183b9ee3a16bcbcd", + "evidence_path": "evidence/20260721T161443Z-6be8388c/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:737610dc010551a96ebcb9476f98971ce796e71dd874ba656f70af6dd57558e2", + "evidence_path": "evidence/20260721T161443Z-6be8388c/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:074ca845481ffac942c00308bb7550f8d0eba6539bf76e4b67a8210d62581ab9", + "evidence_path": "evidence/20260721T161443Z-6be8388c/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:dacd2802e5b94b948425f29965f1c04c5a72ee18259d51114199138a52faf976", + "evidence_path": "evidence/20260721T161443Z-6be8388c/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:9f3db84315b7a7c6dcbede4227052c4d32d7f8176ad046359541747473df2fbd", + "evidence_path": "evidence/20260721T161443Z-6be8388c/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:ab0740c8b414b967e812f6554698e314254c1a9dd8064bb204c42cab4b2a5821", + "evidence_path": "evidence/20260721T161443Z-6be8388c/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:c6a4b64eccfc9061270695b9d08b21b0106861d7aefd2dfc4eecb76a179bf127", + "evidence_path": "evidence/20260721T161443Z-6be8388c/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:47e153fa56e15c7208753a1513bfb370dfeba5941e6cf56d17ac0af4e2393a50", + "evidence_path": "evidence/20260721T161443Z-6be8388c/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:1e40c405b60f7a57a01335c9868a914a394752be75272f5bb1330916dc8ce311", + "evidence_path": "evidence/20260721T161443Z-6be8388c/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:5c001bb000f0a40cc80ff15cb27cd59a382f005279031983b9315a91ddee9fa2", + "evidence_path": "evidence/20260721T161443Z-6be8388c/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:819063a3cc88922c067306eede57e1f522004bb4d27af0052edeb44890bec59f", + "evidence_path": "evidence/20260721T161443Z-6be8388c/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:2d7e677cd2872696ed139b3e0d5ef0295eba13c6368d9a51d1b81ed81c53060a", + "evidence_path": "evidence/20260721T161443Z-6be8388c/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:bd1f47f66f85ef6766c326630bec5c113f717d588f0649288160da830427e9b7", + "evidence_path": "evidence/20260721T161443Z-6be8388c/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:ab72c710a2b7d7d0a06a88a7e6d3c2a9e45450332f5ff8e29a51e3f4c8a2fd58", + "evidence_path": "evidence/20260721T161443Z-6be8388c/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:081092fb0593c02474ad448a2a02da769b870ef3e58ef76e5faec6bb47e21b16", + "evidence_path": "evidence/20260721T161443Z-6be8388c/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:8eca7b8ff60a3eeb287d979d031e3df032c86001a5a8ffcc44fc4cef6ab2f6c7", + "evidence_path": "evidence/20260721T161443Z-6be8388c/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:57d6c1c04755e20565c8fa49f8e4403fcc99832a4652b99ba257abe38176fe95", + "evidence_path": "evidence/20260721T161443Z-6be8388c/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:eff0e969eadb25e5b9925b060080290d8eaaa398830de79fea126dac7f20142e", + "evidence_path": "evidence/20260721T161443Z-6be8388c/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:91da1526465d0a03b12409830878a457a2bae3d7d1f805819aa0cdecfc439177", + "evidence_path": "evidence/20260721T161443Z-6be8388c/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:34449a7b893d76f3cd612e6b676812705652e5cc6d5925d27883bf036c0e4db4", + "evidence_path": "evidence/20260721T161443Z-6be8388c/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:3c8f3e06f6990f651f802b07686e24449bbc470fab14ada99604597d7a07806f", + "evidence_path": "evidence/20260721T161443Z-6be8388c/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:c41a53eac212720abe67be5d4e1b50fd7520955a94c1e0923ca6de500e1c6c93", + "evidence_path": "evidence/20260721T161443Z-6be8388c/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:91fe446067ac220e2d1590df39d9da669a67b6ba8b4768cd61eb9baccfa10e55", + "evidence_path": "evidence/20260721T161443Z-6be8388c/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:74fe3215ef8f102cf992a0ec8532d79fafe104e60313743126ae9326c232e3a0", + "evidence_path": "evidence/20260721T161443Z-6be8388c/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:aae7dc4769cc98623db710be54301b65d2b5cd4fb59bdfd42e1da3d4d18eeb1d", + "evidence_path": "evidence/20260721T161443Z-6be8388c/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:bc89a5cc231010c0050eb0a39de3a065beaabaf55e547062cf26fba6d72eb26e", + "evidence_path": "evidence/20260721T161443Z-6be8388c/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:33bd0be1274df6895f61edf97bd7c545154b8acbaef06a4171cde7136e4271d1", + "evidence_path": "evidence/20260721T161443Z-6be8388c/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:c4a62601506d3ee9b1faa7a2a262bd10d9d8fc5678de68d88cb9c584351845f7", + "evidence_path": "evidence/20260721T161443Z-6be8388c/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:8e4308417963426a02ad616b39455ec3add2813e93744c8435c4a082975ed59d", + "evidence_path": "evidence/20260721T161443Z-6be8388c/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:269c6a7d3420cadd8808939b3f12dff218083e814eac05601e721c1df960fb85", + "evidence_path": "evidence/20260721T161443Z-6be8388c/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:387460c99a8aa0465e16460d828510d6522f476ee14c5e1c984cad29ba72e411", + "evidence_path": "evidence/20260721T161443Z-6be8388c/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run5.json b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run5.json new file mode 100644 index 0000000..8c2a767 --- /dev/null +++ b/docs/case-studies/evidence/replication/m6-mlx-repl/mlx_lm-qwen2.5-7b-instruct-8bit-run5.json @@ -0,0 +1,517 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260721T161544Z-35f43477", + "timestamp": "2026-07-21T16:15:44Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8081/v1", + "model_id": "mlx-community/Qwen2.5-7B-Instruct-8bit", + "declared_quant": "8bit", + "server": { + "preset_name": "mlx_lm", + "reported_version": null, + "quirk_flags": [ + "unconstrained_post_hoc_parse" + ] + }, + "environment": { + "host_hardware_class": "Apple M4 Max, 64GB", + "host_os": "macOS 26.5.2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:f943809a4b6f63c323b836c9c4af35a1e51352c4fce2ca6c57d7d396472f9b85", + "evidence_path": "evidence/20260721T161544Z-35f43477/multi-turn-calendar-followup.json", + "retried": false + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:69f7e7d35da8e1fac6f6c26f425733dbda34e1f5853ed171aeb6cc0efbf8956b", + "evidence_path": "evidence/20260721T161544Z-35f43477/multi-turn-flight-booking.json", + "retried": false + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:a3fdb37d81c79ff7e17b337b65b4e94baabe30d0c6f563b5009baff68093ccab", + "evidence_path": "evidence/20260721T161544Z-35f43477/multi-turn-inventory-order.json", + "retried": false + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:c6f63df7e1f8d76161cfad1d65f47f97fed338d70801f03ddc5e7d1024012e6b", + "evidence_path": "evidence/20260721T161544Z-35f43477/multi-turn-parallel-followup.json", + "retried": false + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:0eefad8b1ba634cb6e0bcdda2046225a98104f0d717385148baa66c555cd5e3d", + "evidence_path": "evidence/20260721T161544Z-35f43477/multi-turn-route.json", + "retried": false + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:868540de8c5d265cb05b5049b62aa1cb304c73b0af435f5a4a982668863a8d99", + "evidence_path": "evidence/20260721T161544Z-35f43477/multi-turn-ticket-update.json", + "retried": false + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "status": "fail", + "failure_reason": "turn 1: no tool call emitted", + "evidence_hash": "sha256:837b878872b0b61a30003d7ddcabe923c6a7a2ca7d1173b4bc6ec32c7b0aee0c", + "evidence_path": "evidence/20260721T161544Z-35f43477/multi-turn-two-hop-chain.json", + "retried": false + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:9a5e66599799828f425b48aea06ef94bda0eb3f04db08ce96b2a59985d428088", + "evidence_path": "evidence/20260721T161544Z-35f43477/negative-auto-arithmetic.json", + "retried": false + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:e4b0b03009c5be861df8a95cf77fa922e9bf3560bc01e1549862aeed7cf40a50", + "evidence_path": "evidence/20260721T161544Z-35f43477/negative-auto-knowledge.json", + "retried": false + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:be1cf43749d72f04cf4d50744d73bf63c2c34c59cbe46cc79bccfc3baf201f05", + "evidence_path": "evidence/20260721T161544Z-35f43477/negative-greeting.json", + "retried": false + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:23c0c0c12960a33d6346ee9f4ae1ddbed809aef105d2b02589d11884a68d97db", + "evidence_path": "evidence/20260721T161544Z-35f43477/negative-invalid-schema.json", + "retried": false + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:17b8d43344610ec6f7b6cc24c5e221f4c0f11f4d6ab0e31a40c60d235ede10a9", + "evidence_path": "evidence/20260721T161544Z-35f43477/negative-long-argument.json", + "retried": false + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:07904da1eeac4d573a5731222006e2cac133560f29693ab7e25c0de1fde752bd", + "evidence_path": "evidence/20260721T161544Z-35f43477/negative-plain-text.json", + "retried": false + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:94aa097f4bc38545342cf03f105b89edd3f589437213ce910168a88c2dc91f8e", + "evidence_path": "evidence/20260721T161544Z-35f43477/negative-unicode-argument.json", + "retried": false + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:d52c47ba9b83b88c6d339f8da55517013c5d08a7216a8ca4a3be4ae2a9a684f9", + "evidence_path": "evidence/20260721T161544Z-35f43477/parallel-city-time.json", + "retried": false + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:3b45198f7039dec8070aee4c1ae4502e4d5b6f03f2f76f55a76ec5e9b5a94931", + "evidence_path": "evidence/20260721T161544Z-35f43477/parallel-different-tools.json", + "retried": false + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:0c1c2aa7d4f4d3573bf6b5b72de392e84d5701569bbd7e9bba49c2da3e56607d", + "evidence_path": "evidence/20260721T161544Z-35f43477/parallel-three-cities.json", + "retried": false + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:dbce1fd6a804dc108c493b18b7df416e5099505e5133a3bbe33a75a9697bda3d", + "evidence_path": "evidence/20260721T161544Z-35f43477/parallel-three-different-tools.json", + "retried": false + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:791f16985de82a1e8f83e82212ceede370cc49d55c801bfa14d484ad8f58556d", + "evidence_path": "evidence/20260721T161544Z-35f43477/parallel-two-calculations.json", + "retried": false + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:59699ebf49629710a85e8bca91326be10c67b307602fe4e833afec2c171caa89", + "evidence_path": "evidence/20260721T161544Z-35f43477/parallel-two-multiple-args.json", + "retried": false + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:da31edb5490aa437a71d61baa42e6071debf500e4293745c84e306030f444fcf", + "evidence_path": "evidence/20260721T161544Z-35f43477/parallel-two-searches.json", + "retried": false + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:e2e59f8a86f5ac2da33e4c46d2147347157396ab64f73ebadb7d9d665d920283", + "evidence_path": "evidence/20260721T161544Z-35f43477/parallel-weather-and-time.json", + "retried": false + }, + { + "id": "single-array-tags", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:7aa9914502c002aacd17e073b4ddcfc58a38ad243564c3897d3a7490236ee150", + "evidence_path": "evidence/20260721T161544Z-35f43477/single-array-tags.json", + "retried": false + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:714372189db1f96e204b4f3b20d92a090a7461bf06c9428e2b118b33d866edeb", + "evidence_path": "evidence/20260721T161544Z-35f43477/single-boolean-flag.json", + "retried": false + }, + { + "id": "single-date-range", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:0d30dd40e6ff57687d337b893c32e95eab81b0f609cb6346a1f28fdcd2ad9bc8", + "evidence_path": "evidence/20260721T161544Z-35f43477/single-date-range.json", + "retried": false + }, + { + "id": "single-decimal-price", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:11199df55b8151eb808fa8f2d43bc618f561f804c214a28b1ab129973bd82eb6", + "evidence_path": "evidence/20260721T161544Z-35f43477/single-decimal-price.json", + "retried": false + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:9f04d853ae4f6322fd741e28c8455b540a8dd2ad67c9dfcb25f33a2a6f3a04bd", + "evidence_path": "evidence/20260721T161544Z-35f43477/single-empty-arguments.json", + "retried": false + }, + { + "id": "single-enum-format", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:327aa884989413660f8091bfdf7b318a707785247b9c486b43aa9bd019963e85", + "evidence_path": "evidence/20260721T161544Z-35f43477/single-enum-format.json", + "retried": false + }, + { + "id": "single-integer-limit", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:8d7b987b6cc2e20930e97a1785cd15cf147827ee4762d326faffde63de084a64", + "evidence_path": "evidence/20260721T161544Z-35f43477/single-integer-limit.json", + "retried": false + }, + { + "id": "single-nested-options", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:416ddc8467458788adbb1f6b5c9308ef701793c5cc239ff84432f97d962f23b3", + "evidence_path": "evidence/20260721T161544Z-35f43477/single-nested-options.json", + "retried": false + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:421f3888a41e5096c9f0ed7b96d9d839ece70e0418eb4fd76574119e968f0be8", + "evidence_path": "evidence/20260721T161544Z-35f43477/single-optional-omitted.json", + "retried": false + }, + { + "id": "single-optional-present", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:1ffab7f2a40b57b043d325a58827ff7d0463b2697e04d18851a5f638b2ef78f1", + "evidence_path": "evidence/20260721T161544Z-35f43477/single-optional-present.json", + "retried": false + }, + { + "id": "single-string-list", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:f354b6138178982472cbbcbee45466d702bac9356fc2f157d1d727703bdd561b", + "evidence_path": "evidence/20260721T161544Z-35f43477/single-string-list.json", + "retried": false + }, + { + "id": "single-weather", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:9d12e9a179feff513ace472989a52c4292878f9768fff5cb35b8ef524c94a320", + "evidence_path": "evidence/20260721T161544Z-35f43477/single-weather.json", + "retried": false + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:e246c0f5394c1d89db4050eb22868bc5de5c8416371bcf2e04fefe96eb2a824e", + "evidence_path": "evidence/20260721T161544Z-35f43477/single-zero-and-negative.json", + "retried": false + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:a4acf0e2315c3703e569d8f36a8f3b6acd897a63ccb10a639c0e05780992bdfe", + "evidence_path": "evidence/20260721T161544Z-35f43477/streaming-array-arguments.json", + "retried": false + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:f1a605a63ac98f287f524f7a27b0741f6e29df506162575570d32b44e8e4eea9", + "evidence_path": "evidence/20260721T161544Z-35f43477/streaming-boolean-argument.json", + "retried": false + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:82a06071c07240eea4a223d9291115551bec1f93957c20f8e539a0ccb11d623f", + "evidence_path": "evidence/20260721T161544Z-35f43477/streaming-empty-arguments.json", + "retried": false + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:c230a1626926cbca67e0444aaac3a2211327af32feef7f79658216b9eb91b589", + "evidence_path": "evidence/20260721T161544Z-35f43477/streaming-enum-arguments.json", + "retried": false + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:ed50b419ab343721c7a667478c71291c9104771c4bfea898270fe414afa29c41", + "evidence_path": "evidence/20260721T161544Z-35f43477/streaming-nested-arguments.json", + "retried": false + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:4e8a7cafef7a12bbd2f0d540222b7e1c95da08d8a2fc671c76370bad04bc6c0d", + "evidence_path": "evidence/20260721T161544Z-35f43477/streaming-parallel-three.json", + "retried": false + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "status": "fail", + "failure_reason": "no tool call emitted", + "evidence_hash": "sha256:d15c3187c0d32aea0e53cf2e70f85aa9325f146537fe8454e2fa59175c938e30", + "evidence_path": "evidence/20260721T161544Z-35f43477/streaming-parallel-two.json", + "retried": false + }, + { + "id": "streaming-weather", + "category": "streaming", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:f50c27149b68be45149497e63b3b5e7e637d4402e8f173249aa40b4ff4788241", + "evidence_path": "evidence/20260721T161544Z-35f43477/streaming-weather.json", + "retried": false + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:0d8fc6f91860aeda193ce624c8d5d010119df286cc4ef5c7402f59cfbacc99c3", + "evidence_path": "evidence/20260721T161544Z-35f43477/tool-choice-auto-call.json", + "retried": false + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:de53361004676d409acddc466c2f4b3768ed08ae334182b5247683d42cfa35c8", + "evidence_path": "evidence/20260721T161544Z-35f43477/tool-choice-auto-no-call.json", + "retried": false + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:b54047a871dd045fa200a428b494cd4a46af981ad5015c874d059fc25eb40f4d", + "evidence_path": "evidence/20260721T161544Z-35f43477/tool-choice-named.json", + "retried": false + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:da636dcd8caa28d8f4455ba2b5077580fac2241bb15607556a897738b2c7caff", + "evidence_path": "evidence/20260721T161544Z-35f43477/tool-choice-named-empty.json", + "retried": false + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "evidence_hash": "sha256:99c446069b2a03c516a26f30b200a63c774d376e5dc85982f81963fe440e5efc", + "evidence_path": "evidence/20260721T161544Z-35f43477/tool-choice-none.json", + "retried": false + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:e3185b9539ce9020fd2a238ae842d546f1f3e72d0625315f6c684896d89a0107", + "evidence_path": "evidence/20260721T161544Z-35f43477/tool-choice-required.json", + "retried": false + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "status": "fail", + "failure_reason": "empty response: no content and no tool call", + "failure_class": "empty_response", + "evidence_hash": "sha256:73cd82c9a3f0a44c8e657a193c561d027e7fcdfe0be8ca5ca0e7353edb573156", + "evidence_path": "evidence/20260721T161544Z-35f43477/tool-choice-required-empty.json", + "retried": false + } + ], + "totals": { + "total": 50, + "passed": 7, + "failed": 43, + "errors": 0, + "skipped": 0 + } +} diff --git a/docs/migrations/2026-08-05-local-path-redaction.md b/docs/migrations/2026-08-05-local-path-redaction.md index f89b8b1..ad268e7 100644 --- a/docs/migrations/2026-08-05-local-path-redaction.md +++ b/docs/migrations/2026-08-05-local-path-redaction.md @@ -35,7 +35,7 @@ python3 tools/redact_local_paths.py \ --source-commit "$SOURCE_COMMIT" \ --result results/.json \ --evidence-root results \ - --ledger docs/migrations/local-path-redaction-ledgers/.json + --ledger migrations/path-redaction/.json ``` Each per-run JSON ledger records the source commit, repository-relative result @@ -52,11 +52,11 @@ transcript formatting, response byte, or result field changed: ```sh python3 tools/redact_local_paths.py \ --check \ - --ledger docs/migrations/local-path-redaction-ledgers/.json \ + --ledger migrations/path-redaction/.json \ --evidence-root results python3 tools/redact_local_paths.py \ - --check-all docs/migrations/local-path-redaction-ledgers \ + --check-all migrations/path-redaction \ --evidence-root results ``` diff --git a/docs/superpowers/specs/2026-07-19-willitcall-design.md b/docs/superpowers/specs/2026-07-19-willitcall-design.md index 3740318..dc62f3c 100644 --- a/docs/superpowers/specs/2026-07-19-willitcall-design.md +++ b/docs/superpowers/specs/2026-07-19-willitcall-design.md @@ -173,15 +173,15 @@ the combined matrix. failure mode. That is WRONG and is retracted. Across both servers only 1 of 86 phi4-mini failures matches any tool-call shape; the rest are prose refusals and narration with no call at all, so phi4-mini's 7/50 is - largely genuine model failure. The claim was generalised from a single + largely genuine model failure. The claim was generalized from a single example - the n=1 error amendment 4 exists to prevent. Evidence and the full retraction are in docs/case-studies/2026-07-20-granite-emits-tool-calls-no-server-parses.md. - Why it matters: as recorded today these cells are indistinguishable from "the model cannot call tools", which is false and is exactly the kind of misattribution amendment 2 exists to prevent. The 7 scenarios each model - passes are precisely the negative traps, where emitting no parsed call is - the correct answer. + passes are precisely the scenarios where making no call is the correct + answer. - Proposal: add `failure_class: "unparsed_tool_call"`, set mechanically when a failing scenario has no `tool_calls` AND its response content matches a known tool-call shape. The detection rule must be conservative and @@ -210,12 +210,12 @@ the combined matrix. 6. Measurement environment uniformity and disclosure (owner, 2026-07-20). M4 published rows measured on two different hosts (wave 1: 16GB MacBook; - waves 2-3: macstudio M4 Max 64GB) without disclosure. For a project whose + waves 2-3: the measurement host, M4 Max 64GB) without disclosure. For a project whose product is trustworthy measurement, undisclosed environment differences are unacceptable. - Result files gain environment fields (host hardware class, server name+version) - additive to schema v2. - - Wave-1 rows are RE-RUN on macstudio so every published row shares one + - Wave-1 rows are RE-RUN on the measurement host so every published row shares one measurement host; the site discloses the environment per row (or one global statement once uniform). - Going forward: mixed-host result sets are fine for the community matrix diff --git a/docs/superpowers/specs/2026-08-05-m7-measurement-contract.md b/docs/superpowers/specs/2026-08-05-m7-measurement-contract.md index 32941b3..99eb989 100644 --- a/docs/superpowers/specs/2026-08-05-m7-measurement-contract.md +++ b/docs/superpowers/specs/2026-08-05-m7-measurement-contract.md @@ -29,9 +29,11 @@ chart is downstream of fixing that. Three further facts established during the audit: - **No published row clears amendment 4.** 32 distinct arms, none replicated. - The n=5 data backing the case studies lives off-repo on macstudio. Amendment 7 - already records that the original n=5 runs measured harness determinism rather - than model variance. + The n=5 data backing the case studies was off-repo, on the measurement host. + Since 2026-10-05 the run files are in + `docs/case-studies/evidence/replication/`. The transcripts are not published. + Amendment 7 already records that the original n=5 runs measured harness + determinism rather than model variance. - **`declared_quant` is null for all 14 Ollama rows**, so the quant axis is unqueryable for the largest server bucket. - **`multi_turn` passes 37/224 (17%), with 22 of 32 rows scoring exactly zero.** @@ -55,8 +57,8 @@ project exists to prevent, present in its own flagship artifact. Required: a distinct `not-measurable` cell state when `errors > 0 && passed + failed == 0`, an accessible label carrying all four -counts (pass/fail/error/skipped), a legend entry, and a non-colour encoding so -the distinction survives greyscale and colour blindness. +counts (pass/fail/error/skipped), a legend entry, and a non-color encoding so +the distinction survives grayscale and color blindness. ### T0.2 Absolute local paths are published @@ -133,7 +135,7 @@ Model identity alone is not sufficient to identify a comparable arm. | Field | Why | |---|---| | `metadata.corpus.{id, revision, sha256, scenario_count}` | `--scenarios` accepts arbitrary definitions (`main.rs:416`). Matching scenario ids do not prove matching prompts or schemas. | -| `metadata.corpus.scoring_version` | Scorer behaviour changes without scenario ids changing. Historical totals must not silently acquire new semantics. | +| `metadata.corpus.scoring_version` | Scorer behavior changes without scenario ids changing. Historical totals must not silently acquire new semantics. | | `metadata.server.decode_mode` | Explicit `grammar_constrained` / `unconstrained_post_hoc` / `unknown`, replacing inference from `quirk_flags` where absence ambiguously means unverified. | | `metadata.server.chat_template.{id, sha256}` | The README names chat template as a stack dimension; results do not record it at all. | | `metadata.server.launch_config_sha256` | Same server version differs by `--jinja`, context size, cache quantization, parser flags. | @@ -147,10 +149,10 @@ Model identity alone is not sufficient to identify a comparable arm. A deterministic migration command plus an explicit 32-entry mapping manifest. **Identity is never inferred from filenames** - the site's current -filename-derived label (`site.rs:504`) is precisely the behaviour being retired. +filename-derived label (`site.rs:504`) is precisely the behavior being retired. - Ollama rows: recover manifest digest and real quantization from saved - manifests or macstudio. Otherwise mark `declared` or `unresolved`. + manifests or the measurement host. Otherwise mark `declared` or `unresolved`. - HF GGUF and MLX rows: map repo to canonical checkpoint, recover an immutable revision. A bare repo id is not an immutable artifact. - Blob-path rows: the blob name already carries a sha256, but blob-to-canonical @@ -234,16 +236,16 @@ Shipped instead: 3. **Outcome-signature inventory.** Keyed on the exact 50-status vector, valued by the stacks sharing it. Better than a histogram for the 7/50 cluster because it shows whether identical totals came from identical scenarios. - Granite's seven passes are exactly the negative category. + Granite passes exactly the seven scenarios where making no call is correct. 4. **Per-capability aggregate across stacks.** Six bars: "passed in X of Y measured stacks". A count of measurements, not a ranking. This is what surfaces the multi-turn cliff. It is **not** filtered to n>=5 arms: with zero replication in `results/`, such a filter renders empty. 5. **Observed pass-count strip plot**, one dot per fully measurable stack, - error-bearing rows in a separate "not fully measurable" panel, labelled + error-bearing rows in a separate "not fully measurable" panel, labeled "distribution of published one-run stack observations". -Every SVG carries ``, `<desc>`, a table fallback, four-state non-colour +Every SVG carries `<title>`, `<desc>`, a table fallback, four-state non-color encoding, and escaped labels. No chart library. ### T2.3 Information architecture @@ -254,7 +256,7 @@ They are a design failure to be compiled into the page's geometry. - Group rows by model; nest quant and server beneath. - Stratify into visible `grammar_constrained` and `unconstrained_post_hoc` bands, each badged. The valid comparison becomes what adjacency shows; the - invalid one requires crossing a labelled boundary. + invalid one requires crossing a labeled boundary. - Human column labels, with the snake_case id in the tooltip. - Model search box. - Move caveat prose below the matrix; keep one sentence above it. @@ -297,7 +299,7 @@ against the embedded corpus. | # | Item | Depends on | Acceptance | |---|---|---|---| -| 1 | T0.1 errors are not failures | - | All-error cells render a distinct non-colour-coded `not measurable` state; accessible label carries all four counts; regression test covers gemma3-shaped input | +| 1 | T0.1 errors are not failures | - | All-error cells render a distinct non-color-coded `not measurable` state; accessible label carries all four counts; regression test covers gemma3-shaped input | | 2 | T0.3 CI gate | - | `pages.yml` builds the site on `pull_request` without deploying; a deliberately broken generator fails the check | | 3 | T0.2 path redaction | - | No absolute `/Users/...` path in any result, transcript, or generated HTML; `evidence_hash` recomputed and the rehash documented; `site_cli.rs:236` assertion replaced | | 4 | T1.1 + T1.2 schema v3 | 3 | v3 emitted by new runs; versioned parsing accepts v1/v2/v3; local paths sanitized before serialization; validator green | diff --git a/registry/evidence/local-file-recovery.json b/registry/evidence/local-file-recovery.json index d598d7f..b545f04 100644 --- a/registry/evidence/local-file-recovery.json +++ b/registry/evidence/local-file-recovery.json @@ -4,7 +4,7 @@ "subject": "llamacpp-granite3.1-dense-8b.json", "model_id": "sha256-44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8", "status": "recovered", - "source": "ssh macstudio: blob existence check and saved-manifest reference search for sha256:44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8; saved manifest granite3.1-dense:8b; 2026-07-20-granite-emits-tool-calls-no-server-parses.md", + "source": "measurement host: blob existence check and saved-manifest reference search for sha256:44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8; saved manifest granite3.1-dense:8b; 2026-07-20-granite-emits-tool-calls-no-server-parses.md", "captured_at": "2026-08-05T18:01:51Z", "sha256": "44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8", "corroborated_ollama_model": "granite3.1-dense:8b", @@ -14,7 +14,7 @@ "subject": "llamacpp-phi4-mini.json", "model_id": "sha256-3c168af1dea0a414299c7d9077e100ac763370e5a98b3c53801a958a47f0a5db", "status": "recovered", - "source": "ssh macstudio: blob existence check and saved-manifest reference search for sha256:3c168af1dea0a414299c7d9077e100ac763370e5a98b3c53801a958a47f0a5db; saved manifest phi4-mini:latest", + "source": "measurement host: blob existence check and saved-manifest reference search for sha256:3c168af1dea0a414299c7d9077e100ac763370e5a98b3c53801a958a47f0a5db; saved manifest phi4-mini:latest", "captured_at": "2026-08-05T18:01:51Z", "sha256": "3c168af1dea0a414299c7d9077e100ac763370e5a98b3c53801a958a47f0a5db", "corroborated_ollama_model": "phi4-mini:latest", @@ -24,7 +24,7 @@ "subject": "llamacpp-watt-tool-8b-q4_k_m.json", "model_id": "watt-tool-8B.Q4_K_M.gguf", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 watt-tool-8B.Q4_K_M.gguf; GET https://huggingface.co/api/models/mradermacher/watt-tool-8B-GGUF; GET https://huggingface.co/api/models/mradermacher/watt-tool-8B-GGUF/tree/0e8a7402ad3f90bd2d65c8ff04f6b6cb7ee3df7a?recursive=true&expand=true", + "source": "measurement host: shasum -a 256 watt-tool-8B.Q4_K_M.gguf; GET https://huggingface.co/api/models/mradermacher/watt-tool-8B-GGUF; GET https://huggingface.co/api/models/mradermacher/watt-tool-8B-GGUF/tree/0e8a7402ad3f90bd2d65c8ff04f6b6cb7ee3df7a?recursive=true&expand=true", "captured_at": "2026-08-05T18:01:51Z", "sha256": "33706b165cd6777e29fdcd777ba1e09bd3b2006b428014e1bad17df3902ec1e7", "upstream_artifact_repository": "mradermacher/watt-tool-8B-GGUF", diff --git a/registry/evidence/ollama-recovery.json b/registry/evidence/ollama-recovery.json index 7b0480a..cae61b3 100644 --- a/registry/evidence/ollama-recovery.json +++ b/registry/evidence/ollama-recovery.json @@ -4,11 +4,11 @@ "subject": "ollama-qwen3-0.6b.json", "model_id": "qwen3:0.6b", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 saved manifest for qwen3:0.6b; jq model layer; jq .file_type from config blob sha256:b0830f4ff6a0220cfd995455206353b0ed23c0aee865218b154b7a75087b4e55", + "source": "measurement host: shasum -a 256 saved manifest for qwen3:0.6b; jq model layer; jq .file_type from config blob sha256:b0830f4ff6a0220cfd995455206353b0ed23c0aee865218b154b7a75087b4e55", "captured_at": "2026-08-05T18:01:51Z", "manifest_digest": "sha256:7df6b6e09427a769808717c0a93cadc4ae99ed4eb8bf5ca557c90846becea435", "model_layer_blob_digest": "sha256:7f4030143c1c477224c5434f8272c662a8b042079a0a584f0a27a1684fe2e1fa", - "gguf_metadata_source": "ssh macstudio python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-7f4030143c1c477224c5434f8272c662a8b042079a0a584f0a27a1684fe2e1fa; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", + "gguf_metadata_source": "measurement host, python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-7f4030143c1c477224c5434f8272c662a8b042079a0a584f0a27a1684fe2e1fa; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", "gguf_metadata_captured_at": "2026-08-06T02:34:30Z", "gguf_metadata": { "version": 3, @@ -28,11 +28,11 @@ "subject": "ollama-qwen3-1.7b.json", "model_id": "qwen3:1.7b", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 saved manifest for qwen3:1.7b; jq model layer; jq .file_type from config blob sha256:517ccaff02fe9f99718ab2f478e611aa2813c36263eeed86c2427bc122d14ad8", + "source": "measurement host: shasum -a 256 saved manifest for qwen3:1.7b; jq model layer; jq .file_type from config blob sha256:517ccaff02fe9f99718ab2f478e611aa2813c36263eeed86c2427bc122d14ad8", "captured_at": "2026-08-05T18:01:51Z", "manifest_digest": "sha256:8f68893c685c3ddff2aa3fffce2aa60a30bb2da65ca488b61fff134a4d1730e7", "model_layer_blob_digest": "sha256:3d0b790534fe4b79525fc3692950408dca41171676ed7e21db57af5c65ef6ab6", - "gguf_metadata_source": "ssh macstudio python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-3d0b790534fe4b79525fc3692950408dca41171676ed7e21db57af5c65ef6ab6; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", + "gguf_metadata_source": "measurement host, python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-3d0b790534fe4b79525fc3692950408dca41171676ed7e21db57af5c65ef6ab6; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", "gguf_metadata_captured_at": "2026-08-06T02:34:30Z", "gguf_metadata": { "version": 3, @@ -52,11 +52,11 @@ "subject": "ollama-qwen3-4b.json", "model_id": "qwen3:4b", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 saved manifest for qwen3:4b; jq model layer; jq .file_type from config blob sha256:e18a783aae5525fd2852fc94c985541a77e791e034abc2d3056474d59de336fc", + "source": "measurement host: shasum -a 256 saved manifest for qwen3:4b; jq model layer; jq .file_type from config blob sha256:e18a783aae5525fd2852fc94c985541a77e791e034abc2d3056474d59de336fc", "captured_at": "2026-08-05T18:01:51Z", "manifest_digest": "sha256:359d7dd4bcdab3d86b87d73ac27966f4dbb9f5efdfcc75d34a8764a09474fae7", "model_layer_blob_digest": "sha256:3e4cb14174460404e7a233e531675303b2fbf7749c02f91864fe311ab6344e4f", - "gguf_metadata_source": "ssh macstudio python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-3e4cb14174460404e7a233e531675303b2fbf7749c02f91864fe311ab6344e4f; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", + "gguf_metadata_source": "measurement host, python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-3e4cb14174460404e7a233e531675303b2fbf7749c02f91864fe311ab6344e4f; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", "gguf_metadata_captured_at": "2026-08-06T02:34:30Z", "gguf_metadata": { "version": 3, @@ -78,11 +78,11 @@ "subject": "ollama-qwen3-8b.json", "model_id": "qwen3:8b", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 saved manifest for qwen3:8b; jq model layer; jq .file_type from config blob sha256:05a61d37b08453e59290add468e3bb2f688e23a01e967fecb0e2fa41218cea76", + "source": "measurement host: shasum -a 256 saved manifest for qwen3:8b; jq model layer; jq .file_type from config blob sha256:05a61d37b08453e59290add468e3bb2f688e23a01e967fecb0e2fa41218cea76", "captured_at": "2026-08-05T18:01:51Z", "manifest_digest": "sha256:500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41", "model_layer_blob_digest": "sha256:a3de86cd1c132c822487ededd47a324c50491393e6565cd14bafa40d0b8e686f", - "gguf_metadata_source": "ssh macstudio python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-a3de86cd1c132c822487ededd47a324c50491393e6565cd14bafa40d0b8e686f; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", + "gguf_metadata_source": "measurement host, python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-a3de86cd1c132c822487ededd47a324c50491393e6565cd14bafa40d0b8e686f; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", "gguf_metadata_captured_at": "2026-08-06T02:34:30Z", "gguf_metadata": { "version": 3, @@ -102,11 +102,11 @@ "subject": "ollama-qwen3-14b.json", "model_id": "qwen3:14b", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 saved manifest for qwen3:14b; jq model layer; jq .file_type from config blob sha256:78b3b822087d5199783c8203553a5a92ce5eb7b683a5a81003f8efea9b399d74", + "source": "measurement host: shasum -a 256 saved manifest for qwen3:14b; jq model layer; jq .file_type from config blob sha256:78b3b822087d5199783c8203553a5a92ce5eb7b683a5a81003f8efea9b399d74", "captured_at": "2026-08-05T18:01:51Z", "manifest_digest": "sha256:bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8", "model_layer_blob_digest": "sha256:a8cc1361f3145dc01f6d77c6c82c9116b9ffe3c97b34716fe20418455876c40e", - "gguf_metadata_source": "ssh macstudio python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-a8cc1361f3145dc01f6d77c6c82c9116b9ffe3c97b34716fe20418455876c40e; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", + "gguf_metadata_source": "measurement host, python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-a8cc1361f3145dc01f6d77c6c82c9116b9ffe3c97b34716fe20418455876c40e; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", "gguf_metadata_captured_at": "2026-08-06T02:34:30Z", "gguf_metadata": { "version": 3, @@ -126,11 +126,11 @@ "subject": "ollama-qwen2.5-7b-instruct.json", "model_id": "qwen2.5:7b-instruct", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 saved manifest for qwen2.5:7b-instruct; jq model layer; jq .file_type from config blob sha256:2f15b3218f0552c60647ce60ada83632d2c09755b16259b13e3e4458e9ae419d", + "source": "measurement host: shasum -a 256 saved manifest for qwen2.5:7b-instruct; jq model layer; jq .file_type from config blob sha256:2f15b3218f0552c60647ce60ada83632d2c09755b16259b13e3e4458e9ae419d", "captured_at": "2026-08-05T18:01:51Z", "manifest_digest": "sha256:845dbda0ea48ed749caafd9e6037047aa19acfcfd82e704d7ca97d631a0b697e", "model_layer_blob_digest": "sha256:2bada8a7450677000f678be90653b85d364de7db25eb5ea54136ada5f3933730", - "gguf_metadata_source": "ssh macstudio python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-2bada8a7450677000f678be90653b85d364de7db25eb5ea54136ada5f3933730; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", + "gguf_metadata_source": "measurement host, python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-2bada8a7450677000f678be90653b85d364de7db25eb5ea54136ada5f3933730; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", "gguf_metadata_captured_at": "2026-08-06T02:34:30Z", "gguf_metadata": { "version": 3, @@ -151,11 +151,11 @@ "subject": "ollama-hermes3-8b.json", "model_id": "hermes3:8b", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 saved manifest for hermes3:8b; jq model layer; jq .file_type from config blob sha256:9dfddf59cf5a85558d8b782db72c12e934b9b539ae48af03861938e4c383c167", + "source": "measurement host: shasum -a 256 saved manifest for hermes3:8b; jq model layer; jq .file_type from config blob sha256:9dfddf59cf5a85558d8b782db72c12e934b9b539ae48af03861938e4c383c167", "captured_at": "2026-08-05T18:01:51Z", "manifest_digest": "sha256:4f6b83f30b62bc3d0cf9be09266db222805ee815c8fd7d8b38f863f655be78b7", "model_layer_blob_digest": "sha256:c8985d236593f7a17da2a3da49588aa951a9b1e57ce97753364fbf59e63af84a", - "gguf_metadata_source": "ssh macstudio python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-c8985d236593f7a17da2a3da49588aa951a9b1e57ce97753364fbf59e63af84a; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", + "gguf_metadata_source": "measurement host, python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-c8985d236593f7a17da2a3da49588aa951a9b1e57ce97753364fbf59e63af84a; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", "gguf_metadata_captured_at": "2026-08-06T02:34:30Z", "gguf_metadata": { "version": 3, @@ -176,11 +176,11 @@ "subject": "ollama-llama3-groq-tool-use-8b.json", "model_id": "llama3-groq-tool-use:8b", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 saved manifest for llama3-groq-tool-use:8b; jq model layer; jq .file_type from config blob sha256:c9300739186a798fe2514859db9235794bd09330c5b62b278a4c4ff34298661b", + "source": "measurement host: shasum -a 256 saved manifest for llama3-groq-tool-use:8b; jq model layer; jq .file_type from config blob sha256:c9300739186a798fe2514859db9235794bd09330c5b62b278a4c4ff34298661b", "captured_at": "2026-08-05T18:01:51Z", "manifest_digest": "sha256:36211dad2b153f09fe2b0c5f3e4e82ee83d27db68e3a00006cd934dfdc5057a4", "model_layer_blob_digest": "sha256:23e78d0ea1ab895bcbe63a12422a731cfd2b1a8bd124a1c91790b2b9c3fc9d14", - "gguf_metadata_source": "ssh macstudio python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-23e78d0ea1ab895bcbe63a12422a731cfd2b1a8bd124a1c91790b2b9c3fc9d14; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", + "gguf_metadata_source": "measurement host, python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-23e78d0ea1ab895bcbe63a12422a731cfd2b1a8bd124a1c91790b2b9c3fc9d14; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", "gguf_metadata_captured_at": "2026-08-06T02:34:30Z", "gguf_metadata": { "version": 3, @@ -198,11 +198,11 @@ "subject": "ollama-llama3.1-8b.json", "model_id": "llama3.1:8b", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 saved manifest for llama3.1:8b; jq model layer; jq .file_type from config blob sha256:455f34728c9b5dd3376378bfb809ee166c145b0b4c1f1a6feca069055066ef9a", + "source": "measurement host: shasum -a 256 saved manifest for llama3.1:8b; jq model layer; jq .file_type from config blob sha256:455f34728c9b5dd3376378bfb809ee166c145b0b4c1f1a6feca069055066ef9a", "captured_at": "2026-08-05T18:01:51Z", "manifest_digest": "sha256:46e0c10c039e019119339687c3c1757cc81b9da49709a3b3924863ba87ca666e", "model_layer_blob_digest": "sha256:667b0c1932bc6ffc593ed1d03f895bf2dc8dc6df21db3042284a6f4416b06a29", - "gguf_metadata_source": "ssh macstudio python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-667b0c1932bc6ffc593ed1d03f895bf2dc8dc6df21db3042284a6f4416b06a29; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", + "gguf_metadata_source": "measurement host, python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-667b0c1932bc6ffc593ed1d03f895bf2dc8dc6df21db3042284a6f4416b06a29; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", "gguf_metadata_captured_at": "2026-08-06T02:34:30Z", "gguf_metadata": { "version": 3, @@ -223,11 +223,11 @@ "subject": "ollama-mistral-7b.json", "model_id": "mistral:7b", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 saved manifest for mistral:7b; jq model layer; jq .file_type from config blob sha256:1064e17101bdd2460dd5c4e03e4f5cc1b38a4dee66084dc91faba294ccb64a92", + "source": "measurement host: shasum -a 256 saved manifest for mistral:7b; jq model layer; jq .file_type from config blob sha256:1064e17101bdd2460dd5c4e03e4f5cc1b38a4dee66084dc91faba294ccb64a92", "captured_at": "2026-08-05T18:01:51Z", "manifest_digest": "sha256:6577803aa9a036369e481d648a2baebb381ebc6e897f2bb9a766a2aa7bfbc1cf", "model_layer_blob_digest": "sha256:f5074b1221da0f5a2910d33b642efa5b9eb58cfdddca1c79e16d7ad28aa2b31f", - "gguf_metadata_source": "ssh macstudio python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-f5074b1221da0f5a2910d33b642efa5b9eb58cfdddca1c79e16d7ad28aa2b31f; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", + "gguf_metadata_source": "measurement host, python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-f5074b1221da0f5a2910d33b642efa5b9eb58cfdddca1c79e16d7ad28aa2b31f; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", "gguf_metadata_captured_at": "2026-08-06T02:34:30Z", "gguf_metadata": { "version": 3, @@ -245,11 +245,11 @@ "subject": "ollama-phi4-mini.json", "model_id": "phi4-mini:latest", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 saved manifest for phi4-mini:latest; jq model layer; jq .file_type from config blob sha256:8c2539a423c4df2d9d2609fba56e7bad554696b66ed0e280c56e0ef4a32498fa", + "source": "measurement host: shasum -a 256 saved manifest for phi4-mini:latest; jq model layer; jq .file_type from config blob sha256:8c2539a423c4df2d9d2609fba56e7bad554696b66ed0e280c56e0ef4a32498fa", "captured_at": "2026-08-05T18:01:51Z", "manifest_digest": "sha256:78fad5d182a7c33065e153a5f8ba210754207ba9d91973f57dffa7f487363753", "model_layer_blob_digest": "sha256:3c168af1dea0a414299c7d9077e100ac763370e5a98b3c53801a958a47f0a5db", - "gguf_metadata_source": "ssh macstudio python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-3c168af1dea0a414299c7d9077e100ac763370e5a98b3c53801a958a47f0a5db; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", + "gguf_metadata_source": "measurement host, python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-3c168af1dea0a414299c7d9077e100ac763370e5a98b3c53801a958a47f0a5db; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", "gguf_metadata_captured_at": "2026-08-06T02:34:30Z", "gguf_metadata": { "version": 3, @@ -270,11 +270,11 @@ "subject": "ollama-gemma3-4b.json", "model_id": "gemma3:4b", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 saved manifest for gemma3:4b; jq model layer; jq .file_type from config blob sha256:b6ae5839783f2ba248e65e4b960ab15f9c4b7118db285827dba6cba9754759e2", + "source": "measurement host: shasum -a 256 saved manifest for gemma3:4b; jq model layer; jq .file_type from config blob sha256:b6ae5839783f2ba248e65e4b960ab15f9c4b7118db285827dba6cba9754759e2", "captured_at": "2026-08-05T18:01:51Z", "manifest_digest": "sha256:a2af6cc3eb7fa8be8504abaf9b04e88f17a119ec3f04a3addf55f92841195f5a", "model_layer_blob_digest": "sha256:aeda25e63ebd698fab8638ffb778e68bed908b960d39d0becc650fa981609d25", - "gguf_metadata_source": "ssh macstudio python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-aeda25e63ebd698fab8638ffb778e68bed908b960d39d0becc650fa981609d25; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", + "gguf_metadata_source": "measurement host, python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-aeda25e63ebd698fab8638ffb778e68bed908b960d39d0becc650fa981609d25; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", "gguf_metadata_captured_at": "2026-08-06T02:34:30Z", "gguf_metadata": { "version": 3, @@ -288,11 +288,11 @@ "subject": "ollama-gemma3-12b.json", "model_id": "gemma3:12b", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 saved manifest for gemma3:12b; jq model layer; jq .file_type from config blob sha256:6819964c2bcf53f6dd3593f9571e91cbf2bab9665493f870f96eeb29873049b4", + "source": "measurement host: shasum -a 256 saved manifest for gemma3:12b; jq model layer; jq .file_type from config blob sha256:6819964c2bcf53f6dd3593f9571e91cbf2bab9665493f870f96eeb29873049b4", "captured_at": "2026-08-05T18:01:51Z", "manifest_digest": "sha256:f4031aab637d1ffa37b42570452ae0e4fad0314754d17ded67322e4b95836f8a", "model_layer_blob_digest": "sha256:e8ad13eff07a78d89926e9e8b882317d082ef5bf9768ad7b50fcdbbcd63748de", - "gguf_metadata_source": "ssh macstudio python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-e8ad13eff07a78d89926e9e8b882317d082ef5bf9768ad7b50fcdbbcd63748de; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", + "gguf_metadata_source": "measurement host, python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-e8ad13eff07a78d89926e9e8b882317d082ef5bf9768ad7b50fcdbbcd63748de; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", "gguf_metadata_captured_at": "2026-08-06T02:34:30Z", "gguf_metadata": { "version": 3, @@ -306,11 +306,11 @@ "subject": "ollama-granite3.1-dense-8b.json", "model_id": "granite3.1-dense:8b", "status": "recovered", - "source": "ssh macstudio: shasum -a 256 saved manifest for granite3.1-dense:8b; jq model layer; jq .file_type from config blob sha256:f9cd69f4077ded7838ccb42fb335e73d5b51dc9ab9e481c7396a9c9ed24dfa34", + "source": "measurement host: shasum -a 256 saved manifest for granite3.1-dense:8b; jq model layer; jq .file_type from config blob sha256:f9cd69f4077ded7838ccb42fb335e73d5b51dc9ab9e481c7396a9c9ed24dfa34", "captured_at": "2026-08-05T18:01:51Z", "manifest_digest": "sha256:34d3be74ec547c3e1969a9ea125e1869f052f7cfd3d0bbbb5aa5e42ec033108c", "model_layer_blob_digest": "sha256:44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8", - "gguf_metadata_source": "ssh macstudio python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", + "gguf_metadata_source": "measurement host, python3 -: read-only Python 3 parser decoded the GGUF header directly from sha256-44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8; inspected general.name, general.basename, general.organization, general.finetune, general.size_label, and general.version", "gguf_metadata_captured_at": "2026-08-06T02:34:30Z", "gguf_metadata": { "version": 3, diff --git a/tools/__pycache__/redact_local_paths.cpython-312.pyc b/tools/__pycache__/redact_local_paths.cpython-312.pyc deleted file mode 100644 index 48687ff..0000000 Binary files a/tools/__pycache__/redact_local_paths.cpython-312.pyc and /dev/null differ diff --git a/tools/__pycache__/test_redact_local_paths.cpython-312.pyc b/tools/__pycache__/test_redact_local_paths.cpython-312.pyc deleted file mode 100644 index a3daa26..0000000 Binary files a/tools/__pycache__/test_redact_local_paths.cpython-312.pyc and /dev/null differ