InferRSI/docs/examples/skill-comparison/report-example.json at main · InferMatrixLab/InferRSI · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
{
  "artifact": "Generated offline report excerpt; full evidence is reproduced by the demo",
  "claim": "Pipeline results only; fixed-output offline clients do not establish skill efficacy. Discovery is separate. Unknown usage and cost remain unknown.",
  "mode": "update",
  "skill": "selected",
  "revisions": {
    "baseline": {
      "git_revision": "953aa2ac39e46573caedfff5179a3e5d36c9be3e",
      "bundle_revision": "sha256:db78dfd40c109c9e99bfefe5006812f2e492d4002c9f5a256f4ec424bed0b59d"
    },
    "candidate": {
      "git_revision": "c269dc05581f58232de90ee5958098620d2aa9cf",
      "bundle_revision": "sha256:7fbb9ba7387ce05ec0ef18b559fbb914b2c2371c537e19c2bceddf18cad334ae"
    }
  },
  "model": {
    "name": "offline-fixture",
    "effort": "test"
  },
  "limits": {
    "repeats": 1,
    "max_task_calls": 8,
    "timeout_seconds": 60,
    "campaign_seconds": 900,
    "token_stop_new_trials": 120
  },
  "exam": {
    "cases": "b6b928910c61938f2b8ed0dcaa07df33796e6bfdc65a9b5df2271254b3d496f9",
    "schema": "89de6085fa63da835e0e6653785771d02abefd665cb2b21ad2df772ffc0b4f62",
    "grader": "a11ef015770706aa33e4c801dcf31c0bb08870c80dc8aa5b8a713d049f16b7ff"
  },
  "frozen_controllers": {
    "tools/skill_eval_compare.py": "8a8b35a40d8dbc0c203b2867cbcd02357ca77d1f63eaf6b0925ca1b0e211600d",
    "tools/skill_eval_pilot.py": "c7ec6c91463df119c9a8adffd772a799fbfdd371c6d25d97740a4b713e0f15aa",
    "src/inferrsi/harness/skill_evaluation.py": "8af9faf0fd083aa2731d284cebcf8caed135d6b3723bf1c869d1d79f81fde19d",
    "src/inferrsi/infra/executors/local.py": "b089cdc7daaaf7834698c2d280a12c5e755915ded926aa039c8de5f521bd46f0",
    "src/inferrsi/infra/processes.py": "3b96ebc7a50913e78e86ff3dcba74c8ee8d03d547d9591ca6488f390ac1e6189",
    "src/inferrsi/infra/artifacts.py": "e72e1bebdbb88e42de430238489252386851f9e6645ed7780f2aad2e90338120"
  },
  "counts": {
    "fix": 1,
    "regression": 1,
    "unchanged_correct": 0,
    "unchanged_incorrect": 0,
    "missing_pair": 1,
    "inconclusive": 1
  },
  "pairs": [
    {
      "case_id": "fix",
      "repeat": 1,
      "outcome": "fix",
      "baseline": {
        "verdict": "FAIL",
        "evaluation_error": null,
        "output": "{\"report_verdict\": \"PASS\", \"claim_scope\": \"production\", \"findings\": [], \"latency_reduction_pct\": 20, \"evidence\": [\"baseline.latencies_ms\", \"candidate.latencies_ms\", \"contract\"], \"reason\": \"Group means are 100/200/300 and 50/160/900: medians 200 and 160 give exactly 20 percent.\"}",
        "checks": {
          "passed": false,
          "assertions": {
            "report_verdict": true,
            "claim_scope": false,
            "findings": true,
            "reduction": true,
            "evidence": true,
            "explanation": true
          }
        },
        "output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
        "retained_output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
        "evidence_sha256": {
          "execution.json": "a6eab3e94000c3aade3ac1cbf5e88370522d91d57e4b28c100ccdbd6e1db8ace",
          "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
          "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",
          "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        },
        "retained_evidence_sha256": {
          "execution.json": "a6eab3e94000c3aade3ac1cbf5e88370522d91d57e4b28c100ccdbd6e1db8ace",
          "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
          "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",
          "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        }
      },
      "candidate": {
        "verdict": "PASS",
        "evaluation_error": null,
        "output": "{\"report_verdict\": \"PASS\", \"claim_scope\": \"fixed_workload\", \"findings\": [], \"latency_reduction_pct\": 20, \"evidence\": [\"baseline.latencies_ms\", \"candidate.latencies_ms\", \"contract\"], \"reason\": \"Group means are 100/200/300 and 50/160/900: medians 200 and 160 give exactly 20 percent.\"}",
        "checks": {
          "passed": true,
          "assertions": {
            "report_verdict": true,
            "claim_scope": true,
            "findings": true,
            "reduction": true,
            "evidence": true,
            "explanation": true
          }
        },
        "output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
        "retained_output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
        "evidence_sha256": {
          "execution.json": "a68d8651dea2a336c856639eb0122c4c0946cef05eb0ecfd6bea8e69bd752209",
          "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
          "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
          "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        },
        "retained_evidence_sha256": {
          "execution.json": "a68d8651dea2a336c856639eb0122c4c0946cef05eb0ecfd6bea8e69bd752209",
          "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
          "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
          "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        }
      }
    },
    {
      "case_id": "regression",
      "repeat": 1,
      "outcome": "regression",
      "baseline": {
        "verdict": "PASS",
        "evaluation_error": null,
        "output": "{\"report_verdict\": \"PASS\", \"claim_scope\": \"fixed_workload\", \"findings\": [], \"latency_reduction_pct\": 20, \"evidence\": [\"baseline.latencies_ms\", \"candidate.latencies_ms\", \"contract\"], \"reason\": \"Group means are 100/200/300 and 50/160/900: medians 200 and 160 give exactly 20 percent.\"}",
        "checks": {
          "passed": true,
          "assertions": {
            "report_verdict": true,
            "claim_scope": true,
            "findings": true,
            "reduction": true,
            "evidence": true,
            "explanation": true
          }
        },
        "output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
        "retained_output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
        "evidence_sha256": {
          "execution.json": "ff72cb6a28db5adf030609cbdd285a391475cee2dee9c8a965565238020fb172",
          "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
          "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
          "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        },
        "retained_evidence_sha256": {
          "execution.json": "ff72cb6a28db5adf030609cbdd285a391475cee2dee9c8a965565238020fb172",
          "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
          "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
          "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        }
      },
      "candidate": {
        "verdict": "FAIL",
        "evaluation_error": null,
        "output": "{\"report_verdict\": \"PASS\", \"claim_scope\": \"production\", \"findings\": [], \"latency_reduction_pct\": 20, \"evidence\": [\"baseline.latencies_ms\", \"candidate.latencies_ms\", \"contract\"], \"reason\": \"Group means are 100/200/300 and 50/160/900: medians 200 and 160 give exactly 20 percent.\"}",
        "checks": {
          "passed": false,
          "assertions": {
            "report_verdict": true,
            "claim_scope": false,
            "findings": true,
            "reduction": true,
            "evidence": true,
            "explanation": true
          }
        },
        "output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
        "retained_output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
        "evidence_sha256": {
          "execution.json": "9191efb9ae1b040d0eeef520aeadbe91db378b461132d22e705e9c8e479cf0c6",
          "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
          "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",
          "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        },
        "retained_evidence_sha256": {
          "execution.json": "9191efb9ae1b040d0eeef520aeadbe91db378b461132d22e705e9c8e479cf0c6",
          "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
          "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",
          "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        }
      }
    },
    {
      "case_id": "error",
      "repeat": 1,
      "outcome": "inconclusive",
      "baseline": {
        "verdict": "PASS",
        "evaluation_error": null,
        "output": "{\"report_verdict\": \"PASS\", \"claim_scope\": \"fixed_workload\", \"findings\": [], \"latency_reduction_pct\": 20, \"evidence\": [\"baseline.latencies_ms\", \"candidate.latencies_ms\", \"contract\"], \"reason\": \"Group means are 100/200/300 and 50/160/900: medians 200 and 160 give exactly 20 percent.\"}",
        "checks": {
          "passed": true,
          "assertions": {
            "report_verdict": true,
            "claim_scope": true,
            "findings": true,
            "reduction": true,
            "evidence": true,
            "explanation": true
          }
        },
        "output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
        "retained_output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
        "evidence_sha256": {
          "execution.json": "9b989e4d07a49ece73426c7aa83dddfb0b881e4fc46d77ef3668aad6c3cd44a3",
          "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
          "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
          "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        },
        "retained_evidence_sha256": {
          "execution.json": "9b989e4d07a49ece73426c7aa83dddfb0b881e4fc46d77ef3668aad6c3cd44a3",
          "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
          "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
          "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
          "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        }
      },
      "candidate": {
        "verdict": "INCONCLUSIVE",
        "evaluation_error": "invalid_output_json",
        "output": "{broken",
        "checks": null,
        "output_sha256": "be302461cda285db3ed2d45fb767b23e08ca8808b35a638009937df22b332d73",
        "retained_output_sha256": "be302461cda285db3ed2d45fb767b23e08ca8808b35a638009937df22b332d73",
        "evidence_sha256": {
          "execution.json": "abff190d9105af18941244f1469515f219c610e590263d9b2f5a59414b07d4b4",
          "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
          "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        },
        "retained_evidence_sha256": {
          "execution.json": "abff190d9105af18941244f1469515f219c610e590263d9b2f5a59414b07d4b4",
          "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
          "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        }
      }
    },
    {
      "case_id": "missing",
      "repeat": 1,
      "outcome": "missing_pair",
      "baseline": {
        "verdict": null,
        "evaluation_error": null,
        "output": null,
        "checks": null,
        "output_sha256": null,
        "retained_output_sha256": null,
        "evidence_sha256": null,
        "retained_evidence_sha256": null
      },
      "candidate": {
        "verdict": null,
        "evaluation_error": null,
        "output": null,
        "checks": null,
        "output_sha256": null,
        "retained_output_sha256": null,
        "evidence_sha256": null,
        "retained_evidence_sha256": null
      }
    }
  ],
  "record_integrity_control": {
    "mutation": "Only candidate result.json verdict/checks changed; grader evidence unchanged",
    "verdict": "INCONCLUSIVE",
    "checks": null,
    "evaluation_error": "retained_trial_record_hash_mismatch",
    "counts": {
      "fix": 1,
      "regression": 0,
      "unchanged_correct": 0,
      "unchanged_incorrect": 0,
      "missing_pair": 1,
      "inconclusive": 2
    },
    "extra_model_calls": 0,
    "reports_identical_after_restore": true
  },
  "status": {
    "client_runtime": {
      "implementation": "tests.fixtures.skill_eval_compare_support.FixedOutputClient",
      "executable": null,
      "executable_sha256": null,
      "version": null
    },
    "recorded_trials": 6,
    "expected_trials": 8,
    "observed_tokens": 120,
    "stop_reason": "token_budget",
    "execution_state": "stopped"
  },
  "status_integrity_control": {
    "mutation": "Only results/status.json changed",
    "status": {
      "execution_state": "unverified",
      "evaluation_error": "retained_status_record_hash_mismatch",
      "expected_trials": 8,
      "recorded_trials": null,
      "observed_tokens": null,
      "stop_reason": null,
      "client_runtime": null
    },
    "extra_model_calls": 0,
    "reports_identical_after_restore": true
  }
}