ci: retain JSON CLI failures for vlt transport retries

Author: mikolalysenko

scripts/backtest-vlt.py

@@ -992,13 +992,20 @@ def __init__(self, ctx, version, mode, shape_name):
         self.record = ctx['record']
         self.envelopes = []
         self.fresh_patched = {}
+        self.cli_failures = []
 
     # CLI -------------------------------------------------------------------
     def cli(self, args, cwd=None):
         command = [self.ctx['cli'], *args, '--json', '--no-telemetry']
         if self.ctx.get('patch_server_url'):
             command += ['--patch-server-url', self.ctx['patch_server_url']]
-        return run(command, cwd or self.project, self.ctx['cli_env'], self.log)
+        code, out, err = run(command, cwd or self.project, self.ctx['cli_env'], self.log)
+        if code:
+            # --json errors go to stdout, including on repeat/revert calls.
+            # Keep them on the result row so the transport retry sees them.
+            self.cli_failures.append(dict(command=str(args[0]), exitCode=code,
+                                          envelope=parse_envelope(out), stderr=tail(err, 3000)))
+        return code, out, err
 
     def patch_run(self, mode, cwd=None):
         cwd = cwd or self.project
@@ -1195,6 +1202,8 @@ def blocked_refusal(self, row, envelopes, reference):
         return clean
 
     def finish(self, row, checks, started):
+        if self.cli_failures:
+            row['cliFailures'] = self.cli_failures
         row['failingChecks'] = [k for k, v in checks.items() if v is False]
         row['notEvaluated'] = [k for k, v in checks.items() if v is None]
         row.setdefault('codes', [])

scripts/tests/test_backtest_harnesses.py

@@ -608,6 +608,29 @@ def test_snapshot_never_follows_a_link(self):
 
 
 class VltRetryTests(unittest.TestCase):
+    def test_json_cli_failures_reach_the_retry_classifier(self):
+        for phase in ('initial', 'repeat', 'rollback'):
+            for status in (504, 404):
+                with self.subTest(phase=phase, status=status), tempfile.TemporaryDirectory() as temp:
+                    cell = vlt.Cell({'out': Path(temp), 'record': {}, 'cli': 'socket-patch',
+                                     'cli_env': {}}, '1.2.0', 'hosted', 'direct')
+                    cell.project.mkdir(parents=True)
+                    row = dict(cell=cell.name, expectedVerdict='patched', passed=False, checks={})
+                    envelope = {'status': 'error',
+                                'error': f'API request failed with status {status}: error code: {status}'}
+                    with patch.object(vlt, 'run', return_value=(1, json.dumps(envelope), '')):
+                        if phase == 'initial':
+                            cell.patch_run('hosted')
+                        elif phase == 'repeat':
+                            cell.repeat(row, row['checks'], b'')
+                        else:
+                            cell.revert()
+                    with patch('sys.stdout'):
+                        cell.finish(row, row['checks'], vlt.time.time())
+                    captured = json.loads((cell.case / 'result.json').read_text())
+                    self.assertEqual(captured['cliFailures'][0]['envelope'], envelope)
+                    self.assertEqual(vlt.transient(captured), status == 504)
+
     def test_only_transport_failures_retry(self):
         self.assertFalse(vlt.transient({'matchesExpectation': True, 'serveProbe': {'curlExit': 7}}))
         self.assertTrue(vlt.transient({'serveProbe': {'curlExit': 7}}))