Improve output parsing

bin/run.sh

@@ -68,7 +68,7 @@ emit_compile_error() {
 build_tests_json() {
     printf '%s' "${test_output}" | jq -Rs '
         def test_name: capture("test\\.(?<n>.+)\\.\\.\\.") | .n;
-        def user_output: capture("\\.\\.\\.((?<o>.+)$)") | .o // "";
+        def user_output: (capture("\\.\\.\\.(?<o>.+)$") | .o) // "";
         def is_test_line: test("[0-9]+/[0-9]+ .*\\.test\\..*\\.\\.\\.");
         def is_summary: startswith("All ") or test("[0-9]+ passed;");
         def is_ok: . == "OK" or endswith("OK");
@@ -124,7 +124,8 @@ build_tests_json() {
                 | ($header_output
                    + (if ($extra_output | length) > 0
                       then "\n" + ($extra_output | join("\n"))
-                      else "" end)) as $output
+                      else "" end)
+                   | sub("^\n"; "")) as $output
                 | if $from_status[0] | is_ok then
                     pass_record($name) | .output = $output
                 else

tests/example-memory-leak/example_memory_leak.zig

@@ -0,0 +1,11 @@
+const std = @import("std");
+const mem = std.mem;
+
+pub const Pair = struct { a: []u8, b: []u8 };
+
+// Buggy: on failure of the second alloc, the first alloc is never freed.
+pub fn makePair(allocator: mem.Allocator) !Pair {
+    const first = try allocator.alloc(u8, 16);
+    const second = try allocator.alloc(u8, 16);
+    return .{ .a = first, .b = second };
+}

tests/example-memory-leak/expected_results.json

@@ -0,0 +1,16 @@
+{
+  "version": 2,
+  "status": "fail",
+  "tests": [
+    {
+      "name": "makePair cleans up on allocation failure",
+      "status": "fail",
+      "message": "FAIL (MemoryLeakDetected)\n/opt/zig/lib/std/mem/Allocator.zig:300:82: <MEMHASH> in allocBytesWithAlignment__anon_<ANON> (std.zig)\n    const byte_ptr = self.rawAlloc(byte_count, alignment, return_address) orelse return error.OutOfMemory;\n                                                                                 ^\n/opt/zig/lib/std/mem/Allocator.zig:286:5: <MEMHASH> in allocWithSizeAndAlignment__anon_<ANON> (std.zig)\n    return self.allocBytesWithAlignment(alignment, byte_count, return_address);\n    ^\n/opt/zig/lib/std/mem/Allocator.zig:274:55: <MEMHASH> in allocAdvancedWithRetAddr (std.zig)\n    const ptr: [*]align(a.toByteUnits()) T = @ptrCast(try self.allocWithSizeAndAlignment(@sizeOf(T), a, n, return_address));\n                                                      ^\n/opt/zig/lib/std/mem/Allocator.zig:198:5: <MEMHASH> in alloc__anon_<ANON> (std.zig)\n    return self.allocAdvancedWithRetAddr(T, null, n, @returnAddress());\n    ^\nexample_memory_leak.zig:9:20: <MEMHASH> in makePair (test_example_memory_leak.zig)\n    const second = try allocator.alloc(u8, 16);\n                   ^\ntest_example_memory_leak.zig:7:15: <MEMHASH> in pairTest (test_example_memory_leak.zig)\n    const p = try mod.makePair(allocator);\n              ^\n/opt/zig/lib/std/testing.zig:1156:21: <MEMHASH> in checkAllAllocationFailures__anon_<ANON> (std.zig)\n                    return error.MemoryLeakDetected;\n                    ^\ntest_example_memory_leak.zig:13:5: <MEMHASH> in test.makePair cleans up on allocation failure (test_example_memory_leak.zig)\n    try std.testing.checkAllAllocationFailures(\n    ^\n[DebugAllocator] (err): memory address <MEMHASH> leaked: \nexample_memory_leak.zig:8:38: <MEMHASH> in makePair (test_example_memory_leak.zig)\n    const first = try allocator.alloc(u8, 16);\n                                     ^\ntest_example_memory_leak.zig:7:31: <MEMHASH> in pairTest (test_example_memory_leak.zig)\n    const p = try mod.makePair(allocator);\n                              ^\n/opt/zig/lib/std/testing.zig:1133:69: <MEMHASH> in checkAllAllocationFailures__anon_<ANON> (std.zig)\n        if (@call(.auto, test_fn, .{failing_allocator_inst.allocator()} ++ extra_args)) |_| {\n                                                                    ^\ntest_example_memory_leak.zig:13:47: <MEMHASH> in test.makePair cleans up on allocation failure (test_example_memory_leak.zig)\n    try std.testing.checkAllAllocationFailures(\n                                              ^\n/opt/zig/lib/compiler/test_runner.zig:291:25: <MEMHASH> in mainTerminal (test_runner.zig)\n        if (test_fn.func()) |_| {\n                        ^\n/opt/zig/lib/compiler/test_runner.zig:73:28: <MEMHASH> in main (test_runner.zig)\n        return mainTerminal(init);\n                           ^\n/opt/zig/lib/std/start.zig:699:88: <MEMHASH> in callMain (std.zig)\n    if (fn_info.params[0].type.? == std.process.Init.Minimal) return wrapMain(root.main(.{\n                                                                                       ^\n/opt/zig/lib/std/start.zig:190:5: <MEMHASH> in _start (std.zig)\n    asm volatile (switch (native_arch) {\n    ^",
+      "output": "fail_index: 1/2\nallocated bytes: 16\nfreed bytes: 0\nallocations: 1\ndeallocations: 0\nallocation that was made to fail: \nexample_memory_leak.zig:9:39: <MEMHASH> in makePair (test_example_memory_leak.zig)\n    const second = try allocator.alloc(u8, 16);\n                                      ^\ntest_example_memory_leak.zig:7:31: <MEMHASH> in pairTest (test_example_memory_leak.zig)\n    const p = try mod.makePair(allocator);\n                              ^\n/opt/zig/lib/std/testing.zig: [output truncated]"
+    },
+    {
+      "name": "sanity pass",
+      "status": "pass"
+    }
+  ]
+}

tests/example-memory-leak/test_example_memory_leak.zig

@@ -0,0 +1,22 @@
+const std = @import("std");
+const testing = std.testing;
+
+const mod = @import("example_memory_leak.zig");
+
+fn pairTest(allocator: std.mem.Allocator) anyerror!void {
+    const p = try mod.makePair(allocator);
+    defer allocator.free(p.a);
+    defer allocator.free(p.b);
+}
+
+test "makePair cleans up on allocation failure" {
+    try std.testing.checkAllAllocationFailures(
+        std.testing.allocator,
+        pairTest,
+        .{},
+    );
+}
+
+test "sanity pass" {
+    try std.testing.expect(1 + 1 == 2);
+}

tests/example-string-mismatch/example_string_mismatch.zig

@@ -0,0 +1,3 @@
+pub fn greeting() []const u8 {
+    return "Goodbye, Mars!";
+}

tests/example-string-mismatch/expected_results.json

@@ -0,0 +1,16 @@
+{
+  "version": 2,
+  "status": "fail",
+  "tests": [
+    {
+      "name": "sanity pass",
+      "status": "pass"
+    },
+    {
+      "name": "greeting is exactly Hello, World!",
+      "status": "fail",
+      "message": "FAIL (TestExpectedEqual)\n/opt/zig/lib/std/testing.zig:682:9: <MEMHASH> in expectEqualStrings (std.zig)\n        return error.TestExpectedEqual;\n        ^\ntest_example_string_mismatch.zig:11:5: <MEMHASH> in test.greeting is exactly Hello, World! (test_example_string_mismatch.zig)\n    try testing.expectEqualStrings(\"Hello, World!\", mod.greeting());\n    ^",
+      "output": "====== expected this output: =========\nHello, World!␃\n\n======== instead found this: =========\nGoodbye, Mars!␃\n\n======================================\nFirst difference occurs on line 1:\nexpected:\nHello, World!\n^ ('\\x48')\nfound:\nGoodbye, Mars!\n^ ('\\x47')"
+    }
+  ]
+}

tests/example-string-mismatch/test_example_string_mismatch.zig

@@ -0,0 +1,12 @@
+const std = @import("std");
+const testing = std.testing;
+
+const mod = @import("example_string_mismatch.zig");
+
+test "sanity pass" {
+    try testing.expect(1 + 1 == 2);
+}
+
+test "greeting is exactly Hello, World!" {
+    try testing.expectEqualStrings("Hello, World!", mod.greeting());
+}