Implement interface version 3

Dockerfile

@@ -21,7 +21,7 @@ FROM alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6ee
 
 # install packages required to run the tests
 # hadolint ignore=DL3018
-RUN apk add --no-cache bash jq
+RUN apk add --no-cache bash gawk jq
 
 RUN addgroup ziggroup \
     && adduser --disabled-password --gecos ziggy --ingroup ziggroup ziggy
@@ -30,7 +30,7 @@ ENV PATH=$PATH:/opt/zig
 
 USER ziggy:ziggroup
 WORKDIR /opt/test-runner
-COPY --chown=ziggy:ziggroup bin/run.sh bin/run.sh
+COPY --chown=ziggy:ziggroup bin/run.sh bin/test-metadata.awk bin/
 # Initialize a zig cache
 COPY --chown=ziggy:ziggroup tests/example-success/example_success.zig init-zig-cache/
 COPY --chown=ziggy:ziggroup tests/example-success/test_example_success.zig init-zig-cache/

bin/run.sh

@@ -9,7 +9,7 @@
 # $3: path to output directory
 
 # Output:
-# Writes a v2 results.json to the output directory, per
+# Writes a v3 results.json to the output directory, per
 # https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md
 
 # Example:
@@ -39,17 +39,21 @@ run_zig_test() {
     local raw_output zig_exit
     raw_output=$(cd "${solution_dir}" && zig test -target x86_64-linux-musl "${test_file}" 2>&1)
     zig_exit=$?
+    # When the test binary dies mid-test, zig appends its trailer to the
+    # unfinished header line. Split the two apart before dropping the trailer,
+    # so the aborted test is still reported instead of vanishing with it.
     printf '%s' "${raw_output}" \
         | sed -e "s#${solution_dir}/\{0,1\}##g" \
-              -e '/error: the following test command failed/,$d'
+              -e 's/error: the following test command failed/\n&/' \
+        | sed -e '/error: the following test command failed/,$d'
     return "${zig_exit}"
 }
 
 # Emit a top-level error report (compile failure) and exit successfully —
 # the runner completed its job even though the solution did not build.
 emit_compile_error() {
     jq -n --arg message "${test_output}" \
-        '{version: 2, status: "error", message: $message}' > "${results_file}"
+        '{version: 3, status: "error", message: $message}' > "${results_file}"
     echo "${slug}: done"
     exit 0
 }
@@ -67,12 +71,17 @@ emit_compile_error() {
 # `classify` turns each block into a single JSON test record.
 build_tests_json() {
     printf '%s' "${test_output}" | jq -Rs '
-        def test_name: capture("test\\.(?<n>.+)\\.\\.\\.") | .n;
+        # Zig prefixes each test with the module it was declared in. That
+        # prefix is matched lazily, so a module whose own name ends in
+        # "test" is not mistaken for the ".test." separator.
+        def test_name: capture("^[0-9]+/[0-9]+ .+?\\.test\\.(?<n>.+)\\.\\.\\.") | .n;
         def user_output: (capture("\\.\\.\\.(?<o>.+)$") | .o) // "";
         def is_test_line: test("[0-9]+/[0-9]+ .*\\.test\\..*\\.\\.\\.");
         def is_summary: startswith("All ") or test("[0-9]+ passed;");
-        def is_ok: . == "OK" or endswith("OK");
-        def is_fail: startswith("FAIL") or test("\\.\\.\\.(FAIL|expected )");
+        # Null-tolerant: a run cut off mid-report leaves a test with no
+        # status line at all, and that test must still be reported.
+        def is_ok: (. // "") | (. == "OK" or endswith("OK"));
+        def is_fail: (. // "") | (startswith("FAIL") or test("\\.\\.\\.(FAIL|expected )"));
 
         # Per-outcome record constructors. These set the leading fields;
         # classify extends records via `.field = ...` in the same order,
@@ -97,7 +106,7 @@ build_tests_json() {
                     .current += [$line]
                 else . end
             )
-            | if .current then .groups += [.current] else .groups end;
+            | if .current then .groups + [.current] else .groups end;
 
         # Pass 2 — turn one line group into one test record.
         # Three shapes handled:
@@ -144,15 +153,44 @@ build_tests_json() {
     '
 }
 
+# Read what the test file itself says about each test — the body of its
+# test block, and the task it belongs to — keyed by test name.
+build_metadata_json() {
+    local path="${solution_dir}/${test_file}"
+    [[ -f "${path}" ]] || { echo '{}'; return; }
+
+    gawk -f ./bin/test-metadata.awk "${path}" | jq -Rs '
+        [ split("\u001e")[1:][]
+          | split("\u001f")
+          | {key: .[1], value: (
+                (if .[2] == "" then {} else {test_code: .[2]} end)
+                + (if .[0] == "" then {} else {task_id: (.[0] | tonumber)} end)
+            )}
+        ] | from_entries
+    '
+}
+
 # Write the final results.json. Truncates each test's "output" field to
-# 500 chars to bound report size.
+# 500 chars to bound report size, and folds in the metadata read from the
+# test file: "test_code" directly after the name, "task_id" last.
+#
+# A test the student added to their own solution file is simply absent from
+# the metadata, and so keeps its result without gaining either field.
 assemble_report() {
     local overall="$1"
     local tests_json="$2"
-    jq -n --arg status "${overall}" --argjson tests "${tests_json}" '
+    local metadata="$3"
+    jq -n --arg status "${overall}" --argjson tests "${tests_json}" \
+          --argjson metadata "${metadata}" '
         def trunc: if length > 500 then .[:481] + " [output truncated]" else . end;
-        {version: 2, status: $status, tests: ($tests | map(
-            if .output then .output |= trunc else . end
+        def with_metadata($meta):
+            {name}
+            + (if $meta.test_code then {test_code: $meta.test_code} else {} end)
+            + del(.name)
+            + (if $meta.task_id then {task_id: $meta.task_id} else {} end);
+        {version: 3, status: $status, tests: ($tests | map(
+            (if .output then .output |= trunc else . end)
+            | with_metadata($metadata[.name] // {})
         ))}
     ' > "${results_file}"
 }
@@ -164,18 +202,21 @@ main() {
 
     local any_failed=0
     test_output=$(run_zig_test) || any_failed=1
-    if (( any_failed )) && [[ "${test_output}" = *error:* ]]; then
+
+    local tests_json metadata overall
+    tests_json=$(build_tests_json)
+
+    if (( any_failed )) && [[ "${tests_json}" == "[]" ]]; then
         emit_compile_error
     fi
 
-    local tests_json overall
-    tests_json=$(build_tests_json)
+    metadata=$(build_metadata_json)
     if (( any_failed == 0 )); then
         overall="pass"
     else
         overall="fail"
     fi
-    assemble_report "${overall}" "${tests_json}"
+    assemble_report "${overall}" "${tests_json}" "${metadata}"
 
     echo "${slug}: done"
 }

bin/test-metadata.awk

@@ -0,0 +1,117 @@
+#!/usr/bin/gawk -f
+
+# Synopsis:
+# Extract per-test metadata from a Zig test file, for version 3 of
+# https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md
+
+# Two things are picked up for each test: the body of its test block, which
+# is reported as "test_code", and the task it belongs to, which concept
+# exercise test files mark with a comment:
+#
+#     // task 1
+#     test "expected minutes in oven" { ... }
+#
+# A marker applies to every test that follows it, until the next marker.
+# Tests declared before the first marker — which, in a practice exercise,
+# means all of them — are left unlinked.
+
+# Output:
+# One record per test, in declaration order:
+#
+#     <RS> task id <US> name <US> test code
+#
+# where RS is 0x1e and US is 0x1f, and the task id is empty when the test is
+# not linked to one. Emitting delimited text rather than JSON keeps every
+# string escaping question in jq, where it is correct by construction.
+
+# Requires GNU awk, for the third argument to match().
+
+BEGIN {
+    # The escape sequences Zig and JSON share; any other escape stands for
+    # the character it precedes. Numeric escapes (\x41, \u{1F600}) are not
+    # understood, so a test named with one is left unlinked rather than
+    # linked to the wrong task.
+    unescaped["n"] = "\n"
+    unescaped["t"] = "\t"
+    unescaped["r"] = "\r"
+}
+
+# Turn the body of a Zig string literal into the text it denotes, which is
+# what zig prints when it runs the test, and so what we match against.
+function unescape(s,    out, c) {
+    while (s != "") {
+        c = substr(s, 1, 1)
+        s = substr(s, 2)
+        if (c != "\\") {
+            out = out c
+            continue
+        }
+        c = substr(s, 1, 1)
+        s = substr(s, 2)
+        out = out (c in unescaped ? unescaped[c] : c)
+    }
+    return out
+}
+
+# Return the index of the closing quote of the literal opening at `i`.
+function skip_literal(line, i, quote,    c) {
+    while (i <= length(line)) {
+        c = substr(line, i, 1)
+        if (c == "\\") i++
+        else if (c == quote) return i
+        i++
+    }
+    return i
+}
+
+# Advance the brace depth across one line of Zig source, ignoring braces
+# inside comments, character literals and string literals. Tracking depth
+# rather than trusting `zig fmt` matters because students may edit and
+# submit the test file of a practice exercise.
+function scan_braces(line, depth,    i, c) {
+    for (i = 1; i <= length(line); i++) {
+        c = substr(line, i, 1)
+        # Line comments and multiline string literals run to end of line.
+        if (c == "/" && substr(line, i + 1, 1) == "/") break
+        if (c == "\\" && substr(line, i + 1, 1) == "\\") break
+        if (c == "\"" || c == "'") i = skip_literal(line, i + 1, c)
+        else if (c == "{") depth++
+        else if (c == "}") depth--
+    }
+    return depth
+}
+
+# Join the captured body lines, stripping the indentation they share and
+# any blank lines top and bottom.
+function dedent(count,    i, margin, out) {
+    margin = -1
+    for (i = 1; i <= count; i++) {
+        if (body[i] ~ /^[[:space:]]*$/) continue
+        match(body[i], /^[[:space:]]*/)
+        if (margin < 0 || RLENGTH < margin) margin = RLENGTH
+    }
+    if (margin < 0) return ""
+    for (i = 1; i <= count; i++) out = out (i == 1 ? "" : "\n") substr(body[i], margin + 1)
+    sub(/^\n+/, "", out)
+    sub(/\n+$/, "", out)
+    return out
+}
+
+match($0, /^[[:space:]]*\/\/[[:space:]]*[Tt]ask[[:space:]]+([0-9]+)/, marker) {
+    task = marker[1]
+    next
+}
+
+# A test block: capture the lines up to the "}" that closes it. The closing
+# line is left out, as is the header, so what remains is the body.
+match($0, /^[[:space:]]*test[[:space:]]+"((\\.|[^"\\])*)"/, decl) {
+    depth = scan_braces($0, 0)
+    count = 0
+    while (depth > 0 && (getline line) > 0) {
+        depth = scan_braces(line, depth)
+        if (depth > 0) body[++count] = line
+    }
+    printf "%c%s%c%s%c%s", 30, task, 31, unescape(decl[1]), 31, dedent(count)
+    delete body
+    next
+}

tests/example-all-fail/expected_results.json

@@ -1,49 +1,58 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "fail",
   "tests": [
     {
       "name": "year not divisible by 4 in common year",
+      "test_code": "try testing.expect(!leap.leap(2015));",
       "status": "fail",
       "message": "/opt/zig/lib/std/testing.zig:615:14: <MEMHASH> in expect (std.zig)\n    if (!ok) return error.TestUnexpectedResult;\n             ^\ntest_example_all_fail.zig:7:5: <MEMHASH> in test.year not divisible by 4 in common year (test_example_all_fail.zig)\n    try testing.expect(!leap.leap(2015));\n    ^"
     },
     {
       "name": "year divisible by 2, not divisible by 4 in common year",
+      "test_code": "try testing.expect(!leap.leap(1970));",
       "status": "fail",
       "message": "/opt/zig/lib/std/testing.zig:615:14: <MEMHASH> in expect (std.zig)\n    if (!ok) return error.TestUnexpectedResult;\n             ^\ntest_example_all_fail.zig:11:5: <MEMHASH> in test.year divisible by 2, not divisible by 4 in common year (test_example_all_fail.zig)\n    try testing.expect(!leap.leap(1970));\n    ^"
     },
     {
       "name": "year divisible by 4, not divisible by 100 in leap year",
+      "test_code": "try testing.expect(leap.leap(1996));",
       "status": "fail",
       "message": "/opt/zig/lib/std/testing.zig:615:14: <MEMHASH> in expect (std.zig)\n    if (!ok) return error.TestUnexpectedResult;\n             ^\ntest_example_all_fail.zig:15:5: <MEMHASH> in test.year divisible by 4, not divisible by 100 in leap year (test_example_all_fail.zig)\n    try testing.expect(leap.leap(1996));\n    ^"
     },
     {
       "name": "year divisible by 4 and 5 is still a leap year",
+      "test_code": "try testing.expect(leap.leap(1960));",
       "status": "fail",
       "message": "/opt/zig/lib/std/testing.zig:615:14: <MEMHASH> in expect (std.zig)\n    if (!ok) return error.TestUnexpectedResult;\n             ^\ntest_example_all_fail.zig:19:5: <MEMHASH> in test.year divisible by 4 and 5 is still a leap year (test_example_all_fail.zig)\n    try testing.expect(leap.leap(1960));\n    ^"
     },
     {
       "name": "year divisible by 100, not divisible by 400 in common year",
+      "test_code": "try testing.expect(!leap.leap(2100));",
       "status": "fail",
       "message": "/opt/zig/lib/std/testing.zig:615:14: <MEMHASH> in expect (std.zig)\n    if (!ok) return error.TestUnexpectedResult;\n             ^\ntest_example_all_fail.zig:23:5: <MEMHASH> in test.year divisible by 100, not divisible by 400 in common year (test_example_all_fail.zig)\n    try testing.expect(!leap.leap(2100));\n    ^"
     },
     {
       "name": "year divisible by 100 but not by 3 is still not a leap year",
+      "test_code": "try testing.expect(!leap.leap(1900));",
       "status": "fail",
       "message": "/opt/zig/lib/std/testing.zig:615:14: <MEMHASH> in expect (std.zig)\n    if (!ok) return error.TestUnexpectedResult;\n             ^\ntest_example_all_fail.zig:27:5: <MEMHASH> in test.year divisible by 100 but not by 3 is still not a leap year (test_example_all_fail.zig)\n    try testing.expect(!leap.leap(1900));\n    ^"
     },
     {
       "name": "year divisible by 400 is leap year",
+      "test_code": "try testing.expect(leap.leap(2000));",
       "status": "fail",
       "message": "/opt/zig/lib/std/testing.zig:615:14: <MEMHASH> in expect (std.zig)\n    if (!ok) return error.TestUnexpectedResult;\n             ^\ntest_example_all_fail.zig:31:5: <MEMHASH> in test.year divisible by 400 is leap year (test_example_all_fail.zig)\n    try testing.expect(leap.leap(2000));\n    ^"
     },
     {
       "name": "year divisible by 400 but not by 125 is still a leap year",
+      "test_code": "try testing.expect(leap.leap(2400));",
       "status": "fail",
       "message": "/opt/zig/lib/std/testing.zig:615:14: <MEMHASH> in expect (std.zig)\n    if (!ok) return error.TestUnexpectedResult;\n             ^\ntest_example_all_fail.zig:35:5: <MEMHASH> in test.year divisible by 400 but not by 125 is still a leap year (test_example_all_fail.zig)\n    try testing.expect(leap.leap(2400));\n    ^"
     },
     {
       "name": "year divisible by 200, not divisible by 400 in common year",
+      "test_code": "try testing.expect(!leap.leap(1800));",
       "status": "fail",
       "message": "/opt/zig/lib/std/testing.zig:615:14: <MEMHASH> in expect (std.zig)\n    if (!ok) return error.TestUnexpectedResult;\n             ^\ntest_example_all_fail.zig:39:5: <MEMHASH> in test.year divisible by 200, not divisible by 400 in common year (test_example_all_fail.zig)\n    try testing.expect(!leap.leap(1800));\n    ^"
     }

tests/example-anagram-success/expected_results.json

@@ -1,34 +1,40 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "pass",
   "tests": [
     {
       "name": "no matches",
+      "test_code": "const expected = [_][]const u8{};\nconst word = \"diaper\";\nconst candidates = [_][]const u8{ \"hello\", \"world\", \"zombies\", \"pants\" };\ntry std.testing.checkAllAllocationFailures(\n    std.testing.allocator,\n    testAnagrams,\n    .{ &expected, word, &candidates },\n);",
       "status": "pass",
       "output": "hello\nworld\nzombies\npants"
     },
     {
       "name": "detects two anagrams",
+      "test_code": "const expected = [_][]const u8{ \"lemons\", \"melons\" };\nconst word = \"solemn\";\nconst candidates = [_][]const u8{ \"lemons\", \"cherry\", \"melons\" };\ntry std.testing.checkAllAllocationFailures(\n    std.testing.allocator,\n    testAnagrams,\n    .{ &expected, word, &candidates },\n);",
       "status": "pass",
       "output": "lemons\ncherry\nmelons\nlemons\ncherry\nmelons\nlemons\ncherry\nmelons\nlemons\ncherry\nmelons"
     },
     {
       "name": "detects three anagrams",
+      "test_code": "const expected = [_][]const u8{ \"gallery\", \"regally\", \"largely\" };\nconst word = \"allergy\";\nconst candidates = [_][]const u8{ \"gallery\", \"ballerina\", \"regally\", \"clergy\", \"largely\", \"leading\" };\ntry std.testing.checkAllAllocationFailures(\n    std.testing.allocator,\n    testAnagrams,\n    .{ &expected, word, &candidates },\n);",
       "status": "pass",
       "output": "gallery\nballerina\nregally\nclergy\nlargely\nleading\ngallery\nballerina\nregally\nclergy\nlargely\nleading\ngallery\nballerina\nregally\nclergy\nlargely\nleading\ngallery\nballerina\nregally\nclergy\nlargely\nleading\ngallery\nballerina\nregally\nclergy\nlargely\nleading"
     },
     {
       "name": "detects anagrams case-insensitively",
+      "test_code": "const expected = [_][]const u8{\"Carthorse\"};\nconst word = \"Orchestra\";\nconst candidates = [_][]const u8{ \"cashregister\", \"Carthorse\", \"radishes\" };\ntry std.testing.checkAllAllocationFailures(\n    std.testing.allocator,\n    testAnagrams,\n    .{ &expected, word, &candidates },\n);",
       "status": "pass",
       "output": "cashregister\nCarthorse\nradishes\ncashregister\nCarthorse\nradishes\ncashregister\nCarthorse\nradishes"
     },
     {
       "name": "words are not anagrams of themselves",
+      "test_code": "const expected = [_][]const u8{};\nconst word = \"BANANA\";\nconst candidates = [_][]const u8{\"BANANA\"};\ntry std.testing.checkAllAllocationFailures(\n    std.testing.allocator,\n    testAnagrams,\n    .{ &expected, word, &candidates },\n);",
       "status": "pass",
       "output": "BANANA"
     },
     {
       "name": "detects many anagrams of 26-letter words",
+      "test_code": "const expected = [_][]const u8{\n    \"ZYXWVUTSRQPONMLKJIHGFEDCBA\",\n    \"MNOPQRSTUVWXYZABCDEFGHIJKL\",\n    \"BADCFEHGJILKNMPORQTSVUXWZY\",\n    \"QWERTYUIOPASDFGHJKLZXCVBNM\",\n    \"GHIJKLMNOPQRSTUVWXYZABCDEF\",\n    \"STUVWXYZABCDEFGHIJKLMNOPQR\",\n    \"DCBAEFGHIJKLMNOPQRSTUVWXYZ\",\n    \"AZBYCXDWEVFUGTHSIRJQKPLOMN\",\n    \"ZAYXBWCVDUESFTGRHQIPJOKLNM\",\n    \"NMLKJIHGFEDCBAZYXWVUTSRQPO\",\n    \"BDFHJLNPRTVXZACEGIKMOQSUWY\",\n    \"YWUSQOMKIGECAXZVTRPNLJHFDB\",\n    \"JKLMNOPQRSTUVWXYZABCDEFGHI\",\n    \"PONMLKJIHGFEDCBAZYXWVUTSRQ\",\n    \"LKJIHGFEDCBAZYXWVUTSRQPONM\",\n    \"CDEFGHIJKLMNOPQRSTUVWXYZAB\",\n    \"EFGHIJKLMNOPQRSTUVWXYZABCD\",\n    \"IJKLMNOPQRSTUVWXYZABCDEFGH\",\n    \"OPQRSTUVWXYZABCDEFGHIJKLMN\",\n    \"UVWXYZABCDEFGHIJKLMNOPQRST\",\n};\nconst word = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\";\ntry std.testing.checkAllAllocationFailures(\n    std.testing.allocator,\n    testAnagrams,\n    .{ &expected, word, &expected },\n);",
       "status": "pass",
       "output": "ZYXWVUTSRQPONMLKJIHGFEDCBA\nMNOPQRSTUVWXYZABCDEFGHIJKL\nBADCFEHGJILKNMPORQTSVUXWZY\nQWERTYUIOPASDFGHJKLZXCVBNM\nGHIJKLMNOPQRSTUVWXYZABCDEF\nSTUVWXYZABCDEFGHIJKLMNOPQR\nDCBAEFGHIJKLMNOPQRSTUVWXYZ\nAZBYCXDWEVFUGTHSIRJQKPLOMN\nZAYXBWCVDUESFTGRHQIPJOKLNM\nNMLKJIHGFEDCBAZYXWVUTSRQPO\nBDFHJLNPRTVXZACEGIKMOQSUWY\nYWUSQOMKIGECAXZVTRPNLJHFDB\nJKLMNOPQRSTUVWXYZABCDEFGHI\nPONMLKJIHGFEDCBAZYXWVUTSRQ\nLKJIHGFEDCBAZYXWVUTSRQPONM\nCDEFGHIJKLMNOPQRSTUVWXYZAB\nEFGHIJKLMNOPQRSTUVWXYZABCD\nIJKLMNOPQRSTUVWXYZABCD [output truncated]"
     }

tests/example-concept-tasks/example_concept_tasks.zig

@@ -0,0 +1,9 @@
+pub const expected_minutes_in_oven: u32 = 40;
+
+pub fn remainingMinutesInOven(actual_minutes_in_oven: u32) u32 {
+    return expected_minutes_in_oven - actual_minutes_in_oven;
+}
+
+pub fn preparationTimeInMinutes(number_of_layers: u32) u32 {
+    return number_of_layers * 2;
+}