From c360b5d1075df88c36a1e86f0f1c8db5e69f581a Mon Sep 17 00:00:00 2001 From: Eric Willigers Date: Mon, 10 Aug 2026 13:20:02 +1000 Subject: [PATCH] Implement interface version 3 We now report test_code for all tests, and task_id for concept exercise tests. We extract the task id from test suite comments like // task 1 --- Dockerfile | 4 +- bin/run.sh | 71 ++++++++--- bin/test-metadata.awk | 117 ++++++++++++++++++ tests/example-all-fail/expected_results.json | 11 +- .../expected_results.json | 8 +- .../example_concept_tasks.zig | 9 ++ .../expected_results.json | 35 ++++++ .../test_example_concept_tasks.zig | 26 ++++ tests/example-cut-off/example_cut_off.zig | 12 ++ tests/example-cut-off/expected_results.json | 16 +++ .../example-cut-off/test_example_cut_off.zig | 20 +++ .../example_debug_error_output.zig | 6 + .../expected_results.json | 26 ++++ .../test_example_debug_error_output.zig | 20 +++ .../example-empty-file/expected_results.json | 2 +- .../expected_results.json | 7 +- .../example-memory-leak/expected_results.json | 4 +- .../expected_results.json | 8 +- .../expected_results.json | 11 +- .../example_solution_own_test.zig | 11 ++ .../expected_results.json | 16 +++ .../test_example_solution_own_test.zig | 12 ++ .../expected_results.json | 4 +- tests/example-success/expected_results.json | 11 +- .../expected_results.json | 2 +- .../example_test_code_edge_cases.zig | 7 ++ .../expected_results.json | 25 ++++ .../test_example_test_code_edge_cases.zig | 25 ++++ .../example-user-output/expected_results.json | 3 +- 29 files changed, 501 insertions(+), 28 deletions(-) create mode 100644 bin/test-metadata.awk create mode 100644 tests/example-concept-tasks/example_concept_tasks.zig create mode 100644 tests/example-concept-tasks/expected_results.json create mode 100644 tests/example-concept-tasks/test_example_concept_tasks.zig create mode 100644 tests/example-cut-off/example_cut_off.zig create mode 100644 tests/example-cut-off/expected_results.json create mode 100644 tests/example-cut-off/test_example_cut_off.zig create mode 100644 tests/example-debug-error-output/example_debug_error_output.zig create mode 100644 tests/example-debug-error-output/expected_results.json create mode 100644 tests/example-debug-error-output/test_example_debug_error_output.zig create mode 100644 tests/example-solution-own-test/example_solution_own_test.zig create mode 100644 tests/example-solution-own-test/expected_results.json create mode 100644 tests/example-solution-own-test/test_example_solution_own_test.zig create mode 100644 tests/example-test-code-edge-cases/example_test_code_edge_cases.zig create mode 100644 tests/example-test-code-edge-cases/expected_results.json create mode 100644 tests/example-test-code-edge-cases/test_example_test_code_edge_cases.zig diff --git a/Dockerfile b/Dockerfile index 323efa8..5ee5586 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ FROM alpine:3.24.1@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6ee # install packages required to run the tests # hadolint ignore=DL3018 -RUN apk add --no-cache bash jq +RUN apk add --no-cache bash gawk jq RUN addgroup ziggroup \ && adduser --disabled-password --gecos ziggy --ingroup ziggroup ziggy @@ -30,7 +30,7 @@ ENV PATH=$PATH:/opt/zig USER ziggy:ziggroup WORKDIR /opt/test-runner -COPY --chown=ziggy:ziggroup bin/run.sh bin/run.sh +COPY --chown=ziggy:ziggroup bin/run.sh bin/test-metadata.awk bin/ # Initialize a zig cache COPY --chown=ziggy:ziggroup tests/example-success/example_success.zig init-zig-cache/ COPY --chown=ziggy:ziggroup tests/example-success/test_example_success.zig init-zig-cache/ diff --git a/bin/run.sh b/bin/run.sh index fce01ca..e36ea97 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -9,7 +9,7 @@ # $3: path to output directory # Output: -# Writes a v2 results.json to the output directory, per +# Writes a v3 results.json to the output directory, per # https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md # Example: @@ -39,9 +39,13 @@ run_zig_test() { local raw_output zig_exit raw_output=$(cd "${solution_dir}" && zig test -target x86_64-linux-musl "${test_file}" 2>&1) zig_exit=$? + # When the test binary dies mid-test, zig appends its trailer to the + # unfinished header line. Split the two apart before dropping the trailer, + # so the aborted test is still reported instead of vanishing with it. printf '%s' "${raw_output}" \ | sed -e "s#${solution_dir}/\{0,1\}##g" \ - -e '/error: the following test command failed/,$d' + -e 's/error: the following test command failed/\n&/' \ + | sed -e '/error: the following test command failed/,$d' return "${zig_exit}" } @@ -49,7 +53,7 @@ run_zig_test() { # the runner completed its job even though the solution did not build. emit_compile_error() { jq -n --arg message "${test_output}" \ - '{version: 2, status: "error", message: $message}' > "${results_file}" + '{version: 3, status: "error", message: $message}' > "${results_file}" echo "${slug}: done" exit 0 } @@ -67,12 +71,17 @@ emit_compile_error() { # `classify` turns each block into a single JSON test record. build_tests_json() { printf '%s' "${test_output}" | jq -Rs ' - def test_name: capture("test\\.(?.+)\\.\\.\\.") | .n; + # Zig prefixes each test with the module it was declared in. That + # prefix is matched lazily, so a module whose own name ends in + # "test" is not mistaken for the ".test." separator. + def test_name: capture("^[0-9]+/[0-9]+ .+?\\.test\\.(?.+)\\.\\.\\.") | .n; def user_output: (capture("\\.\\.\\.(?.+)$") | .o) // ""; def is_test_line: test("[0-9]+/[0-9]+ .*\\.test\\..*\\.\\.\\."); def is_summary: startswith("All ") or test("[0-9]+ passed;"); - def is_ok: . == "OK" or endswith("OK"); - def is_fail: startswith("FAIL") or test("\\.\\.\\.(FAIL|expected )"); + # Null-tolerant: a run cut off mid-report leaves a test with no + # status line at all, and that test must still be reported. + def is_ok: (. // "") | (. == "OK" or endswith("OK")); + def is_fail: (. // "") | (startswith("FAIL") or test("\\.\\.\\.(FAIL|expected )")); # Per-outcome record constructors. These set the leading fields; # classify extends records via `.field = ...` in the same order, @@ -97,7 +106,7 @@ build_tests_json() { .current += [$line] else . end ) - | if .current then .groups += [.current] else .groups end; + | if .current then .groups + [.current] else .groups end; # Pass 2 — turn one line group into one test record. # Three shapes handled: @@ -144,15 +153,44 @@ build_tests_json() { ' } +# Read what the test file itself says about each test — the body of its +# test block, and the task it belongs to — keyed by test name. +build_metadata_json() { + local path="${solution_dir}/${test_file}" + [[ -f "${path}" ]] || { echo '{}'; return; } + + gawk -f ./bin/test-metadata.awk "${path}" | jq -Rs ' + [ split("\u001e")[1:][] + | split("\u001f") + | {key: .[1], value: ( + (if .[2] == "" then {} else {test_code: .[2]} end) + + (if .[0] == "" then {} else {task_id: (.[0] | tonumber)} end) + )} + ] | from_entries + ' +} + # Write the final results.json. Truncates each test's "output" field to -# 500 chars to bound report size. +# 500 chars to bound report size, and folds in the metadata read from the +# test file: "test_code" directly after the name, "task_id" last. +# +# A test the student added to their own solution file is simply absent from +# the metadata, and so keeps its result without gaining either field. assemble_report() { local overall="$1" local tests_json="$2" - jq -n --arg status "${overall}" --argjson tests "${tests_json}" ' + local metadata="$3" + jq -n --arg status "${overall}" --argjson tests "${tests_json}" \ + --argjson metadata "${metadata}" ' def trunc: if length > 500 then .[:481] + " [output truncated]" else . end; - {version: 2, status: $status, tests: ($tests | map( - if .output then .output |= trunc else . end + def with_metadata($meta): + {name} + + (if $meta.test_code then {test_code: $meta.test_code} else {} end) + + del(.name) + + (if $meta.task_id then {task_id: $meta.task_id} else {} end); + {version: 3, status: $status, tests: ($tests | map( + (if .output then .output |= trunc else . end) + | with_metadata($metadata[.name] // {}) ))} ' > "${results_file}" } @@ -164,18 +202,21 @@ main() { local any_failed=0 test_output=$(run_zig_test) || any_failed=1 - if (( any_failed )) && [[ "${test_output}" = *error:* ]]; then + + local tests_json metadata overall + tests_json=$(build_tests_json) + + if (( any_failed )) && [[ "${tests_json}" == "[]" ]]; then emit_compile_error fi - local tests_json overall - tests_json=$(build_tests_json) + metadata=$(build_metadata_json) if (( any_failed == 0 )); then overall="pass" else overall="fail" fi - assemble_report "${overall}" "${tests_json}" + assemble_report "${overall}" "${tests_json}" "${metadata}" echo "${slug}: done" } diff --git a/bin/test-metadata.awk b/bin/test-metadata.awk new file mode 100644 index 0000000..840c527 --- /dev/null +++ b/bin/test-metadata.awk @@ -0,0 +1,117 @@ +#!/usr/bin/gawk -f + +# Synopsis: +# Extract per-test metadata from a Zig test file, for version 3 of +# https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md + +# Two things are picked up for each test: the body of its test block, which +# is reported as "test_code", and the task it belongs to, which concept +# exercise test files mark with a comment: +# +# // task 1 +# test "expected minutes in oven" { ... } +# +# A marker applies to every test that follows it, until the next marker. +# Tests declared before the first marker — which, in a practice exercise, +# means all of them — are left unlinked. + +# Output: +# One record per test, in declaration order: +# +# task id name test code +# +# where RS is 0x1e and US is 0x1f, and the task id is empty when the test is +# not linked to one. Emitting delimited text rather than JSON keeps every +# string escaping question in jq, where it is correct by construction. + +# Requires GNU awk, for the third argument to match(). + +BEGIN { + # The escape sequences Zig and JSON share; any other escape stands for + # the character it precedes. Numeric escapes (\x41, \u{1F600}) are not + # understood, so a test named with one is left unlinked rather than + # linked to the wrong task. + unescaped["n"] = "\n" + unescaped["t"] = "\t" + unescaped["r"] = "\r" +} + +# Turn the body of a Zig string literal into the text it denotes, which is +# what zig prints when it runs the test, and so what we match against. +function unescape(s, out, c) { + while (s != "") { + c = substr(s, 1, 1) + s = substr(s, 2) + if (c != "\\") { + out = out c + continue + } + c = substr(s, 1, 1) + s = substr(s, 2) + out = out (c in unescaped ? unescaped[c] : c) + } + return out +} + +# Return the index of the closing quote of the literal opening at `i`. +function skip_literal(line, i, quote, c) { + while (i <= length(line)) { + c = substr(line, i, 1) + if (c == "\\") i++ + else if (c == quote) return i + i++ + } + return i +} + +# Advance the brace depth across one line of Zig source, ignoring braces +# inside comments, character literals and string literals. Tracking depth +# rather than trusting `zig fmt` matters because students may edit and +# submit the test file of a practice exercise. +function scan_braces(line, depth, i, c) { + for (i = 1; i <= length(line); i++) { + c = substr(line, i, 1) + # Line comments and multiline string literals run to end of line. + if (c == "/" && substr(line, i + 1, 1) == "/") break + if (c == "\\" && substr(line, i + 1, 1) == "\\") break + if (c == "\"" || c == "'") i = skip_literal(line, i + 1, c) + else if (c == "{") depth++ + else if (c == "}") depth-- + } + return depth +} + +# Join the captured body lines, stripping the indentation they share and +# any blank lines top and bottom. +function dedent(count, i, margin, out) { + margin = -1 + for (i = 1; i <= count; i++) { + if (body[i] ~ /^[[:space:]]*$/) continue + match(body[i], /^[[:space:]]*/) + if (margin < 0 || RLENGTH < margin) margin = RLENGTH + } + if (margin < 0) return "" + for (i = 1; i <= count; i++) out = out (i == 1 ? "" : "\n") substr(body[i], margin + 1) + sub(/^\n+/, "", out) + sub(/\n+$/, "", out) + return out +} + +match($0, /^[[:space:]]*\/\/[[:space:]]*[Tt]ask[[:space:]]+([0-9]+)/, marker) { + task = marker[1] + next +} + +# A test block: capture the lines up to the "}" that closes it. The closing +# line is left out, as is the header, so what remains is the body. +match($0, /^[[:space:]]*test[[:space:]]+"((\\.|[^"\\])*)"/, decl) { + depth = scan_braces($0, 0) + count = 0 + while (depth > 0 && (getline line) > 0) { + depth = scan_braces(line, depth) + if (depth > 0) body[++count] = line + } + printf "%c%s%c%s%c%s", 30, task, 31, unescape(decl[1]), 31, dedent(count) + delete body + next +} diff --git a/tests/example-all-fail/expected_results.json b/tests/example-all-fail/expected_results.json index e1a01e1..11969c8 100644 --- a/tests/example-all-fail/expected_results.json +++ b/tests/example-all-fail/expected_results.json @@ -1,49 +1,58 @@ { - "version": 2, + "version": 3, "status": "fail", "tests": [ { "name": "year not divisible by 4 in common year", + "test_code": "try testing.expect(!leap.leap(2015));", "status": "fail", "message": "/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_all_fail.zig:7:5: in test.year not divisible by 4 in common year (test_example_all_fail.zig)\n try testing.expect(!leap.leap(2015));\n ^" }, { "name": "year divisible by 2, not divisible by 4 in common year", + "test_code": "try testing.expect(!leap.leap(1970));", "status": "fail", "message": "/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_all_fail.zig:11:5: in test.year divisible by 2, not divisible by 4 in common year (test_example_all_fail.zig)\n try testing.expect(!leap.leap(1970));\n ^" }, { "name": "year divisible by 4, not divisible by 100 in leap year", + "test_code": "try testing.expect(leap.leap(1996));", "status": "fail", "message": "/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_all_fail.zig:15:5: in test.year divisible by 4, not divisible by 100 in leap year (test_example_all_fail.zig)\n try testing.expect(leap.leap(1996));\n ^" }, { "name": "year divisible by 4 and 5 is still a leap year", + "test_code": "try testing.expect(leap.leap(1960));", "status": "fail", "message": "/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_all_fail.zig:19:5: in test.year divisible by 4 and 5 is still a leap year (test_example_all_fail.zig)\n try testing.expect(leap.leap(1960));\n ^" }, { "name": "year divisible by 100, not divisible by 400 in common year", + "test_code": "try testing.expect(!leap.leap(2100));", "status": "fail", "message": "/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_all_fail.zig:23:5: in test.year divisible by 100, not divisible by 400 in common year (test_example_all_fail.zig)\n try testing.expect(!leap.leap(2100));\n ^" }, { "name": "year divisible by 100 but not by 3 is still not a leap year", + "test_code": "try testing.expect(!leap.leap(1900));", "status": "fail", "message": "/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_all_fail.zig:27:5: in test.year divisible by 100 but not by 3 is still not a leap year (test_example_all_fail.zig)\n try testing.expect(!leap.leap(1900));\n ^" }, { "name": "year divisible by 400 is leap year", + "test_code": "try testing.expect(leap.leap(2000));", "status": "fail", "message": "/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_all_fail.zig:31:5: in test.year divisible by 400 is leap year (test_example_all_fail.zig)\n try testing.expect(leap.leap(2000));\n ^" }, { "name": "year divisible by 400 but not by 125 is still a leap year", + "test_code": "try testing.expect(leap.leap(2400));", "status": "fail", "message": "/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_all_fail.zig:35:5: in test.year divisible by 400 but not by 125 is still a leap year (test_example_all_fail.zig)\n try testing.expect(leap.leap(2400));\n ^" }, { "name": "year divisible by 200, not divisible by 400 in common year", + "test_code": "try testing.expect(!leap.leap(1800));", "status": "fail", "message": "/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_all_fail.zig:39:5: in test.year divisible by 200, not divisible by 400 in common year (test_example_all_fail.zig)\n try testing.expect(!leap.leap(1800));\n ^" } diff --git a/tests/example-anagram-success/expected_results.json b/tests/example-anagram-success/expected_results.json index aba74ba..b483980 100644 --- a/tests/example-anagram-success/expected_results.json +++ b/tests/example-anagram-success/expected_results.json @@ -1,34 +1,40 @@ { - "version": 2, + "version": 3, "status": "pass", "tests": [ { "name": "no matches", + "test_code": "const expected = [_][]const u8{};\nconst word = \"diaper\";\nconst candidates = [_][]const u8{ \"hello\", \"world\", \"zombies\", \"pants\" };\ntry std.testing.checkAllAllocationFailures(\n std.testing.allocator,\n testAnagrams,\n .{ &expected, word, &candidates },\n);", "status": "pass", "output": "hello\nworld\nzombies\npants" }, { "name": "detects two anagrams", + "test_code": "const expected = [_][]const u8{ \"lemons\", \"melons\" };\nconst word = \"solemn\";\nconst candidates = [_][]const u8{ \"lemons\", \"cherry\", \"melons\" };\ntry std.testing.checkAllAllocationFailures(\n std.testing.allocator,\n testAnagrams,\n .{ &expected, word, &candidates },\n);", "status": "pass", "output": "lemons\ncherry\nmelons\nlemons\ncherry\nmelons\nlemons\ncherry\nmelons\nlemons\ncherry\nmelons" }, { "name": "detects three anagrams", + "test_code": "const expected = [_][]const u8{ \"gallery\", \"regally\", \"largely\" };\nconst word = \"allergy\";\nconst candidates = [_][]const u8{ \"gallery\", \"ballerina\", \"regally\", \"clergy\", \"largely\", \"leading\" };\ntry std.testing.checkAllAllocationFailures(\n std.testing.allocator,\n testAnagrams,\n .{ &expected, word, &candidates },\n);", "status": "pass", "output": "gallery\nballerina\nregally\nclergy\nlargely\nleading\ngallery\nballerina\nregally\nclergy\nlargely\nleading\ngallery\nballerina\nregally\nclergy\nlargely\nleading\ngallery\nballerina\nregally\nclergy\nlargely\nleading\ngallery\nballerina\nregally\nclergy\nlargely\nleading" }, { "name": "detects anagrams case-insensitively", + "test_code": "const expected = [_][]const u8{\"Carthorse\"};\nconst word = \"Orchestra\";\nconst candidates = [_][]const u8{ \"cashregister\", \"Carthorse\", \"radishes\" };\ntry std.testing.checkAllAllocationFailures(\n std.testing.allocator,\n testAnagrams,\n .{ &expected, word, &candidates },\n);", "status": "pass", "output": "cashregister\nCarthorse\nradishes\ncashregister\nCarthorse\nradishes\ncashregister\nCarthorse\nradishes" }, { "name": "words are not anagrams of themselves", + "test_code": "const expected = [_][]const u8{};\nconst word = \"BANANA\";\nconst candidates = [_][]const u8{\"BANANA\"};\ntry std.testing.checkAllAllocationFailures(\n std.testing.allocator,\n testAnagrams,\n .{ &expected, word, &candidates },\n);", "status": "pass", "output": "BANANA" }, { "name": "detects many anagrams of 26-letter words", + "test_code": "const expected = [_][]const u8{\n \"ZYXWVUTSRQPONMLKJIHGFEDCBA\",\n \"MNOPQRSTUVWXYZABCDEFGHIJKL\",\n \"BADCFEHGJILKNMPORQTSVUXWZY\",\n \"QWERTYUIOPASDFGHJKLZXCVBNM\",\n \"GHIJKLMNOPQRSTUVWXYZABCDEF\",\n \"STUVWXYZABCDEFGHIJKLMNOPQR\",\n \"DCBAEFGHIJKLMNOPQRSTUVWXYZ\",\n \"AZBYCXDWEVFUGTHSIRJQKPLOMN\",\n \"ZAYXBWCVDUESFTGRHQIPJOKLNM\",\n \"NMLKJIHGFEDCBAZYXWVUTSRQPO\",\n \"BDFHJLNPRTVXZACEGIKMOQSUWY\",\n \"YWUSQOMKIGECAXZVTRPNLJHFDB\",\n \"JKLMNOPQRSTUVWXYZABCDEFGHI\",\n \"PONMLKJIHGFEDCBAZYXWVUTSRQ\",\n \"LKJIHGFEDCBAZYXWVUTSRQPONM\",\n \"CDEFGHIJKLMNOPQRSTUVWXYZAB\",\n \"EFGHIJKLMNOPQRSTUVWXYZABCD\",\n \"IJKLMNOPQRSTUVWXYZABCDEFGH\",\n \"OPQRSTUVWXYZABCDEFGHIJKLMN\",\n \"UVWXYZABCDEFGHIJKLMNOPQRST\",\n};\nconst word = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\";\ntry std.testing.checkAllAllocationFailures(\n std.testing.allocator,\n testAnagrams,\n .{ &expected, word, &expected },\n);", "status": "pass", "output": "ZYXWVUTSRQPONMLKJIHGFEDCBA\nMNOPQRSTUVWXYZABCDEFGHIJKL\nBADCFEHGJILKNMPORQTSVUXWZY\nQWERTYUIOPASDFGHJKLZXCVBNM\nGHIJKLMNOPQRSTUVWXYZABCDEF\nSTUVWXYZABCDEFGHIJKLMNOPQR\nDCBAEFGHIJKLMNOPQRSTUVWXYZ\nAZBYCXDWEVFUGTHSIRJQKPLOMN\nZAYXBWCVDUESFTGRHQIPJOKLNM\nNMLKJIHGFEDCBAZYXWVUTSRQPO\nBDFHJLNPRTVXZACEGIKMOQSUWY\nYWUSQOMKIGECAXZVTRPNLJHFDB\nJKLMNOPQRSTUVWXYZABCDEFGHI\nPONMLKJIHGFEDCBAZYXWVUTSRQ\nLKJIHGFEDCBAZYXWVUTSRQPONM\nCDEFGHIJKLMNOPQRSTUVWXYZAB\nEFGHIJKLMNOPQRSTUVWXYZABCD\nIJKLMNOPQRSTUVWXYZABCD [output truncated]" } diff --git a/tests/example-concept-tasks/example_concept_tasks.zig b/tests/example-concept-tasks/example_concept_tasks.zig new file mode 100644 index 0000000..9fba626 --- /dev/null +++ b/tests/example-concept-tasks/example_concept_tasks.zig @@ -0,0 +1,9 @@ +pub const expected_minutes_in_oven: u32 = 40; + +pub fn remainingMinutesInOven(actual_minutes_in_oven: u32) u32 { + return expected_minutes_in_oven - actual_minutes_in_oven; +} + +pub fn preparationTimeInMinutes(number_of_layers: u32) u32 { + return number_of_layers * 2; +} diff --git a/tests/example-concept-tasks/expected_results.json b/tests/example-concept-tasks/expected_results.json new file mode 100644 index 0000000..a3c1099 --- /dev/null +++ b/tests/example-concept-tasks/expected_results.json @@ -0,0 +1,35 @@ +{ + "version": 3, + "status": "pass", + "tests": [ + { + "name": "a test declared before any marker is not linked to a task", + "test_code": "try testing.expectEqual(40, lasagna.expected_minutes_in_oven);", + "status": "pass" + }, + { + "name": "expected minutes in oven", + "test_code": "try testing.expectEqual(40, lasagna.expected_minutes_in_oven);", + "status": "pass", + "task_id": 1 + }, + { + "name": "remaining minutes in oven", + "test_code": "try testing.expectEqual(15, lasagna.remainingMinutesInOven(25));", + "status": "pass", + "task_id": 1 + }, + { + "name": "preparation time for one layer", + "test_code": "try testing.expectEqual(2, lasagna.preparationTimeInMinutes(1));", + "status": "pass", + "task_id": 2 + }, + { + "name": "preparation time for four layers", + "test_code": "try testing.expectEqual(8, lasagna.preparationTimeInMinutes(4));", + "status": "pass", + "task_id": 2 + } + ] +} diff --git a/tests/example-concept-tasks/test_example_concept_tasks.zig b/tests/example-concept-tasks/test_example_concept_tasks.zig new file mode 100644 index 0000000..f733b91 --- /dev/null +++ b/tests/example-concept-tasks/test_example_concept_tasks.zig @@ -0,0 +1,26 @@ +const std = @import("std"); +const testing = std.testing; + +const lasagna = @import("example_concept_tasks.zig"); + +test "a test declared before any marker is not linked to a task" { + try testing.expectEqual(40, lasagna.expected_minutes_in_oven); +} + +// task 1 +test "expected minutes in oven" { + try testing.expectEqual(40, lasagna.expected_minutes_in_oven); +} + +test "remaining minutes in oven" { + try testing.expectEqual(15, lasagna.remainingMinutesInOven(25)); +} + +// task 2 +test "preparation time for one layer" { + try testing.expectEqual(2, lasagna.preparationTimeInMinutes(1)); +} + +test "preparation time for four layers" { + try testing.expectEqual(8, lasagna.preparationTimeInMinutes(4)); +} diff --git a/tests/example-cut-off/example_cut_off.zig b/tests/example-cut-off/example_cut_off.zig new file mode 100644 index 0000000..fd77a06 --- /dev/null +++ b/tests/example-cut-off/example_cut_off.zig @@ -0,0 +1,12 @@ +const std = @import("std"); + +pub fn value() u32 { + return 1; +} + +// A solution that ends the process takes the test runner down with it, so +// zig never prints a status line for the test that was running, nor the +// summary that normally closes its report. +pub fn bail() void { + std.process.exit(1); +} diff --git a/tests/example-cut-off/expected_results.json b/tests/example-cut-off/expected_results.json new file mode 100644 index 0000000..644ae06 --- /dev/null +++ b/tests/example-cut-off/expected_results.json @@ -0,0 +1,16 @@ +{ + "version": 3, + "status": "fail", + "tests": [ + { + "name": "first test passes", + "test_code": "try testing.expectEqual(1, mod.value());", + "status": "pass" + }, + { + "name": "second test ends the process", + "test_code": "mod.bail();", + "status": "fail" + } + ] +} diff --git a/tests/example-cut-off/test_example_cut_off.zig b/tests/example-cut-off/test_example_cut_off.zig new file mode 100644 index 0000000..152b270 --- /dev/null +++ b/tests/example-cut-off/test_example_cut_off.zig @@ -0,0 +1,20 @@ +// A run cut off mid-report: the first test passes, the second ends the +// process, and the third never runs. The aborted test must still be +// reported as failed rather than dropped, and the tests that did run must +// keep their results. +const std = @import("std"); +const testing = std.testing; + +const mod = @import("example_cut_off.zig"); + +test "first test passes" { + try testing.expectEqual(1, mod.value()); +} + +test "second test ends the process" { + mod.bail(); +} + +test "third test never runs" { + try testing.expect(true); +} diff --git a/tests/example-debug-error-output/example_debug_error_output.zig b/tests/example-debug-error-output/example_debug_error_output.zig new file mode 100644 index 0000000..4ca3cb7 --- /dev/null +++ b/tests/example-debug-error-output/example_debug_error_output.zig @@ -0,0 +1,6 @@ +const std = @import("std"); + +pub fn greet() []const u8 { + std.debug.print("error: greet was called with no arguments\n", .{}); + return "hi"; +} diff --git a/tests/example-debug-error-output/expected_results.json b/tests/example-debug-error-output/expected_results.json new file mode 100644 index 0000000..ddf791f --- /dev/null +++ b/tests/example-debug-error-output/expected_results.json @@ -0,0 +1,26 @@ +{ + "version": 3, + "status": "fail", + "tests": [ + { + "name": "passing test alongside error-like debug output", + "test_code": "try testing.expect(mod.greet().len == 2);", + "status": "pass", + "output": "error: greet was called with no arguments" + }, + { + "name": "failing test alongside error-like debug output", + "test_code": "try testing.expect(mod.greet().len == 99);", + "status": "fail", + "message": "FAIL (TestUnexpectedResult)\n/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_debug_error_output.zig:15:5: in test.failing test alongside error-like debug output (test_example_debug_error_output.zig)\n try testing.expect(mod.greet().len == 99);\n ^", + "output": "error: greet was called with no arguments" + }, + { + "name": "failing line carries an error-like comment", + "test_code": "try testing.expect(mod.greet().len == 99); // error: expected a longer greeting", + "status": "fail", + "message": "FAIL (TestUnexpectedResult)\n/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_debug_error_output.zig:19:5: in test.failing line carries an error-like comment (test_example_debug_error_output.zig)\n try testing.expect(mod.greet().len == 99); // error: expected a longer greeting\n ^", + "output": "error: greet was called with no arguments" + } + ] +} diff --git a/tests/example-debug-error-output/test_example_debug_error_output.zig b/tests/example-debug-error-output/test_example_debug_error_output.zig new file mode 100644 index 0000000..79a87aa --- /dev/null +++ b/tests/example-debug-error-output/test_example_debug_error_output.zig @@ -0,0 +1,20 @@ +// Output that looks like a compiler diagnostic, from the two places it can +// come from: what the solution prints, and the source line a stack trace +// echoes back. Neither is a compile error, so the individual test results +// must survive rather than being replaced by one top-level message. +const std = @import("std"); +const testing = std.testing; + +const mod = @import("example_debug_error_output.zig"); + +test "passing test alongside error-like debug output" { + try testing.expect(mod.greet().len == 2); +} + +test "failing test alongside error-like debug output" { + try testing.expect(mod.greet().len == 99); +} + +test "failing line carries an error-like comment" { + try testing.expect(mod.greet().len == 99); // error: expected a longer greeting +} diff --git a/tests/example-empty-file/expected_results.json b/tests/example-empty-file/expected_results.json index 6c7e768..28ac40f 100644 --- a/tests/example-empty-file/expected_results.json +++ b/tests/example-empty-file/expected_results.json @@ -1,5 +1,5 @@ { - "version": 2, + "version": 3, "status": "error", "message": "test_example_empty_file.zig:7:29: error: root source file struct 'example_empty_file' has no member named 'leap'\n try testing.expect(!leap.leap(2015));\n ~~~~^~~~~\nexample_empty_file.zig:1:1: note: struct declared here\n\n^\ntest_example_empty_file.zig:11:29: error: root source file struct 'example_empty_file' has no member named 'leap'\n try testing.expect(!leap.leap(1970));\n ~~~~^~~~~\nexample_empty_file.zig:1:1: note: struct declared here\n\n^\ntest_example_empty_file.zig:15:28: error: root source file struct 'example_empty_file' has no member named 'leap'\n try testing.expect(leap.leap(1996));\n ~~~~^~~~~\nexample_empty_file.zig:1:1: note: struct declared here\n\n^\ntest_example_empty_file.zig:19:28: error: root source file struct 'example_empty_file' has no member named 'leap'\n try testing.expect(leap.leap(1960));\n ~~~~^~~~~\nexample_empty_file.zig:1:1: note: struct declared here\n\n^\ntest_example_empty_file.zig:23:29: error: root source file struct 'example_empty_file' has no member named 'leap'\n try testing.expect(!leap.leap(2100));\n ~~~~^~~~~\nexample_empty_file.zig:1:1: note: struct declared here\n\n^\ntest_example_empty_file.zig:27:29: error: root source file struct 'example_empty_file' has no member named 'leap'\n try testing.expect(!leap.leap(1900));\n ~~~~^~~~~\nexample_empty_file.zig:1:1: note: struct declared here\n\n^\ntest_example_empty_file.zig:31:28: error: root source file struct 'example_empty_file' has no member named 'leap'\n try testing.expect(leap.leap(2000));\n ~~~~^~~~~\nexample_empty_file.zig:1:1: note: struct declared here\n\n^\ntest_example_empty_file.zig:35:28: error: root source file struct 'example_empty_file' has no member named 'leap'\n try testing.expect(leap.leap(2400));\n ~~~~^~~~~\nexample_empty_file.zig:1:1: note: struct declared here\n\n^\ntest_example_empty_file.zig:39:29: error: root source file struct 'example_empty_file' has no member named 'leap'\n try testing.expect(!leap.leap(1800));\n ~~~~^~~~~\nexample_empty_file.zig:1:1: note: struct declared here\n\n^" } diff --git a/tests/example-error-exercise/expected_results.json b/tests/example-error-exercise/expected_results.json index 7bba5c9..7816da0 100644 --- a/tests/example-error-exercise/expected_results.json +++ b/tests/example-error-exercise/expected_results.json @@ -1,25 +1,30 @@ { - "version": 2, + "version": 3, "status": "fail", "tests": [ { "name": "zero steps for one", + "test_code": "const expected: usize = 0;\nconst actual = try collatz.steps(1);\ntry testing.expectEqual(expected, actual);", "status": "pass" }, { "name": "divide if even", + "test_code": "const expected: usize = 4;\nconst actual = try collatz.steps(16);\ntry testing.expectEqual(expected, actual);", "status": "pass" }, { "name": "even and odd steps", + "test_code": "const expected: usize = 9;\nconst actual = try collatz.steps(12);\ntry testing.expectEqual(expected, actual);", "status": "pass" }, { "name": "large number of even and odd steps", + "test_code": "const expected: usize = 152;\nconst actual = try collatz.steps(1_000_000);\ntry testing.expectEqual(expected, actual);", "status": "pass" }, { "name": "zero is an error", + "test_code": "const expected = ComputationError.IllegalArgument;\nconst actual = collatz.steps(0);\ntry testing.expectError(expected, actual);", "status": "fail", "message": "FAIL (TestExpectedError)\n/opt/zig/lib/std/testing.zig:65:9: in expectError__anon_ (std.zig)\n return error.TestExpectedError;\n ^\ntest_example_error_exercise.zig:34:5: in test.zero is an error (test_example_error_exercise.zig)\n try testing.expectError(expected, actual);\n ^" } diff --git a/tests/example-memory-leak/expected_results.json b/tests/example-memory-leak/expected_results.json index b54e9de..3171b8f 100644 --- a/tests/example-memory-leak/expected_results.json +++ b/tests/example-memory-leak/expected_results.json @@ -1,15 +1,17 @@ { - "version": 2, + "version": 3, "status": "fail", "tests": [ { "name": "makePair cleans up on allocation failure", + "test_code": "try std.testing.checkAllAllocationFailures(\n std.testing.allocator,\n pairTest,\n .{},\n);", "status": "fail", "message": "FAIL (MemoryLeakDetected)\n/opt/zig/lib/std/mem/Allocator.zig:300:82: in allocBytesWithAlignment__anon_ (std.zig)\n const byte_ptr = self.rawAlloc(byte_count, alignment, return_address) orelse return error.OutOfMemory;\n ^\n/opt/zig/lib/std/mem/Allocator.zig:286:5: in allocWithSizeAndAlignment__anon_ (std.zig)\n return self.allocBytesWithAlignment(alignment, byte_count, return_address);\n ^\n/opt/zig/lib/std/mem/Allocator.zig:274:55: in allocAdvancedWithRetAddr (std.zig)\n const ptr: [*]align(a.toByteUnits()) T = @ptrCast(try self.allocWithSizeAndAlignment(@sizeOf(T), a, n, return_address));\n ^\n/opt/zig/lib/std/mem/Allocator.zig:198:5: in alloc__anon_ (std.zig)\n return self.allocAdvancedWithRetAddr(T, null, n, @returnAddress());\n ^\nexample_memory_leak.zig:9:20: in makePair (test_example_memory_leak.zig)\n const second = try allocator.alloc(u8, 16);\n ^\ntest_example_memory_leak.zig:7:15: in pairTest (test_example_memory_leak.zig)\n const p = try mod.makePair(allocator);\n ^\n/opt/zig/lib/std/testing.zig:1156:21: in checkAllAllocationFailures__anon_ (std.zig)\n return error.MemoryLeakDetected;\n ^\ntest_example_memory_leak.zig:13:5: in test.makePair cleans up on allocation failure (test_example_memory_leak.zig)\n try std.testing.checkAllAllocationFailures(\n ^\n[DebugAllocator] (err): memory address leaked: \nexample_memory_leak.zig:8:38: in makePair (test_example_memory_leak.zig)\n const first = try allocator.alloc(u8, 16);\n ^\ntest_example_memory_leak.zig:7:31: in pairTest (test_example_memory_leak.zig)\n const p = try mod.makePair(allocator);\n ^\n/opt/zig/lib/std/testing.zig:1133:69: in checkAllAllocationFailures__anon_ (std.zig)\n if (@call(.auto, test_fn, .{failing_allocator_inst.allocator()} ++ extra_args)) |_| {\n ^\ntest_example_memory_leak.zig:13:47: in test.makePair cleans up on allocation failure (test_example_memory_leak.zig)\n try std.testing.checkAllAllocationFailures(\n ^\n/opt/zig/lib/compiler/test_runner.zig:291:25: in mainTerminal (test_runner.zig)\n if (test_fn.func()) |_| {\n ^\n/opt/zig/lib/compiler/test_runner.zig:73:28: in main (test_runner.zig)\n return mainTerminal(init);\n ^\n/opt/zig/lib/std/start.zig:699:88: in callMain (std.zig)\n if (fn_info.params[0].type.? == std.process.Init.Minimal) return wrapMain(root.main(.{\n ^\n/opt/zig/lib/std/start.zig:190:5: in _start (std.zig)\n asm volatile (switch (native_arch) {\n ^", "output": "fail_index: 1/2\nallocated bytes: 16\nfreed bytes: 0\nallocations: 1\ndeallocations: 0\nallocation that was made to fail: \nexample_memory_leak.zig:9:39: in makePair (test_example_memory_leak.zig)\n const second = try allocator.alloc(u8, 16);\n ^\ntest_example_memory_leak.zig:7:31: in pairTest (test_example_memory_leak.zig)\n const p = try mod.makePair(allocator);\n ^\n/opt/zig/lib/std/testing.zig: [output truncated]" }, { "name": "sanity pass", + "test_code": "try std.testing.expect(1 + 1 == 2);", "status": "pass" } ] diff --git a/tests/example-output-truncated/expected_results.json b/tests/example-output-truncated/expected_results.json index 02238f8..ca28fbb 100644 --- a/tests/example-output-truncated/expected_results.json +++ b/tests/example-output-truncated/expected_results.json @@ -1,37 +1,43 @@ { - "version": 2, + "version": 3, "status": "fail", "tests": [ { "name": "no matches", + "test_code": "const expected = [_][]const u8{};\nconst word = \"diaper\";\nconst candidates = [_][]const u8{ \"hello\", \"world\", \"zombies\", \"pants\" };\ntry std.testing.checkAllAllocationFailures(\n std.testing.allocator,\n testAnagrams,\n .{ &expected, word, &candidates },\n);", "status": "pass", "output": "hello\nworld\nzombies\npants\nhello\nworld\nzombies\npants" }, { "name": "detects two anagrams", + "test_code": "const expected = [_][]const u8{ \"lemons\", \"melons\" };\nconst word = \"solemn\";\nconst candidates = [_][]const u8{ \"lemons\", \"cherry\", \"melons\" };\ntry std.testing.checkAllAllocationFailures(\n std.testing.allocator,\n testAnagrams,\n .{ &expected, word, &candidates },\n);", "status": "fail", "message": "FAIL (TestExpectedEqual)\n/opt/zig/lib/std/testing.zig:118:17: in expectEqualInner__anon_ (std.zig)\n return error.TestExpectedEqual;\n ^\n/opt/zig/lib/std/testing.zig:83:5: in expectEqual (test_example_output_truncated.zig)\n return expectEqualInner(T, expected, actual);\n ^\ntest_example_output_truncated.zig:14:5: in testAnagrams (test_example_output_truncated.zig)\n try testing.expectEqual(expected.len, actual.count());\n ^\n/opt/zig/lib/std/testing.zig:1124:9: in checkAllAllocationFailures__anon_ (std.zig)\n try @call(.auto, test_fn, .{failing_allocator_inst.allocator()} ++ extra_args);\n ^\ntest_example_output_truncated.zig:35:5: in test.detects two anagrams (test_example_output_truncated.zig)\n try std.testing.checkAllAllocationFailures(\n ^", "output": "lemons\ncherry\nmelons\nexpected 2, found 0" }, { "name": "detects three anagrams", + "test_code": "const expected = [_][]const u8{ \"gallery\", \"regally\", \"largely\" };\nconst word = \"allergy\";\nconst candidates = [_][]const u8{ \"gallery\", \"ballerina\", \"regally\", \"clergy\", \"largely\", \"leading\" };\ntry std.testing.checkAllAllocationFailures(\n std.testing.allocator,\n testAnagrams,\n .{ &expected, word, &candidates },\n);", "status": "fail", "message": "FAIL (TestExpectedEqual)\n/opt/zig/lib/std/testing.zig:118:17: in expectEqualInner__anon_ (std.zig)\n return error.TestExpectedEqual;\n ^\n/opt/zig/lib/std/testing.zig:83:5: in expectEqual (test_example_output_truncated.zig)\n return expectEqualInner(T, expected, actual);\n ^\ntest_example_output_truncated.zig:14:5: in testAnagrams (test_example_output_truncated.zig)\n try testing.expectEqual(expected.len, actual.count());\n ^\n/opt/zig/lib/std/testing.zig:1124:9: in checkAllAllocationFailures__anon_ (std.zig)\n try @call(.auto, test_fn, .{failing_allocator_inst.allocator()} ++ extra_args);\n ^\ntest_example_output_truncated.zig:46:5: in test.detects three anagrams (test_example_output_truncated.zig)\n try std.testing.checkAllAllocationFailures(\n ^", "output": "gallery\nballerina\nregally\nclergy\nlargely\nleading\nexpected 3, found 0" }, { "name": "detects anagrams case-insensitively", + "test_code": "const expected = [_][]const u8{\"Carthorse\"};\nconst word = \"Orchestra\";\nconst candidates = [_][]const u8{ \"cashregister\", \"Carthorse\", \"radishes\" };\ntry std.testing.checkAllAllocationFailures(\n std.testing.allocator,\n testAnagrams,\n .{ &expected, word, &candidates },\n);", "status": "fail", "message": "FAIL (TestExpectedEqual)\n/opt/zig/lib/std/testing.zig:118:17: in expectEqualInner__anon_ (std.zig)\n return error.TestExpectedEqual;\n ^\n/opt/zig/lib/std/testing.zig:83:5: in expectEqual (test_example_output_truncated.zig)\n return expectEqualInner(T, expected, actual);\n ^\ntest_example_output_truncated.zig:14:5: in testAnagrams (test_example_output_truncated.zig)\n try testing.expectEqual(expected.len, actual.count());\n ^\n/opt/zig/lib/std/testing.zig:1124:9: in checkAllAllocationFailures__anon_ (std.zig)\n try @call(.auto, test_fn, .{failing_allocator_inst.allocator()} ++ extra_args);\n ^\ntest_example_output_truncated.zig:57:5: in test.detects anagrams case-insensitively (test_example_output_truncated.zig)\n try std.testing.checkAllAllocationFailures(\n ^", "output": "cashregister\nCarthorse\nradishes\nexpected 1, found 0" }, { "name": "words are not anagrams of themselves", + "test_code": "const expected = [_][]const u8{};\nconst word = \"BANANA\";\nconst candidates = [_][]const u8{\"BANANA\"};\ntry std.testing.checkAllAllocationFailures(\n std.testing.allocator,\n testAnagrams,\n .{ &expected, word, &candidates },\n);", "status": "pass", "output": "BANANA\nBANANA" }, { "name": "detects many anagrams of 26-letter words", + "test_code": "const expected = [_][]const u8{\n \"ZYXWVUTSRQPONMLKJIHGFEDCBA\",\n \"MNOPQRSTUVWXYZABCDEFGHIJKL\",\n \"BADCFEHGJILKNMPORQTSVUXWZY\",\n \"QWERTYUIOPASDFGHJKLZXCVBNM\",\n \"GHIJKLMNOPQRSTUVWXYZABCDEF\",\n \"STUVWXYZABCDEFGHIJKLMNOPQR\",\n \"DCBAEFGHIJKLMNOPQRSTUVWXYZ\",\n \"AZBYCXDWEVFUGTHSIRJQKPLOMN\",\n \"ZAYXBWCVDUESFTGRHQIPJOKLNM\",\n \"NMLKJIHGFEDCBAZYXWVUTSRQPO\",\n \"BDFHJLNPRTVXZACEGIKMOQSUWY\",\n \"YWUSQOMKIGECAXZVTRPNLJHFDB\",\n \"JKLMNOPQRSTUVWXYZABCDEFGHI\",\n \"PONMLKJIHGFEDCBAZYXWVUTSRQ\",\n \"LKJIHGFEDCBAZYXWVUTSRQPONM\",\n \"CDEFGHIJKLMNOPQRSTUVWXYZAB\",\n \"EFGHIJKLMNOPQRSTUVWXYZABCD\",\n \"IJKLMNOPQRSTUVWXYZABCDEFGH\",\n \"OPQRSTUVWXYZABCDEFGHIJKLMN\",\n \"UVWXYZABCDEFGHIJKLMNOPQRST\",\n};\nconst word = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\";\ntry std.testing.checkAllAllocationFailures(\n std.testing.allocator,\n testAnagrams,\n .{ &expected, word, &expected },\n);", "status": "fail", "message": "FAIL (TestExpectedEqual)\n/opt/zig/lib/std/testing.zig:118:17: in expectEqualInner__anon_ (std.zig)\n return error.TestExpectedEqual;\n ^\n/opt/zig/lib/std/testing.zig:83:5: in expectEqual (test_example_output_truncated.zig)\n return expectEqualInner(T, expected, actual);\n ^\ntest_example_output_truncated.zig:14:5: in testAnagrams (test_example_output_truncated.zig)\n try testing.expectEqual(expected.len, actual.count());\n ^\n/opt/zig/lib/std/testing.zig:1124:9: in checkAllAllocationFailures__anon_ (std.zig)\n try @call(.auto, test_fn, .{failing_allocator_inst.allocator()} ++ extra_args);\n ^\ntest_example_output_truncated.zig:99:5: in test.detects many anagrams of 26-letter words (test_example_output_truncated.zig)\n try std.testing.checkAllAllocationFailures(\n ^", "output": "ZYXWVUTSRQPONMLKJIHGFEDCBA\nMNOPQRSTUVWXYZABCDEFGHIJKL\nBADCFEHGJILKNMPORQTSVUXWZY\nQWERTYUIOPASDFGHJKLZXCVBNM\nGHIJKLMNOPQRSTUVWXYZABCDEF\nSTUVWXYZABCDEFGHIJKLMNOPQR\nDCBAEFGHIJKLMNOPQRSTUVWXYZ\nAZBYCXDWEVFUGTHSIRJQKPLOMN\nZAYXBWCVDUESFTGRHQIPJOKLNM\nNMLKJIHGFEDCBAZYXWVUTSRQPO\nBDFHJLNPRTVXZACEGIKMOQSUWY\nYWUSQOMKIGECAXZVTRPNLJHFDB\nJKLMNOPQRSTUVWXYZABCDEFGHI\nPONMLKJIHGFEDCBAZYXWVUTSRQ\nLKJIHGFEDCBAZYXWVUTSRQPONM\nCDEFGHIJKLMNOPQRSTUVWXYZAB\nEFGHIJKLMNOPQRSTUVWXYZABCD\nIJKLMNOPQRSTUVWXYZABCD [output truncated]" diff --git a/tests/example-partial-fail/expected_results.json b/tests/example-partial-fail/expected_results.json index 174cf7b..c61abec 100644 --- a/tests/example-partial-fail/expected_results.json +++ b/tests/example-partial-fail/expected_results.json @@ -1,43 +1,52 @@ { - "version": 2, + "version": 3, "status": "fail", "tests": [ { "name": "year not divisible by 4 in common year", + "test_code": "try testing.expect(!leap.leap(2015));", "status": "pass" }, { "name": "year divisible by 2, not divisible by 4 in common year", + "test_code": "try testing.expect(!leap.leap(1970));", "status": "pass" }, { "name": "year divisible by 4, not divisible by 100 in leap year", + "test_code": "try testing.expect(leap.leap(1996));", "status": "pass" }, { "name": "year divisible by 4 and 5 is still a leap year", + "test_code": "try testing.expect(leap.leap(1960));", "status": "pass" }, { "name": "year divisible by 100, not divisible by 400 in common year", + "test_code": "try testing.expect(!leap.leap(2100));", "status": "fail", "message": "/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_partial_fail.zig:23:5: in test.year divisible by 100, not divisible by 400 in common year (test_example_partial_fail.zig)\n try testing.expect(!leap.leap(2100));\n ^" }, { "name": "year divisible by 100 but not by 3 is still not a leap year", + "test_code": "try testing.expect(!leap.leap(1900));", "status": "fail", "message": "/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_partial_fail.zig:27:5: in test.year divisible by 100 but not by 3 is still not a leap year (test_example_partial_fail.zig)\n try testing.expect(!leap.leap(1900));\n ^" }, { "name": "year divisible by 400 is leap year", + "test_code": "try testing.expect(leap.leap(2000));", "status": "pass" }, { "name": "year divisible by 400 but not by 125 is still a leap year", + "test_code": "try testing.expect(leap.leap(2400));", "status": "pass" }, { "name": "year divisible by 200, not divisible by 400 in common year", + "test_code": "try testing.expect(!leap.leap(1800));", "status": "fail", "message": "/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_partial_fail.zig:39:5: in test.year divisible by 200, not divisible by 400 in common year (test_example_partial_fail.zig)\n try testing.expect(!leap.leap(1800));\n ^" } diff --git a/tests/example-solution-own-test/example_solution_own_test.zig b/tests/example-solution-own-test/example_solution_own_test.zig new file mode 100644 index 0000000..6ce238e --- /dev/null +++ b/tests/example-solution-own-test/example_solution_own_test.zig @@ -0,0 +1,11 @@ +const std = @import("std"); + +pub fn value() u32 { + return 1; +} + +// Students may keep their own tests in the solution file. Zig runs them +// alongside ours, and they carry no test_code or task_id of their own. +test "a test the student wrote" { + try std.testing.expect(value() == 1); +} diff --git a/tests/example-solution-own-test/expected_results.json b/tests/example-solution-own-test/expected_results.json new file mode 100644 index 0000000..fe48c75 --- /dev/null +++ b/tests/example-solution-own-test/expected_results.json @@ -0,0 +1,16 @@ +{ + "version": 3, + "status": "pass", + "tests": [ + { + "name": "a test from the exercise", + "test_code": "try testing.expectEqual(1, mod.value());", + "status": "pass", + "task_id": 1 + }, + { + "name": "a test the student wrote", + "status": "pass" + } + ] +} diff --git a/tests/example-solution-own-test/test_example_solution_own_test.zig b/tests/example-solution-own-test/test_example_solution_own_test.zig new file mode 100644 index 0000000..4aba6a6 --- /dev/null +++ b/tests/example-solution-own-test/test_example_solution_own_test.zig @@ -0,0 +1,12 @@ +// The slug ends in "test", so the module zig reports here is +// "test_example_solution_own_test" — which keeps us honest about finding +// the ".test." that separates the module from the test name. +const std = @import("std"); +const testing = std.testing; + +const mod = @import("example_solution_own_test.zig"); + +// task 1 +test "a test from the exercise" { + try testing.expectEqual(1, mod.value()); +} diff --git a/tests/example-string-mismatch/expected_results.json b/tests/example-string-mismatch/expected_results.json index 77be273..e61897e 100644 --- a/tests/example-string-mismatch/expected_results.json +++ b/tests/example-string-mismatch/expected_results.json @@ -1,13 +1,15 @@ { - "version": 2, + "version": 3, "status": "fail", "tests": [ { "name": "sanity pass", + "test_code": "try testing.expect(1 + 1 == 2);", "status": "pass" }, { "name": "greeting is exactly Hello, World!", + "test_code": "try testing.expectEqualStrings(\"Hello, World!\", mod.greeting());", "status": "fail", "message": "FAIL (TestExpectedEqual)\n/opt/zig/lib/std/testing.zig:682:9: in expectEqualStrings (std.zig)\n return error.TestExpectedEqual;\n ^\ntest_example_string_mismatch.zig:11:5: in test.greeting is exactly Hello, World! (test_example_string_mismatch.zig)\n try testing.expectEqualStrings(\"Hello, World!\", mod.greeting());\n ^", "output": "====== expected this output: =========\nHello, World!␃\n\n======== instead found this: =========\nGoodbye, Mars!␃\n\n======================================\nFirst difference occurs on line 1:\nexpected:\nHello, World!\n^ ('\\x48')\nfound:\nGoodbye, Mars!\n^ ('\\x47')" diff --git a/tests/example-success/expected_results.json b/tests/example-success/expected_results.json index 19c8ae1..b9f547b 100644 --- a/tests/example-success/expected_results.json +++ b/tests/example-success/expected_results.json @@ -1,41 +1,50 @@ { - "version": 2, + "version": 3, "status": "pass", "tests": [ { "name": "year not divisible by 4 in common year", + "test_code": "try testing.expect(!leap.leap(2015));", "status": "pass" }, { "name": "year divisible by 2, not divisible by 4 in common year", + "test_code": "try testing.expect(!leap.leap(1970));", "status": "pass" }, { "name": "year divisible by 4, not divisible by 100 in leap year", + "test_code": "try testing.expect(leap.leap(1996));", "status": "pass" }, { "name": "year divisible by 4 and 5 is still a leap year", + "test_code": "try testing.expect(leap.leap(1960));", "status": "pass" }, { "name": "year divisible by 100, not divisible by 400 in common year", + "test_code": "try testing.expect(!leap.leap(2100));", "status": "pass" }, { "name": "year divisible by 100 but not by 3 is still not a leap year", + "test_code": "try testing.expect(!leap.leap(1900));", "status": "pass" }, { "name": "year divisible by 400 is leap year", + "test_code": "try testing.expect(leap.leap(2000));", "status": "pass" }, { "name": "year divisible by 400 but not by 125 is still a leap year", + "test_code": "try testing.expect(leap.leap(2400));", "status": "pass" }, { "name": "year divisible by 200, not divisible by 400 in common year", + "test_code": "try testing.expect(!leap.leap(1800));", "status": "pass" } ] diff --git a/tests/example-syntax-error/expected_results.json b/tests/example-syntax-error/expected_results.json index ff0d42d..932ed46 100644 --- a/tests/example-syntax-error/expected_results.json +++ b/tests/example-syntax-error/expected_results.json @@ -1,5 +1,5 @@ { - "version": 2, + "version": 3, "status": "error", "message": "example_syntax_error.zig:1:16: error: expected '(', found '^'\npub fn leap1236^&gv13n\n ^" } diff --git a/tests/example-test-code-edge-cases/example_test_code_edge_cases.zig b/tests/example-test-code-edge-cases/example_test_code_edge_cases.zig new file mode 100644 index 0000000..77b4589 --- /dev/null +++ b/tests/example-test-code-edge-cases/example_test_code_edge_cases.zig @@ -0,0 +1,7 @@ +pub fn brace() []const u8 { + return "}"; +} + +pub fn nested() u32 { + return 3; +} diff --git a/tests/example-test-code-edge-cases/expected_results.json b/tests/example-test-code-edge-cases/expected_results.json new file mode 100644 index 0000000..9f39f05 --- /dev/null +++ b/tests/example-test-code-edge-cases/expected_results.json @@ -0,0 +1,25 @@ +{ + "version": 3, + "status": "pass", + "tests": [ + { + "name": "brace inside a string literal", + "test_code": "try testing.expectEqualStrings(\"}\", edge.brace());", + "status": "pass" + }, + { + "name": "nested blocks keep their relative indentation", + "test_code": "const cfg = .{ .a = 1, .b = .{ .c = 2 } };\nif (cfg.a == 1) {\n try testing.expectEqual(3, edge.nested());\n}", + "status": "pass" + }, + { + "name": "empty body", + "status": "pass" + }, + { + "name": "multiline string literal in the body", + "test_code": "const text =\n \\\\line one\n \\\\line two\n;\ntry testing.expect(text.len > 0);", + "status": "pass" + } + ] +} diff --git a/tests/example-test-code-edge-cases/test_example_test_code_edge_cases.zig b/tests/example-test-code-edge-cases/test_example_test_code_edge_cases.zig new file mode 100644 index 0000000..9f9f2d5 --- /dev/null +++ b/tests/example-test-code-edge-cases/test_example_test_code_edge_cases.zig @@ -0,0 +1,25 @@ +const std = @import("std"); +const testing = std.testing; + +const edge = @import("example_test_code_edge_cases.zig"); + +test "brace inside a string literal" { + try testing.expectEqualStrings("}", edge.brace()); +} + +test "nested blocks keep their relative indentation" { + const cfg = .{ .a = 1, .b = .{ .c = 2 } }; + if (cfg.a == 1) { + try testing.expectEqual(3, edge.nested()); + } +} + +test "empty body" {} + +test "multiline string literal in the body" { + const text = + \\line one + \\line two + ; + try testing.expect(text.len > 0); +} diff --git a/tests/example-user-output/expected_results.json b/tests/example-user-output/expected_results.json index 65ff06d..ca6df5e 100644 --- a/tests/example-user-output/expected_results.json +++ b/tests/example-user-output/expected_results.json @@ -1,9 +1,10 @@ { - "version": 2, + "version": 3, "status": "fail", "tests": [ { "name": "greet returns a five-letter greeting", + "test_code": "const result = try mod.greet();\ntry testing.expect(result.len == 5);", "status": "fail", "message": "FAIL (TestUnexpectedResult)\n/opt/zig/lib/std/testing.zig:615:14: in expect (std.zig)\n if (!ok) return error.TestUnexpectedResult;\n ^\ntest_example_user_output.zig:8:5: in test.greet returns a five-letter greeting (test_example_user_output.zig)\n try testing.expect(result.len == 5);\n ^", "output": "debug: computed bad greeting"