From 1e7c90e8026a856c6ab8c15f05032c2fa6aba964 Mon Sep 17 00:00:00 2001 From: Michael Allwright Date: Mon, 5 Oct 2026 18:04:24 +0200 Subject: [PATCH 1/3] Make run.sh POSIX sh --- bin/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/run.sh b/bin/run.sh index 5db936b..b420940 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -29,7 +29,7 @@ compilation_errors_file_name="compilation-errors" test_output_file_name="test-output.xml" results_file="${output_dir}/results.json" binary_file="${build_dir}/${slug}" -test_file_path="${build_dir}/${slug//-/_}_test.cpp" +test_file_path="${build_dir}/$(echo "${slug}" | tr - _)_test.cpp" # Create the output directory if it doesn't exist mkdir -p "${output_dir}" @@ -47,7 +47,7 @@ cmake -DEXERCISM_TEST_SUITE=1 -DEXERCISM_RUN_ALL_TESTS=1 . make 2> "${compilation_errors_file_name}" # In case of compilation errors the executable will not be created -[[ -f "./${slug}" ]] && chmod +x "./${slug}" && "./${slug}" -r xml -o "${test_output_file_name}" +[ -f "./${slug}" ] && chmod +x "./${slug}" && "./${slug}" -r xml -o "${test_output_file_name}" /opt/test-runner/bin/exercism_parser "${build_dir}/${test_output_file_name}" "${results_file}" "${build_dir}/${compilation_errors_file_name}" "${test_file_path}" cd - From e171a53c71a266942a8e546d576488a03df8a7be Mon Sep 17 00:00:00 2001 From: Jeremy Walker Date: Thu, 22 Oct 2026 13:22:18 +0900 Subject: [PATCH 2/3] Fix run-tests.sh and drop std::ptr_fun run-tests.sh checked $? after the rm that cleaned up the temporary files, so it saw the exit status of rm and the suite never failed on a mismatch. It now checks the result of diff directly. The line-number pattern used \d, which sed does not understand, so line numbers were never normalised. It now uses [0-9]\{1,\}. std::ptr_fun and std::not1 were removed in C++17, so helpers.h now uses a lambda instead. Co-authored-by: Michael Allwright Co-Authored-By: Claude Opus 5.5 --- bin/run-tests.sh | 19 +++++-------------- include/helpers.h | 8 ++++---- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/bin/run-tests.sh b/bin/run-tests.sh index 204542d..d13d8e0 100755 --- a/bin/run-tests.sh +++ b/bin/run-tests.sh @@ -22,26 +22,17 @@ for test_dir in tests/*; do bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}" - # OPTIONAL: Normalize the results file - # If the results.json file contains information that changes between - # different test runs (e.g. timing information or paths), you should normalize - # the results file to allow the diff comparison below to work as expected - # sed -i -E \ - # -e 's/Elapsed time: [0-9]+\.[0-9]+ seconds//g' \ - # -e "s~${test_dir_path}~/solution~g" \ - # "${results_file_path}" - # Add trailing newlines. Ignore line numbers. for i in "${results_file_path}" "${expected_results_file_path}"; do - sed 's/:\d\+:/:NN:/g' "${i}" "${i}" > "${i}.cleaned" + sed \ + -e 's/:[0-9]\{1,\}:/:NN:/g' \ + "${i}" "${i}" > "${i}.cleaned" done echo "${test_dir_name}: comparing results.json to expected_results.json" - diff "${results_file_path}.cleaned" "${expected_results_file_path}.cleaned" - rm "${results_file_path}.cleaned" "${expected_results_file_path}.cleaned" - - if [ $? -ne 0 ]; then + if ! diff "${results_file_path}.cleaned" "${expected_results_file_path}.cleaned"; then exit_code=1 fi + rm "${results_file_path}.cleaned" "${expected_results_file_path}.cleaned" done exit ${exit_code} diff --git a/include/helpers.h b/include/helpers.h index c8924bb..6243fae 100644 --- a/include/helpers.h +++ b/include/helpers.h @@ -9,15 +9,15 @@ // trim from start (in place) static inline void ltrim(std::string &s) { - s.erase(s.begin(), std::find_if(s.begin(), s.end(), - std::not1(std::ptr_fun(std::isspace)))); + s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) + { return !std::isspace(ch); })); } // trim from end (in place) static inline void rtrim(std::string &s) { - s.erase(std::find_if(s.rbegin(), s.rend(), - std::not1(std::ptr_fun(std::isspace))) + s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) + { return !std::isspace(ch); }) .base(), s.end()); } From 7fb2912bada1854afca6490dd34d7d038506b68a Mon Sep 17 00:00:00 2001 From: Jeremy Walker Date: Thu, 22 Oct 2026 14:55:32 +0900 Subject: [PATCH 3/3] Compare build errors by status only The expected results stored the exact compiler output, which changes with every compiler upgrade. For example, example-empty-file was recorded with GCC 12.2.1 in 2023 and the image now has GCC 15.2.0. Build errors are now compared by their status, so the suite no longer breaks when the compiler changes. Co-authored-by: Michael Allwright Co-Authored-By: Claude Opus 5.5 --- bin/run-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/run-tests.sh b/bin/run-tests.sh index d13d8e0..b760867 100755 --- a/bin/run-tests.sh +++ b/bin/run-tests.sh @@ -26,6 +26,7 @@ for test_dir in tests/*; do for i in "${results_file_path}" "${expected_results_file_path}"; do sed \ -e 's/:[0-9]\{1,\}:/:NN:/g' \ + -e '/^{"version": 3, "status": "error"/ s/"message": ".*"}$/"message": ""}/' \ "${i}" "${i}" > "${i}.cleaned" done echo "${test_dir_name}: comparing results.json to expected_results.json"