Allow running a portion of binaryen test suite (#2286)

Author: quantum5

README.md

@@ -383,7 +383,7 @@ The `check.py` script supports some options:
  * If an interpreter is provided, we run the output through it, checking for
    parse errors.
  * If tests are provided, we run exactly those. If none are provided, we run
-   them all.
+   them all. To see what tests are available, run `./check.py --list-suites`.
  * Some tests require `emcc` or `nodejs` in the path. They will not run if the
    tool cannot be found, and you'll see a warning.
  * We have tests from upstream in `tests/spec`, in git submodules. Running

check.py

@@ -19,6 +19,7 @@
 import subprocess
 import sys
 import unittest
+from collections import OrderedDict
 
 from scripts.test.support import run_command, split_wast, write_wast
 from scripts.test.shared import (
@@ -254,6 +255,10 @@ def run_wasm_metadce_tests():
 
 
 def run_wasm_reduce_tests():
+  if not has_shell_timeout():
+    print('\n[ skipping wasm-reduce testcases]\n')
+    return
+
   print('\n[ checking wasm-reduce testcases]\n')
 
   # fixed testcases
@@ -285,13 +290,13 @@ def run_wasm_reduce_tests():
 def run_spec_tests():
   print('\n[ checking wasm-shell spec testcases... ]\n')
 
-  if len(requested) == 0:
+  if not options.spec_tests:
     # FIXME we support old and new memory formats, for now, until 0xc, and so can't pass this old-style test.
     BLACKLIST = ['binary.wast']
     # FIXME to update the spec to 0xd, we need to implement (register "name") for import.wast
     spec_tests = [os.path.join('spec', t) for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'spec'))) if t not in BLACKLIST]
   else:
-    spec_tests = requested[:]
+    spec_tests = options.spec_tests[:]
 
   for t in spec_tests:
     if t.startswith('spec') and t.endswith('.wast'):
@@ -412,6 +417,10 @@ def run_validator_tests():
 
 
 def run_vanilla_tests():
+  if not (has_vanilla_emcc and has_vanilla_llvm and 0):
+    print('\n[ skipping emcc WASM_BACKEND testcases...]\n')
+    return
+
   print('\n[ checking emcc WASM_BACKEND testcases...]\n')
 
   try:
@@ -522,30 +531,37 @@ def run_unittest():
     raise Exception("unittest failed")
 
 
+TEST_SUITES = OrderedDict([
+  ('help-messages', run_help_tests),
+  ('wasm-opt', run_wasm_opt_tests),
+  ('asm2wasm', asm2wasm.test_asm2wasm),
+  ('asm2wasm-binary', asm2wasm.test_asm2wasm_binary),
+  ('wasm-dis', run_wasm_dis_tests),
+  ('crash', run_crash_tests),
+  ('dylink', run_dylink_tests),
+  ('ctor-eval', run_ctor_eval_tests),
+  ('wasm-metadce', run_wasm_metadce_tests),
+  ('wasm-reduce', run_wasm_reduce_tests),
+  ('spec', run_spec_tests),
+  ('binaryenjs', binaryenjs.test_binaryen_js),
+  ('lld', lld.test_wasm_emscripten_finalize),
+  ('wasm2js', wasm2js.test_wasm2js),
+  ('validator', run_validator_tests),
+  ('vanilla', run_vanilla_tests),
+  ('gcc', run_gcc_tests),
+  ('unit', run_unittest),
+])
+
+
 # Run all the tests
 def main():
-  run_help_tests()
-  run_wasm_opt_tests()
-  asm2wasm.test_asm2wasm()
-  asm2wasm.test_asm2wasm_binary()
-  run_wasm_dis_tests()
-  run_crash_tests()
-  run_dylink_tests()
-  run_ctor_eval_tests()
-  run_wasm_metadce_tests()
-  if has_shell_timeout():
-    run_wasm_reduce_tests()
-  run_spec_tests()
-  binaryenjs.test_binaryen_js()
-  lld.test_wasm_emscripten_finalize()
-  wasm2js.test_wasm2js()
-  run_validator_tests()
-  if has_vanilla_emcc and has_vanilla_llvm and 0:
-    run_vanilla_tests()
-  print('\n[ checking example testcases... ]\n')
-  if options.run_gcc_tests:
-    run_gcc_tests()
-  run_unittest()
+  if options.list_suites:
+    for suite in TEST_SUITES.keys():
+      print(suite)
+    return 0
+
+  for test in requested or TEST_SUITES.keys():
+    TEST_SUITES[test]()
 
   # Check/display the results
   if shared.num_failures == 0:

scripts/test/shared.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+
 import argparse
 import difflib
 import glob
@@ -41,13 +43,6 @@ def parse_args(args):
       action='store_false',
       help=('If set, the whole test suite will run to completion independent of'
             ' earlier errors.'))
-  parser.add_argument(
-      '--run-gcc-tests', dest='run_gcc_tests', action='store_true', default=True,
-      help=('Chooses whether to run the tests that require building with native'
-            ' GCC. Default: true.'))
-  parser.add_argument(
-      '--no-run-gcc-tests', dest='run_gcc_tests', action='store_false',
-      help='If set, disables the native GCC tests.')
 
   parser.add_argument(
       '--interpreter', dest='interpreter', default='',
@@ -73,8 +68,15 @@ def parse_args(args):
       help=('If specified, all unfreed (but still referenced) pointers at the'
             ' end of execution are considered memory leaks. Default: disabled.'))
   parser.add_argument(
-      'positional_args', metavar='tests', nargs=argparse.REMAINDER,
-      help='Names specific tests to run.')
+      '--spec-test', action='append', nargs='*', default=[], dest='spec_tests',
+      help='Names specific spec tests to run.')
+  parser.add_argument(
+      'positional_args', metavar='TEST_SUITE', nargs='*',
+      help=('Names specific test suites to run. Use --list-suites to see a '
+            'list of all test suites'))
+  parser.add_argument(
+      '--list-suites', action='store_true',
+      help='List the test suites that can be run.')
 
   return parser.parse_args(args)
 
@@ -89,7 +91,7 @@ def parse_args(args):
 def warn(text):
   global warnings
   warnings.append(text)
-  print('warning:', text)
+  print('warning:', text, file=sys.stderr)
 
 
 # setup