Auto-update spec test outputs (#2481)

This makes auto_update_tests.py update spec test outputs (ones that are
printed with `spectest.print` import) and extracts spec tests blacklist
into shared.py with comments for reasons why each of them fails.

Also deletes if-label-scope.fail.wast.log because it does not seem to
match with any of existing tests.

Author: aheejin

auto_update_tests.py

@@ -310,6 +310,20 @@ def update_reduce_tests():
         support.run_command(shared.WASM_DIS + ['c.wasm', '-o', expected])
 
 
+def update_spec_tests():
+    print('\n[ updating wasm-shell spec testcases... ]\n')
+
+    for t in shared.options.spec_tests:
+        print('..', os.path.basename(t))
+
+        cmd = shared.WASM_SHELL + [t]
+        expected = os.path.join(shared.get_test_dir('spec'), 'expected-output', os.path.basename(t) + '.log')
+        if os.path.isfile(expected):
+            stdout = support.run_command(cmd, stderr=subprocess.PIPE)
+            with open(expected, 'w') as o:
+                o.write(stdout)
+
+
 TEST_SUITES = OrderedDict([
     ('wasm-opt', update_wasm_opt_tests),
     ('asm2wasm', update_asm_js_tests),
@@ -318,6 +332,7 @@ def update_reduce_tests():
     ('ctor-eval', update_ctor_eval_tests),
     ('wasm-metadce', update_metadce_tests),
     ('wasm-reduce', update_reduce_tests),
+    ('spec', update_spec_tests),
     ('binaryenjs', update_binaryen_js_tests),
     ('lld', lld.update_lld_tests),
     ('wasm2js', wasm2js.update_wasm2js_tests),

check.py

@@ -278,27 +278,11 @@ def run_wasm_reduce_tests():
 def run_spec_tests():
     print('\n[ checking wasm-shell spec testcases... ]\n')
 
-    if not shared.options.spec_tests:
-        # FIXME we support old and new memory formats, for now, until 0xc, and so can't pass this old-style test.
-        BLACKLIST = ['binary.wast']
-        # FIXME to update the spec to 0xd, we need to implement (register "name") for import.wast
-        spec_tests = shared.get_tests(shared.get_test_dir('spec'), ['.wast'])
-        spec_tests = [t for t in spec_tests if os.path.basename(t) not in BLACKLIST]
-    else:
-        spec_tests = shared.options.spec_tests[:]
-
-    for wast in spec_tests:
+    for wast in shared.options.spec_tests:
         print('..', os.path.basename(wast))
 
-        # skip checks for some tests
-        if os.path.basename(wast) in ['linking.wast', 'nop.wast', 'stack.wast', 'typecheck.wast', 'unwind.wast']:    # FIXME
-            continue
-
         def run_spec_test(wast):
             cmd = shared.WASM_SHELL + [wast]
-            # we must skip the stack machine portions of spec tests or apply other extra args
-            extra = {}
-            cmd = cmd + (extra.get(os.path.basename(wast)) or [])
             return support.run_command(cmd, stderr=subprocess.PIPE)
 
         def run_opt_test(wast):

scripts/test/shared.py

@@ -397,6 +397,22 @@ def get_tests(test_dir, extensions=[]):
     warn('no functional emcc submodule found')
 
 
+if not options.spec_tests:
+    options.spec_tests = get_tests(get_test_dir('spec'), ['.wast'])
+else:
+    options.spec_tests = options.spec_tests[:]
+
+SPEC_TEST_BLACKLIST = [
+    'binary.wast',   # Cannot parse binary modules
+    'linking.wast',  # No support for 'register' command
+    'nop.wast',      # Stacky code
+    'stack.wast',    # Stacky code
+    'unwind.wast'    # Stacky code
+]
+options.spec_tests = [t for t in options.spec_tests if os.path.basename(t) not
+                      in SPEC_TEST_BLACKLIST]
+
+
 # check utilities