Update spec test suite (#2484)

This updates spec test suite to that of the current up-to-date version
of https://github.com/WebAssembly/spec repo.

- All failing tests are added in `BLACKLIST` in shared.py with reasons.
 - For tests that already existed and was passing and started failing
   after the update, we add the new test to the blacklist and preserve
   the old file by renaming it to 'old_[FILENAME].wast' not to lose test
   coverage. When the cause of the error is fixed or the unsupported
   construct gets support so the new test passes, we can delete the
   corresponding 'old_[FILENAME].wast' file.
- Adds support for `spectest.print_[type] style imports.

Author: aheejin

check.py

@@ -314,15 +314,9 @@ def check_expected(actual, expected):
         check_expected(actual, expected)
 
         # skip binary checks for tests that reuse previous modules by name, as that's a wast-only feature
-        if os.path.basename(wast) in ['exports.wast']:  # FIXME
+        if 'exports.wast' in os.path.basename(wast):  # FIXME
             continue
 
-        # we must ignore some binary format splits
-        splits_to_skip = {
-            'func.wast': [2],
-            'return.wast': [2]
-        }
-
         # check binary format. here we can verify execution of the final
         # result, no need for an output verification
         # some wast files cannot be split:
@@ -332,11 +326,6 @@ def check_expected(actual, expected):
             split_num = 0
             actual = ''
             for module, asserts in support.split_wast(wast):
-                skip = splits_to_skip.get(os.path.basename(wast)) or []
-                if split_num in skip:
-                    print('        skipping split module', split_num - 1)
-                    split_num += 1
-                    continue
                 print('        testing split module', split_num)
                 split_num += 1
                 support.write_wast('split.wast', module, asserts)

scripts/test/shared.py

@@ -402,12 +402,67 @@ def get_tests(test_dir, extensions=[]):
 else:
     options.spec_tests = options.spec_tests[:]
 
+# 11/27/2019: We updated the spec test suite to upstream spec repo. For some
+# files that started failing after this update, we added the new files to this
+# blacklist and preserved old ones by renaming them to 'old_[FILENAME].wast'
+# not to lose coverage. When the cause of the error is fixed or the unsupported
+# construct gets support so the new test passes, we can delete the
+# corresponding 'old_[FILENAME].wast' file. When you fix the new file and
+# delete the old file, make sure you rename the corresponding .wast.log file in
+# expected-output/ if any.
 SPEC_TEST_BLACKLIST = [
-    'binary.wast',   # Cannot parse binary modules
-    'linking.wast',  # No support for 'register' command
-    'nop.wast',      # Stacky code
-    'stack.wast',    # Stacky code
-    'unwind.wast'    # Stacky code
+    # Stacky code / notation
+    'block.wast',
+    'call.wast',
+    'float_exprs.wast',
+    'globals.wast',
+    'loop.wast',
+    'nop.wast',
+    'select.wast',
+    'stack.wast',
+    'unwind.wast',
+
+    # Binary module
+    'binary.wast',
+    'binary-leb128.wast',
+    'custom.wast',
+
+    # Empty 'then' or 'else' in 'if'
+    'if.wast',
+    'local_set.wast',
+    'store.wast',
+
+    # No module in a file
+    'token.wast',
+    'utf8-custom-section-id.wast',
+    'utf8-import-field.wast',
+    'utf8-import-module.wast',
+    'utf8-invalid-encoding.wast',
+
+    # 'register' command
+    'imports.wast',
+    'linking.wast',
+
+    # Misc. unsupported constructs
+    'call_indirect.wast',  # Empty (param) and (result)
+    'const.wast',  # Unparenthesized expression
+    'data.wast',  # Various unsupported (data) notations
+    'elem.wast',  # Unsupported 'offset' syntax in (elem)
+    'exports.wast',  # Multiple inlined exports for a function
+    'func.wast',  # Forward named type reference
+    'skip-stack-guard-page.wast',  # Hexadecimal style (0x..) in memory offset
+
+    # Untriaged: We don't know the cause of the error yet
+    'address.wast',  # wasm2js 'assert_return' failure
+    'br_if.wast',  # Validation error
+    'float_literals.wast',  # 'assert_return' failure
+    'int_literals.wast',  # 'assert_return' failure
+    'local_tee.wast',  # Validation failure
+    'memory_grow.wast',  # 'assert_return' failure
+    'start.wast',  # Assertion failure
+    'type.wast',  # 'assertion_invalid' failure
+    'unreachable.wast',  # Validation failure
+    'unreached-invalid.wast'  # 'assert_invalid' failure
 ]
 options.spec_tests = [t for t in options.spec_tests if os.path.basename(t) not
                       in SPEC_TEST_BLACKLIST]

scripts/test/spectest.js

@@ -1,3 +1,20 @@
-export function print(...args) {
-  console.log(...args);
+export function print() {
+  console.log();
+}
+export function print_i32(arg) {
+  console.log(arg, ' : i32');
+}
+export function print_f32(arg) {
+  console.log(arg, ' : f32');
+}
+export function print_f64(arg) {
+  console.log(arg, ' : f64');
+}
+export function print_i32_f32(arg0, arg1) {
+  console.log(arg0, ' : i32');
+  console.log(arg1, ' : f32');
+}
+export function print_f64_f64(arg0, arg1) {
+  console.log(arg0, ' : f64');
+  console.log(arg1, ' : f64');
 }

scripts/test/wasm2js.py

@@ -20,7 +20,7 @@
 from scripts.test import support
 
 tests = shared.get_tests(shared.options.binaryen_test)
-spec_tests = shared.get_tests(shared.get_test_dir('spec'), ['.wast'])
+spec_tests = shared.options.spec_tests
 spec_tests = [t for t in spec_tests if '.fail' not in t]
 wasm2js_tests = shared.get_tests(shared.get_test_dir('wasm2js'), ['.wast'])
 assert_tests = ['wasm2js.wast.asserts']

src/emscripten-optimizer/istring.h

@@ -151,6 +151,9 @@ struct IString {
   bool startsWith(const char* prefix) const {
     return stripPrefix(prefix) != nullptr;
   }
+  bool startsWith(const IString& prefix) const {
+    return startsWith(prefix.str);
+  }
 
   size_t size() const { return str ? strlen(str) : 0; }
 };

src/shell-interface.h

@@ -98,7 +98,7 @@ struct ShellExternalInterface : ModuleInstance::ExternalInterface {
   void importGlobals(std::map<Name, Literal>& globals, Module& wasm) override {
     // add spectest globals
     ModuleUtils::iterImportedGlobals(wasm, [&](Global* import) {
-      if (import->module == SPECTEST && import->base == GLOBAL) {
+      if (import->module == SPECTEST && import->base.startsWith(GLOBAL)) {
         switch (import->type) {
           case i32:
             globals[import->name] = Literal(int32_t(666));
@@ -133,7 +133,7 @@ struct ShellExternalInterface : ModuleInstance::ExternalInterface {
   }
 
   Literal callImport(Function* import, LiteralList& arguments) override {
-    if (import->module == SPECTEST && import->base == PRINT) {
+    if (import->module == SPECTEST && import->base.startsWith(PRINT)) {
       for (auto argument : arguments) {
         std::cout << argument << " : " << argument.type << '\n';
       }

src/tools/wasm-shell.cpp

@@ -164,7 +164,7 @@ static void run_asserts(Name moduleName,
         };
         ModuleUtils::iterImportedGlobals(wasm, reportUnknownImport);
         ModuleUtils::iterImportedFunctions(wasm, [&](Importable* import) {
-          if (import->module == SPECTEST && import->base == PRINT) {
+          if (import->module == SPECTEST && import->base.startsWith(PRINT)) {
             // We can handle it.
           } else {
             reportUnknownImport(import);
@@ -181,7 +181,7 @@ static void run_asserts(Name moduleName,
             // spec tests consider it illegal to use spectest.print in a table
             if (auto* import = wasm.getFunction(name)) {
               if (import->imported() && import->module == SPECTEST &&
-                  import->base == PRINT) {
+                  import->base.startsWith(PRINT)) {
                 std::cerr << "cannot put spectest.print in table\n";
                 invalid = true;
               }

test/spec/README.md

@@ -1,2 +1,11 @@
-This repository holds a mirror of the spec testsuite which is maintained
-[here](https://github.com/WebAssembly/spec/tree/master/ml-proto/test).
+This directory contains tests for the core WebAssembly semantics, as described in [Semantics.md](https://github.com/WebAssembly/design/blob/master/Semantics.md) and specified by the [spec interpreter](https://github.com/WebAssembly/spec/blob/master/interpreter).
+
+Tests are written in the [S-Expression script format](https://github.com/WebAssembly/spec/blob/master/interpreter/README.md#s-expression-syntax) defined by the interpreter.
+
+The test suite can be run with the spec interpreter as follows:
+```
+./run.py --wasm <path-to-wasm-interpreter>
+```
+where the path points to the spec interpreter executable (or a tool that understands similar options). If the binary is in the working directory, this option can be omitted.
+
+In addition, the option `--js <path-to-js-interpreter>` can be given to point to a stand-alone JavaScript interpreter supporting the WebAssembly API. If provided, all tests are also executed in JavaScript.