Python3-ify check.py and auto_update_tests.py (#2270)

I fixed flatten.bin.txt which seems to have just had some corrupted data, and I removed some fancy unicode from the spec comments tests, which I'm not sure it's important enough to figure out how to fix.

Fixes #1691

Author: kripken

.travis.yml

@@ -69,7 +69,7 @@ jobs:
         - if [ -f clang-tidy-diff.sh ]; then ./clang-tidy-diff.sh; fi
         - make -j2 install
         - cd ${TRAVIS_BUILD_DIR}
-        - ./check.py --binaryen-bin=${BUILD_SUBDIR}/install/bin
+        - python3 ./check.py --binaryen-bin=${BUILD_SUBDIR}/install/bin
 
     - <<: *test-ubuntu
       env: |
@@ -125,11 +125,11 @@ jobs:
         - alpine() { docker exec -it alpine "$@"; }
       install:
         - alpine apk update
-        - alpine apk add build-base cmake git python2
+        - alpine apk add build-base cmake git python2 python3
       script:
         - alpine cmake .
         - alpine make -j2
-        - alpine ./check.py
+        - alpine python3 ./check.py
 
     # Build statically linked release binaries with gcc 6.3 on Alpine Linux
     # (inside chroot). If building a tagged commit, then deploy release tarball

auto_update_tests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2015 WebAssembly Community Group participants
 #
@@ -19,7 +19,7 @@
 import subprocess
 import sys
 
-from scripts.test.support import run_command, split_wast, node_test_glue, node_has_webassembly
+from scripts.test.support import run_command, split_wast, write_wast, node_test_glue, node_has_webassembly
 from scripts.test.shared import (
     ASM2WASM, MOZJS, NODEJS, WASM_OPT, WASM_AS, WASM_DIS,
     WASM_CTOR_EVAL, WASM_REDUCE, WASM_METADCE, BINARYEN_INSTALL_DIR,
@@ -30,7 +30,7 @@
 
 
 def update_asm_js_tests():
-  print '[ processing and updating testcases... ]\n'
+  print('[ processing and updating testcases... ]\n')
   for asm in sorted(os.listdir('test')):
     if asm.endswith('.asm.js'):
       for precise in [0, 1, 2]:
@@ -57,15 +57,15 @@ def update_asm_js_tests():
             cmd += ['--no-legalize-javascript-ffi']
           if precise and opts:
             # test mem init importing
-            open('a.mem', 'wb').write(asm)
+            open('a.mem', 'wb').write(bytes(asm, 'utf-8'))
             cmd += ['--mem-init=a.mem']
             if asm[0] == 'e':
               cmd += ['--mem-base=1024']
           if '4GB' in asm:
             cmd += ['--mem-max=4294967296']
           if 'i64' in asm or 'wasm-only' in asm or 'noffi' in asm:
             cmd += ['--wasm-only']
-          print ' '.join(cmd)
+          print(' '.join(cmd))
           actual = run_command(cmd)
           with open(os.path.join('test', wasm), 'w') as o:
             o.write(actual)
@@ -75,33 +75,33 @@ def update_asm_js_tests():
 
 
 def update_wasm_opt_tests():
-  print '\n[ checking wasm-opt -o notation... ]\n'
+  print('\n[ checking wasm-opt -o notation... ]\n')
   wast = os.path.join('test', 'hello_world.wast')
   cmd = WASM_OPT + [wast, '-o', 'a.wast', '-S']
   run_command(cmd)
   open(wast, 'w').write(open('a.wast').read())
 
-  print '\n[ checking wasm-opt parsing & printing... ]\n'
+  print('\n[ checking wasm-opt parsing & printing... ]\n')
   for t in sorted(os.listdir(os.path.join('test', 'print'))):
     if t.endswith('.wast'):
-      print '..', t
+      print('..', t)
       wasm = os.path.basename(t).replace('.wast', '')
       cmd = WASM_OPT + [os.path.join('test', 'print', t), '--print', '-all']
-      print '    ', ' '.join(cmd)
+      print('    ', ' '.join(cmd))
       actual = subprocess.check_output(cmd)
-      print cmd, actual
-      with open(os.path.join('test', 'print', wasm + '.txt'), 'w') as o:
+      print(cmd, actual)
+      with open(os.path.join('test', 'print', wasm + '.txt'), 'wb') as o:
         o.write(actual)
       cmd = WASM_OPT + [os.path.join('test', 'print', t), '--print-minified', '-all']
-      print '    ', ' '.join(cmd)
+      print('    ', ' '.join(cmd))
       actual = subprocess.check_output(cmd)
-      with open(os.path.join('test', 'print', wasm + '.minified.txt'), 'w') as o:
+      with open(os.path.join('test', 'print', wasm + '.minified.txt'), 'wb') as o:
         o.write(actual)
 
-  print '\n[ checking wasm-opt passes... ]\n'
+  print('\n[ checking wasm-opt passes... ]\n')
   for t in sorted(os.listdir(os.path.join('test', 'passes'))):
     if t.endswith(('.wast', '.wasm')):
-      print '..', t
+      print('..', t)
       binary = '.wasm' in t
       base = os.path.basename(t).replace('.wast', '').replace('.wasm', '')
       passname = base
@@ -112,8 +112,7 @@ def update_wasm_opt_tests():
       actual = ''
       for module, asserts in split_wast(t):
         assert len(asserts) == 0
-        with open('split.wast', 'w') as o:
-          o.write(module)
+        write_wast('split.wast', module)
         cmd = WASM_OPT + opts + ['split.wast', '--print']
         actual += run_command(cmd)
       with open(os.path.join('test', 'passes', base + ('.bin' if binary else '') + '.txt'), 'w') as o:
@@ -127,21 +126,21 @@ def update_wasm_opt_tests():
           with open(t + '.wat', 'w') as o:
             o.write(i.read())
 
-  print '\n[ checking wasm-opt testcases... ]\n'
+  print('\n[ checking wasm-opt testcases... ]\n')
   for t in os.listdir('test'):
     if t.endswith('.wast') and not t.startswith('spec'):
-      print '..', t
+      print('..', t)
       t = os.path.join('test', t)
       f = t + '.from-wast'
       cmd = WASM_OPT + [t, '--print', '-all']
       actual = run_command(cmd)
       actual = actual.replace('printing before:\n', '')
       open(f, 'w').write(actual)
 
-  print '\n[ checking wasm-opt debugInfo read-write... ]\n'
+  print('\n[ checking wasm-opt debugInfo read-write... ]\n')
   for t in os.listdir('test'):
     if t.endswith('.fromasm') and 'debugInfo' in t:
-      print '..', t
+      print('..', t)
       t = os.path.join('test', t)
       f = t + '.read-written'
       run_command(WASM_AS + [t, '--source-map=a.map', '-o', 'a.wasm', '-g'])
@@ -151,21 +150,21 @@ def update_wasm_opt_tests():
 
 
 def update_bin_fmt_tests():
-  print '\n[ checking binary format testcases... ]\n'
+  print('\n[ checking binary format testcases... ]\n')
   for wast in sorted(os.listdir('test')):
     if wast.endswith('.wast') and wast not in []:  # blacklist some known failures
       for debug_info in [0, 1]:
         cmd = WASM_AS + [os.path.join('test', wast), '-o', 'a.wasm', '-all']
         if debug_info:
           cmd += ['-g']
-        print ' '.join(cmd)
+        print(' '.join(cmd))
         if os.path.exists('a.wasm'):
           os.unlink('a.wasm')
         subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         assert os.path.exists('a.wasm')
 
         cmd = WASM_DIS + ['a.wasm', '-o', 'a.wast']
-        print ' '.join(cmd)
+        print(' '.join(cmd))
         if os.path.exists('a.wast'):
           os.unlink('a.wast')
         subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -179,7 +178,7 @@ def update_bin_fmt_tests():
 
 
 def update_example_tests():
-  print '\n[ checking example testcases... ]\n'
+  print('\n[ checking example testcases... ]\n')
   for t in sorted(os.listdir(os.path.join('test', 'example'))):
     output_file = os.path.join(options.binaryen_bin, 'example')
     libdir = os.path.join(BINARYEN_INSTALL_DIR, 'lib')
@@ -188,11 +187,11 @@ def update_example_tests():
       # check if there is a trace in the file, if so, we should build it
       out = subprocess.Popen([os.path.join('scripts', 'clean_c_api_trace.py'), os.path.join('test', 'example', t)], stdout=subprocess.PIPE).communicate()[0]
       if len(out) == 0:
-        print '  (no trace in ', t, ')'
+        print('  (no trace in ', t, ')')
         continue
-      print '  (will check trace in ', t, ')'
+      print('  (will check trace in ', t, ')')
       src = 'trace.cpp'
-      with open(src, 'w') as o:
+      with open(src, 'wb') as o:
         o.write(out)
       expected = os.path.join('test', 'example', t + '.txt')
     else:
@@ -204,26 +203,26 @@ def update_example_tests():
     extra = [os.environ.get('CC') or 'gcc',
              src, '-c', '-o', 'example.o',
              '-Isrc', '-g', '-L' + libdir, '-pthread']
-    print 'build: ', ' '.join(extra)
+    print('build: ', ' '.join(extra))
     if src.endswith('.cpp'):
       extra += ['-std=c++11']
-    print os.getcwd()
+    print(os.getcwd())
     subprocess.check_call(extra)
     # Link against the binaryen C library DSO, using rpath
     cmd = ['example.o', '-L' + libdir, '-lbinaryen', '-Wl,-rpath,' + os.path.abspath(libdir)] + cmd
-    print '  ', t, src, expected
+    print('  ', t, src, expected)
     if os.environ.get('COMPILER_FLAGS'):
       for f in os.environ.get('COMPILER_FLAGS').split(' '):
         cmd.append(f)
     cmd = [os.environ.get('CXX') or 'g++', '-std=c++11'] + cmd
     try:
-      print 'link: ', ' '.join(cmd)
+      print('link: ', ' '.join(cmd))
       subprocess.check_call(cmd)
-      print 'run...', output_file
+      print('run...', output_file)
       proc = subprocess.Popen([output_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
       actual, err = proc.communicate()
       assert proc.returncode == 0, [proc.returncode, actual, err]
-      with open(expected, 'w') as o:
+      with open(expected, 'wb') as o:
         o.write(actual)
     finally:
       os.remove(output_file)
@@ -233,10 +232,10 @@ def update_example_tests():
 
 
 def update_wasm_dis_tests():
-  print '\n[ checking wasm-dis on provided binaries... ]\n'
+  print('\n[ checking wasm-dis on provided binaries... ]\n')
   for t in os.listdir('test'):
     if t.endswith('.wasm') and not t.startswith('spec'):
-      print '..', t
+      print('..', t)
       t = os.path.join('test', t)
       cmd = WASM_DIS + [t]
       if os.path.isfile(t + '.map'):
@@ -248,19 +247,19 @@ def update_wasm_dis_tests():
 
 def update_binaryen_js_tests():
   if not (MOZJS or NODEJS):
-    print 'no vm to run binaryen.js tests'
+    print('no vm to run binaryen.js tests')
     return
 
   if not os.path.exists(BINARYEN_JS):
-    print 'no binaryen.js build to test'
+    print('no binaryen.js build to test')
     return
 
-  print '\n[ checking binaryen.js testcases... ]\n'
+  print('\n[ checking binaryen.js testcases... ]\n')
   node_has_wasm = NODEJS and node_has_webassembly(NODEJS)
   for s in sorted(os.listdir(os.path.join('test', 'binaryen.js'))):
     if not s.endswith('.js'):
       continue
-    print s
+    print(s)
     f = open('a.js', 'w')
     f.write(open(BINARYEN_JS).read())
     if NODEJS:
@@ -279,14 +278,14 @@ def update_binaryen_js_tests():
       with open(os.path.join('test', 'binaryen.js', s + '.txt'), 'w') as o:
         o.write(out)
     else:
-      print 'Skipping ' + test_path + ' because WebAssembly might not be supported'
+      print('Skipping ' + test_path + ' because WebAssembly might not be supported')
 
 
 def update_ctor_eval_tests():
-  print '\n[ checking wasm-ctor-eval... ]\n'
+  print('\n[ checking wasm-ctor-eval... ]\n')
   for t in os.listdir(os.path.join('test', 'ctor-eval')):
     if t.endswith(('.wast', '.wasm')):
-      print '..', t
+      print('..', t)
       t = os.path.join('test', 'ctor-eval', t)
       ctors = open(t + '.ctors').read().strip()
       cmd = WASM_CTOR_EVAL + [t, '-o', 'a.wast', '-S', '--ctors', ctors]
@@ -298,10 +297,10 @@ def update_ctor_eval_tests():
 
 
 def update_metadce_tests():
-  print '\n[ checking wasm-metadce... ]\n'
+  print('\n[ checking wasm-metadce... ]\n')
   for t in os.listdir(os.path.join('test', 'metadce')):
     if t.endswith(('.wast', '.wasm')):
-      print '..', t
+      print('..', t)
       t = os.path.join('test', 'metadce', t)
       graph = t + '.graph.txt'
       cmd = WASM_METADCE + [t, '--graph-file=' + graph, '-o', 'a.wast', '-S']
@@ -317,14 +316,14 @@ def update_metadce_tests():
 def update_reduce_tests():
   if not has_shell_timeout():
     return
-  print '\n[ checking wasm-reduce ]\n'
+  print('\n[ checking wasm-reduce ]\n')
   for t in os.listdir(os.path.join('test', 'reduce')):
     if t.endswith('.wast'):
-      print '..', t
+      print('..', t)
       t = os.path.join('test', 'reduce', t)
       # convert to wasm
       run_command(WASM_AS + [t, '-o', 'a.wasm'])
-      print run_command(WASM_REDUCE + ['a.wasm', '--command=%s b.wasm --fuzz-exec' % WASM_OPT[0], '-t', 'b.wasm', '-w', 'c.wasm'])
+      print(run_command(WASM_REDUCE + ['a.wasm', '--command=%s b.wasm --fuzz-exec' % WASM_OPT[0], '-t', 'b.wasm', '-w', 'c.wasm']))
       expected = t + '.txt'
       run_command(WASM_DIS + ['c.wasm', '-o', expected])
 
@@ -342,7 +341,7 @@ def main():
   update_reduce_tests()
   update_binaryen_js_tests()
 
-  print '\n[ success! ]'
+  print('\n[ success! ]')
 
 
 if __name__ == '__main__':