Add shared memories (#1069)

Begin to implement wasm threading proposal in https://github.com/WebAssembly/threads/blob/master/proposals/threads/Overview.md

This PR just has shared memory attribute with wast and binary support.

Author: dschuff

auto_update_tests.py

@@ -4,7 +4,7 @@
 
 from scripts.test.support import run_command, split_wast
 from scripts.test.shared import (
-    ASM2WASM, S2WASM, WASM_SHELL, WASM_OPT, WASM_AS, WASM_DIS, WASM_CTOR_EVAL)
+    ASM2WASM, MOZJS, S2WASM, WASM_SHELL, WASM_OPT, WASM_AS, WASM_DIS, WASM_CTOR_EVAL)
 
 print '[ processing and updating testcases... ]\n'
 
@@ -199,6 +199,7 @@
     print '..', t
     t = os.path.join('test', t)
     cmd = WASM_DIS + [t]
+    if os.path.isfile(t + '.map'): cmd += ['--source-map', t + '.map']
     actual = run_command(cmd)
 
     open(t + '.fromBinary', 'w').write(actual)
@@ -223,18 +224,19 @@
         with open(out, 'w') as o: o.write(actual)
         with open(out + '.stdout', 'w') as o: o.write(stdout)
 
-print '\n[ checking binaryen.js testcases... ]\n'
-
-for s in sorted(os.listdir(os.path.join('test', 'binaryen.js'))):
-  if not s.endswith('.js'): continue
-  print s
-  f = open('a.js', 'w')
-  f.write(open(os.path.join('bin', 'binaryen.js')).read())
-  f.write(open(os.path.join('test', 'binaryen.js', s)).read())
-  f.close()
-  cmd = ['mozjs', 'a.js']
-  out = run_command(cmd, stderr=subprocess.STDOUT)
-  open(os.path.join('test', 'binaryen.js', s + '.txt'), 'w').write(out)
+if MOZJS:
+  print '\n[ checking binaryen.js testcases... ]\n'
+
+  for s in sorted(os.listdir(os.path.join('test', 'binaryen.js'))):
+    if not s.endswith('.js'): continue
+    print s
+    f = open('a.js', 'w')
+    f.write(open(os.path.join('bin', 'binaryen.js')).read())
+    f.write(open(os.path.join('test', 'binaryen.js', s)).read())
+    f.close()
+    cmd = [MOZJS, 'a.js']
+    out = run_command(cmd, stderr=subprocess.STDOUT)
+    with open(os.path.join('test', 'binaryen.js', s + '.txt'), 'w') as o: o.write(out)
 
 print '\n[ checking wasm-ctor-eval... ]\n'
 

scripts/test/s2wasm.py

@@ -29,7 +29,8 @@ def test_s2wasm():
       os.path.join(options.binaryen_test, 'dot_s', 'basics.s'),
       '--import-memory']
   output = run_command(cmd)
-  fail_if_not_contained(output, '(import "env" "memory" (memory $0 1))')
+  fail_if_not_contained(
+      output, '(import "env" "memory" (memory $0 1))')
 
   for dot_s_dir in ['dot_s', 'llvm_autogenerated']:
     dot_s_path = os.path.join(options.binaryen_test, dot_s_dir)

src/passes/Print.cpp

@@ -670,6 +670,7 @@ struct PrintSExpression : public Visitor<PrintSExpression> {
     printName(curr->name) << ' ';
     o << curr->initial;
     if (curr->max && curr->max != Memory::kMaxSize) o << ' ' << curr->max;
+    if (curr->shared) o << " shared";
     o << ")";
   }
   void visitMemory(Memory* curr) {

src/wasm-binary.h

@@ -515,6 +515,11 @@ enum MemoryAccess {
   NaturalAlignment = 0
 };
 
+enum MemoryFlags {
+  HasMaximum = 1 << 0,
+  IsShared = 1 << 1
+};
+
 } // namespace BinaryConsts
 
 
@@ -560,7 +565,7 @@ class WasmBinaryWriter : public Visitor<WasmBinaryWriter, void> {
   void write();
   void writeHeader();
   int32_t writeU32LEBPlaceholder();
-  void writeResizableLimits(Address initial, Address maximum, bool hasMaximum);
+  void writeResizableLimits(Address initial, Address maximum, bool hasMaximum, bool shared);
   int32_t startSection(BinaryConsts::Section code);
   void finishSection(int32_t start);
   int32_t startSubsection(BinaryConsts::UserSections::Subsection code);
@@ -712,7 +717,7 @@ class WasmBinaryBuilder {
 
   // gets a name in the combined function import+defined function space
   Name getFunctionIndexName(Index i);
-  void getResizableLimits(Address& initial, Address& max, Address defaultIfNoMax);
+  void getResizableLimits(Address& initial, Address& max, bool& shared, Address defaultIfNoMax);
   void readImports();
 
   std::vector<FunctionType*> functionTypes; // types of defined functions

src/wasm-s-parser.h

@@ -196,6 +196,7 @@ class SExpressionWasmBuilder {
   Expression* makeReturn(Element& s);
 
   WasmType parseOptionalResultType(Element& s, Index& i);
+  Index parseMemoryLimits(Element& s, Index i);
 
   void stringToBinary(const char* input, size_t size, std::vector<char>& data);
   void parseMemory(Element& s, bool preParseImport = false);

src/wasm.h

@@ -632,8 +632,9 @@ class Memory {
   // See comment in Table.
   bool exists;
   bool imported;
+  bool shared;
 
-  Memory() : initial(0), max(kMaxSize), exists(false), imported(false) {
+  Memory() : initial(0), max(kMaxSize), exists(false), imported(false), shared(false) {
     name = Name::fromInt(0);
   }
 };

src/wasm/wasm-binary.cpp

@@ -71,8 +71,11 @@ int32_t WasmBinaryWriter::writeU32LEBPlaceholder() {
   return ret;
 }
 
-void WasmBinaryWriter::writeResizableLimits(Address initial, Address maximum, bool hasMaximum) {
-  uint32_t flags = hasMaximum ? 1 : 0;
+void WasmBinaryWriter::writeResizableLimits(Address initial, Address maximum,
+                                            bool hasMaximum, bool shared) {
+  uint32_t flags =
+      (hasMaximum ? (uint32_t) BinaryConsts::HasMaximum : 0U) |
+      (shared ? (uint32_t) BinaryConsts::IsShared : 0U);
   o << U32LEB(flags);
   o << U32LEB(initial);
   if (hasMaximum) {
@@ -113,7 +116,8 @@ void WasmBinaryWriter::writeMemory() {
   if (debug) std::cerr << "== writeMemory" << std::endl;
   auto start = startSection(BinaryConsts::Section::Memory);
   o << U32LEB(1); // Define 1 memory
-  writeResizableLimits(wasm->memory.initial, wasm->memory.max, wasm->memory.max != Memory::kMaxSize);
+  writeResizableLimits(wasm->memory.initial, wasm->memory.max,
+                       wasm->memory.max != Memory::kMaxSize, wasm->memory.shared);
   finishSection(start);
 }
 
@@ -161,11 +165,12 @@ void WasmBinaryWriter::writeImports() {
       case ExternalKind::Function: o << U32LEB(getFunctionTypeIndex(import->functionType)); break;
       case ExternalKind::Table: {
         o << S32LEB(BinaryConsts::EncodedType::AnyFunc);
-        writeResizableLimits(wasm->table.initial, wasm->table.max, wasm->table.max != Table::kMaxSize);
+        writeResizableLimits(wasm->table.initial, wasm->table.max, wasm->table.max != Table::kMaxSize, /*shared=*/false);
         break;
       }
       case ExternalKind::Memory: {
-        writeResizableLimits(wasm->memory.initial, wasm->memory.max, wasm->memory.max != Memory::kMaxSize);
+        writeResizableLimits(wasm->memory.initial, wasm->memory.max,
+                             wasm->memory.max != Memory::kMaxSize, wasm->memory.shared);
         break;
       }
       case ExternalKind::Global:
@@ -368,7 +373,7 @@ void WasmBinaryWriter::writeFunctionTableDeclaration() {
   auto start = startSection(BinaryConsts::Section::Table);
   o << U32LEB(1); // Declare 1 table.
   o << S32LEB(BinaryConsts::EncodedType::AnyFunc);
-  writeResizableLimits(wasm->table.initial, wasm->table.max, wasm->table.max != Table::kMaxSize);
+  writeResizableLimits(wasm->table.initial, wasm->table.max, wasm->table.max != Table::kMaxSize, /*shared=*/false);
   finishSection(start);
 }
 
@@ -1237,7 +1242,7 @@ void WasmBinaryBuilder::readMemory() {
     throw ParseException("Memory cannot be both imported and defined");
   }
   wasm.memory.exists = true;
-  getResizableLimits(wasm.memory.initial, wasm.memory.max, Memory::kMaxSize);
+  getResizableLimits(wasm.memory.initial, wasm.memory.max, wasm.memory.shared, Memory::kMaxSize);
 }
 
 void WasmBinaryBuilder::readSignatures() {
@@ -1284,10 +1289,13 @@ Name WasmBinaryBuilder::getFunctionIndexName(Index i) {
   }
 }
 
-void WasmBinaryBuilder::getResizableLimits(Address& initial, Address& max, Address defaultIfNoMax) {
+void WasmBinaryBuilder::getResizableLimits(Address& initial, Address& max, bool &shared, Address defaultIfNoMax) {
   auto flags = getU32LEB();
   initial = getU32LEB();
-  bool hasMax = flags & 0x1;
+  bool hasMax = flags & BinaryConsts::HasMaximum;
+  bool isShared = flags & BinaryConsts::IsShared;
+  if (isShared && !hasMax) throw ParseException("shared memory must have max size");
+  shared = isShared;
   if (hasMax) max = getU32LEB();
   else max = defaultIfNoMax;
 }
@@ -1320,13 +1328,15 @@ void WasmBinaryBuilder::readImports() {
         if (elementType != BinaryConsts::EncodedType::AnyFunc) throw ParseException("Imported table type is not AnyFunc");
         wasm.table.exists = true;
         wasm.table.imported = true;
-        getResizableLimits(wasm.table.initial, wasm.table.max, Table::kMaxSize);
+        bool is_shared;
+        getResizableLimits(wasm.table.initial, wasm.table.max, is_shared, Table::kMaxSize);
+        if (is_shared) throw ParseException("Tables may not be shared");
         break;
       }
       case ExternalKind::Memory: {
         wasm.memory.exists = true;
         wasm.memory.imported = true;
-        getResizableLimits(wasm.memory.initial, wasm.memory.max, Memory::kMaxSize);
+        getResizableLimits(wasm.memory.initial, wasm.memory.max, wasm.memory.shared, Memory::kMaxSize);
         break;
       }
       case ExternalKind::Global: {
@@ -1759,7 +1769,9 @@ void WasmBinaryBuilder::readFunctionTableDeclaration() {
   wasm.table.exists = true;
   auto elemType = getS32LEB();
   if (elemType != BinaryConsts::EncodedType::AnyFunc) throw ParseException("ElementType must be AnyFunc in MVP");
-  getResizableLimits(wasm.table.initial, wasm.table.max, Table::kMaxSize);
+  bool is_shared;
+  getResizableLimits(wasm.table.initial, wasm.table.max, is_shared, Table::kMaxSize);
+  if (is_shared) throw ParseException("Tables may not be shared");
 }
 
 void WasmBinaryBuilder::readTableElements() {

src/wasm/wasm-s-parser.cpp

@@ -1440,10 +1440,28 @@ void SExpressionWasmBuilder::stringToBinary(const char* input, size_t size, std:
   data.resize(actual);
 }
 
+Index SExpressionWasmBuilder::parseMemoryLimits(Element& s, Index i) {
+  wasm.memory.initial = getCheckedAddress(s[i++], "excessive memory init");
+  if (i == s.size()) return i;
+  while (i < s.size() && s[i]->isStr()) {
+    auto* curr = s[i]->c_str();
+    i++;
+    if (strstr(curr, "shared")) {
+      wasm.memory.shared = strncmp(curr, "notshared", 9) != 0;
+      break;
+    }
+    uint64_t max = atoll(curr);
+    if (max > Memory::kMaxSize) throw ParseException("total memory must be <= 4GB");
+    wasm.memory.max = max;
+  }
+  return i;
+}
+
 void SExpressionWasmBuilder::parseMemory(Element& s, bool preParseImport) {
   if (wasm.memory.exists) throw ParseException("too many memories");
   wasm.memory.exists = true;
   wasm.memory.imported = preParseImport;
+  wasm.memory.shared = false;
   Index i = 1;
   if (s[i]->dollared()) {
     wasm.memory.name = s[i++]->str();
@@ -1478,14 +1496,9 @@ void SExpressionWasmBuilder::parseMemory(Element& s, bool preParseImport) {
       return;
     }
   }
-  wasm.memory.initial = getCheckedAddress(s[i++], "excessive memory init");
-  if (i == s.size()) return;
-  if (s[i]->isStr()) {
-    uint64_t max = atoll(s[i]->c_str());
-    if (max > Memory::kMaxSize) throw ParseException("total memory must be <= 4GB");
-    wasm.memory.max = max;
-    i++;
-  }
+  i = parseMemoryLimits(s, i);
+
+  // Parse memory initializers.
   while (i < s.size()) {
     Element& curr = *s[i];
     size_t j = 1;
@@ -1683,12 +1696,7 @@ void SExpressionWasmBuilder::parseImport(Element& s) {
     }
     // ends with the table element type
   } else if (im->kind == ExternalKind::Memory) {
-    if (j < inner.size()) {
-      wasm.memory.initial = getCheckedAddress(inner[j++], "excessive memory init size");
-    }
-    if (j < inner.size()) {
-      wasm.memory.max = getCheckedAddress(inner[j++], "excessive memory max size");
-    }
+    j = parseMemoryLimits(inner, j);
   }
   if (wasm.getImportOrNull(im->name)) throw ParseException("duplicate import", s.line, s.col);
   wasm.addImport(im.release());

test/memory-shared.wast

@@ -0,0 +1,4 @@
+(module
+ (memory $0 23 256 shared)
+)
+

test/memory-shared.wast.from-wast

@@ -0,0 +1,3 @@
+(module
+ (memory $0 23 256 shared)
+)