Fuzzer: Add an option to preserve imports and exports (#7300)

Normally the fuzzer will add new imports (for the JS stuff we can
call), and new exports as it adds functions. This adds an option to
avoid that, and instead keep the imports and exports fixed. This
is useful when we are used to modify an existing fuzzer's testcase,
as its connection to the JS side should be considered fixed (and it
will run in that fuzzer's JS, not ours).

This is added as

--fuzz-preserve-imports-exports

for wasm-opt.

Author: kripken

CHANGELOG.md

@@ -15,6 +15,10 @@ full changeset diff at the end of each section.
 Current Trunk
 -------------
 
+ - Add an option to preserve imports and exports in the fuzzer (for fuzzer
+   harnesses where they only want Binaryen to modify their given testcases, not
+   generate new things in them).
+
 v122
 ----
 

scripts/fuzz_opt.py

@@ -1649,7 +1649,6 @@ def handle(self, wasm):
         # run, or if the wasm errored during instantiation, which can happen due
         # to a testcase with a segment out of bounds, say).
         if output != IGNORE and not output.startswith(INSTANTIATE_ERROR):
-
             assert FUZZ_EXEC_CALL_PREFIX in output
 
     def ensure(self):
@@ -1756,6 +1755,52 @@ def can_run_on_wasm(self, wasm):
         return not CLOSED_WORLD and all_disallowed(['shared-everything']) and not NANS
 
 
+# Test --fuzz-preserve-imports-exports, which never modifies imports or exports.
+class PreserveImportsExports(TestCaseHandler):
+    frequency = 0.1
+
+    def handle(self, wasm):
+        # We will later verify that no imports or exports changed, by comparing
+        # to the unprocessed original text.
+        original = run([in_bin('wasm-opt'), wasm] + FEATURE_OPTS + ['--print'])
+
+        # We leave if the module has (ref exn) in struct fields (because we have
+        # no way to generate an exn in a non-function context, and if we picked
+        # that struct for a global, we'd end up needing a (ref exn) in the
+        # global scope, which is impossible). The fuzzer is designed to be
+        # careful not to emit that in testcases, but after the optimizer runs,
+        # we may end up with struct fields getting refined to that, so we need
+        # this extra check (which should be hit very rarely).
+        structs = [line for line in original.split('\n') if '(struct ' in line]
+        if '(ref exn)' in '\n'.join(structs):
+            note_ignored_vm_run('has non-nullable exn in struct')
+            return
+
+        # Generate some random input data.
+        data = abspath('preserve_input.dat')
+        make_random_input(random_size(), data)
+
+        # Process the existing wasm file.
+        processed = run([in_bin('wasm-opt'), data] + FEATURE_OPTS + [
+            '-ttf',
+            '--fuzz-preserve-imports-exports',
+            '--initial-fuzz=' + wasm,
+            '--print',
+        ])
+
+        def get_relevant_lines(wat):
+            # Imports and exports are relevant.
+            lines = [line for line in wat.splitlines() if '(export ' in line or '(import ' in line]
+
+            # Ignore type names, which may vary (e.g. one file may have $5 and
+            # another may call the same type $17).
+            lines = [re.sub(r'[(]type [$][0-9a-zA-Z_$]+[)]', '', line) for line in lines]
+
+            return '\n'.join(lines)
+
+        compare(get_relevant_lines(original), get_relevant_lines(processed), 'Preserve')
+
+
 # The global list of all test case handlers
 testcase_handlers = [
     FuzzExec(),
@@ -1770,6 +1815,7 @@ def can_run_on_wasm(self, wasm):
     RoundtripText(),
     ClusterFuzz(),
     Two(),
+    PreserveImportsExports(),
 ]
 
 

src/tools/fuzzing.h

@@ -128,6 +128,9 @@ class TranslateToFuzzReader {
   void pickPasses(OptimizationOptions& options);
   void setAllowMemory(bool allowMemory_) { allowMemory = allowMemory_; }
   void setAllowOOB(bool allowOOB_) { allowOOB = allowOOB_; }
+  void setPreserveImportsAndExports(bool preserveImportsAndExports_) {
+    preserveImportsAndExports = preserveImportsAndExports_;
+  }
 
   void build();
 
@@ -146,6 +149,13 @@ class TranslateToFuzzReader {
   // of bounds (which traps in wasm, and is undefined behavior in C).
   bool allowOOB = true;
 
+  // Whether we preserve imports and exports. Normally we add imports (for
+  // logging and other useful functionality for testing), and add exports of
+  // functions as we create them. With this set, we add neither imports nor
+  // exports, which is useful if the tool using us only wants us to mutate an
+  // existing testcase (using initial-content).
+  bool preserveImportsAndExports = false;
+
   // Whether we allow the fuzzer to add unreachable code when generating changes
   // to existing code. This is randomized during startup, but could be an option
   // like the above options eventually if we find that useful.

src/tools/fuzzing/fuzzing.cpp

@@ -614,10 +614,12 @@ void TranslateToFuzzReader::setupGlobals() {
   // run the wasm.
   for (auto& global : wasm.globals) {
     if (global->imported()) {
-      // Remove import info from imported globals, and give them a simple
-      // initializer.
-      global->module = global->base = Name();
-      global->init = makeConst(global->type);
+      if (!preserveImportsAndExports) {
+        // Remove import info from imported globals, and give them a simple
+        // initializer.
+        global->module = global->base = Name();
+        global->init = makeConst(global->type);
+      }
     } else {
       // If the initialization referred to an imported global, it no longer can
       // point to the same global after we make it a non-imported global unless
@@ -695,7 +697,7 @@ void TranslateToFuzzReader::setupTags() {
   // As in modifyInitialFunctions(), we can't allow arbitrary tag imports, which
   // would trap when the fuzzing infrastructure doesn't know what to provide.
   for (auto& tag : wasm.tags) {
-    if (tag->imported()) {
+    if (tag->imported() && !preserveImportsAndExports) {
       tag->module = tag->base = Name();
     }
   }
@@ -707,7 +709,7 @@ void TranslateToFuzzReader::setupTags() {
   }
 
   // Add the fuzzing support tags manually sometimes.
-  if (oneIn(2)) {
+  if (!preserveImportsAndExports && oneIn(2)) {
     auto wasmTag = builder.makeTag(Names::getValidTagName(wasm, "wasmtag"),
                                    Signature(Type::i32, Type::none));
     wasmTag->module = "fuzzing-support";
@@ -779,9 +781,12 @@ void TranslateToFuzzReader::finalizeMemory() {
     memory->max =
       std::min(Address(memory->initial + 1), Address(Memory::kMaxSize32));
   }
-  // Avoid an imported memory (which the fuzz harness would need to handle).
-  for (auto& memory : wasm.memories) {
-    memory->module = memory->base = Name();
+
+  if (!preserveImportsAndExports) {
+    // Avoid an imported memory (which the fuzz harness would need to handle).
+    for (auto& memory : wasm.memories) {
+      memory->module = memory->base = Name();
+    }
   }
 }
 
@@ -826,8 +831,11 @@ void TranslateToFuzzReader::finalizeTable() {
     assert(ReasonableMaxTableSize <= Table::kMaxSize);
 
     table->max = oneIn(2) ? Address(Table::kUnlimitedSize) : table->initial;
-    // Avoid an imported table (which the fuzz harness would need to handle).
-    table->module = table->base = Name();
+
+    if (!preserveImportsAndExports) {
+      // Avoid an imported table (which the fuzz harness would need to handle).
+      table->module = table->base = Name();
+    }
   }
 }
 
@@ -841,8 +849,9 @@ void TranslateToFuzzReader::shuffleExports() {
   // we emit invokes for a function right after it (so we end up calling the
   // same code several times in succession, but interleaving it with others may
   // find more things). But we also keep a good chance for the natural order
-  // here, as it may help some initial content.
-  if (wasm.exports.empty() || oneIn(2)) {
+  // here, as it may help some initial content. Note we cannot do this if we are
+  // preserving the exports, as their order is something we must maintain.
+  if (wasm.exports.empty() || preserveImportsAndExports || oneIn(2)) {
     return;
   }
 
@@ -881,14 +890,24 @@ void TranslateToFuzzReader::addImportLoggingSupport() {
     Name baseName = std::string("log-") + type.toString();
     func->name = Names::getValidFunctionName(wasm, baseName);
     logImportNames[type] = func->name;
-    func->module = "fuzzing-support";
-    func->base = baseName;
+    if (!preserveImportsAndExports) {
+      func->module = "fuzzing-support";
+      func->base = baseName;
+    } else {
+      // We cannot add an import, so just make it a trivial function (this is
+      // simpler than avoiding calls to logging in all the rest of the logic).
+      func->body = builder.makeNop();
+    }
     func->type = Signature(type, Type::none);
     wasm.addFunction(std::move(func));
   }
 }
 
 void TranslateToFuzzReader::addImportCallingSupport() {
+  if (preserveImportsAndExports) {
+    return;
+  }
+
   if (wasm.features.hasReferenceTypes() && closedWorld) {
     // In closed world mode we must *remove* the call-ref* imports, if they
     // exist in the initial content. These are not valid to call in closed-world
@@ -983,8 +1002,13 @@ void TranslateToFuzzReader::addImportThrowingSupport() {
   throwImportName = Names::getValidFunctionName(wasm, "throw");
   auto func = std::make_unique<Function>();
   func->name = throwImportName;
-  func->module = "fuzzing-support";
-  func->base = "throw";
+  if (!preserveImportsAndExports) {
+    func->module = "fuzzing-support";
+    func->base = "throw";
+  } else {
+    // As with logging, implement in a trivial way when we cannot add imports.
+    func->body = builder.makeNop();
+  }
   func->type = Signature(Type::i32, Type::none);
   wasm.addFunction(std::move(func));
 }
@@ -999,8 +1023,9 @@ void TranslateToFuzzReader::addImportTableSupport() {
   }
 
   // If a "table" export already exists, skip fuzzing these imports, as the
-  // current export may not contain a valid table for it.
-  if (wasm.getExportOrNull("table")) {
+  // current export may not contain a valid table for it. We also skip if we are
+  // not adding imports or exports.
+  if (wasm.getExportOrNull("table") || preserveImportsAndExports) {
     return;
   }
 
@@ -1033,8 +1058,9 @@ void TranslateToFuzzReader::addImportTableSupport() {
 }
 
 void TranslateToFuzzReader::addImportSleepSupport() {
-  if (!oneIn(4)) {
-    // Fuzz this somewhat rarely, as it may be slow.
+  // Fuzz this somewhat rarely, as it may be slow, and only when we can add
+  // imports.
+  if (preserveImportsAndExports || !oneIn(4)) {
     return;
   }
 
@@ -1087,12 +1113,15 @@ void TranslateToFuzzReader::addHashMemorySupport() {
   auto* body = builder.makeBlock(contents);
   auto* hasher = wasm.addFunction(builder.makeFunction(
     "hashMemory", Signature(Type::none, Type::i32), {Type::i32}, body));
-  wasm.addExport(
-    builder.makeExport(hasher->name, hasher->name, ExternalKind::Function));
-  // Export memory so JS fuzzing can use it
-  if (!wasm.getExportOrNull("memory")) {
-    wasm.addExport(builder.makeExport(
-      "memory", wasm.memories[0]->name, ExternalKind::Memory));
+
+  if (!preserveImportsAndExports) {
+    wasm.addExport(
+      builder.makeExport(hasher->name, hasher->name, ExternalKind::Function));
+    // Export memory so JS fuzzing can use it
+    if (!wasm.getExportOrNull("memory")) {
+      wasm.addExport(builder.makeExport(
+        "memory", wasm.memories[0]->name, ExternalKind::Memory));
+    }
   }
 }
 
@@ -1340,7 +1369,7 @@ Function* TranslateToFuzzReader::addFunction() {
       return t.isDefaultable();
     });
   if (validExportParams && (numAddedFunctions == 0 || oneIn(2)) &&
-      !wasm.getExportOrNull(func->name)) {
+      !wasm.getExportOrNull(func->name) && !preserveImportsAndExports) {
     auto* export_ = new Export;
     export_->name = func->name;
     export_->value = func->name;
@@ -1805,8 +1834,10 @@ void TranslateToFuzzReader::modifyInitialFunctions() {
   for (Index i = 0; i < wasm.functions.size(); i++) {
     auto* func = wasm.functions[i].get();
     // We can't allow extra imports, as the fuzzing infrastructure wouldn't
-    // know what to provide. Keep only our own fuzzer imports.
-    if (func->imported() && func->module == "fuzzing-support") {
+    // know what to provide. Keep only our own fuzzer imports (or, if we are
+    // preserving imports, keep them all).
+    if (func->imported() &&
+        (func->module == "fuzzing-support" || preserveImportsAndExports)) {
       continue;
     }
     FunctionCreationContext context(*this, func);
@@ -1907,7 +1938,12 @@ void TranslateToFuzzReader::addInvocations(Function* func) {
   }
   body->list.set(invocations);
   wasm.addFunction(std::move(invoker));
-  wasm.addExport(builder.makeExport(name, name, ExternalKind::Function));
+
+  // Most of the benefit of invocations is lost when we do not add exports for
+  // them, but still, they might be called by existing functions.
+  if (!preserveImportsAndExports) {
+    wasm.addExport(builder.makeExport(name, name, ExternalKind::Function));
+  }
 }
 
 Expression* TranslateToFuzzReader::make(Type type) {

src/tools/wasm-opt.cpp

@@ -85,6 +85,7 @@ int main(int argc, const char* argv[]) {
   bool fuzzPasses = false;
   bool fuzzMemory = true;
   bool fuzzOOB = true;
+  bool fuzzPreserveImportsAndExports = false;
   std::string emitSpecWrapper;
   std::string emitWasm2CWrapper;
   std::string inputSourceMapFilename;
@@ -178,6 +179,14 @@ int main(int argc, const char* argv[]) {
          WasmOptOption,
          Options::Arguments::Zero,
          [&](Options* o, const std::string& arguments) { fuzzOOB = false; })
+    .add("--fuzz-preserve-imports-exports",
+         "",
+         "don't add imports and exports in -ttf mode",
+         WasmOptOption,
+         Options::Arguments::Zero,
+         [&](Options* o, const std::string& arguments) {
+           fuzzPreserveImportsAndExports = true;
+         })
     .add("--emit-spec-wrapper",
          "-esw",
          "Emit a wasm spec interpreter wrapper file that can run the wasm with "
@@ -310,6 +319,7 @@ int main(int argc, const char* argv[]) {
     }
     reader.setAllowMemory(fuzzMemory);
     reader.setAllowOOB(fuzzOOB);
+    reader.setPreserveImportsAndExports(fuzzPreserveImportsAndExports);
     reader.build();
     if (options.passOptions.validate) {
       if (!WasmValidator().validate(wasm, options.passOptions)) {

test/lit/fuzz-preserve-imports-exports.wast

@@ -0,0 +1,49 @@
+;; Test the flag to preserve imports and exports in fuzzer generation.
+
+;; Generate fuzz output using this wat as initial contents, and with the flag to
+;; preserve imports and exports. There should be no new imports or exports, and
+;; old ones must stay the same.
+
+;; RUN: wasm-opt %s.ttf --initial-fuzz=%s -all -ttf --fuzz-preserve-imports-exports \
+;; RUN:          --metrics -S -o - | filecheck %s --check-prefix=PRESERVE
+
+;; PRESERVE: [exports]      : 1
+;; PRESERVE: [imports]      : 5
+
+;; [sic] - we do not close ("))") some imports, which have info in the wat
+;; which we do not care about.
+;; PRESERVE:  (import "a" "d" (memory $imemory
+;; PRESERVE:  (import "a" "e" (table $itable
+;; PRESERVE:  (import "a" "b" (global $iglobal i32))
+;; PRESERVE:  (import "a" "f" (func $ifunc
+;; PRESERVE:  (import "a" "c" (tag $itag
+
+;; PRESERVE:  (export "foo" (func $foo))
+
+;; And, without the flag, we do generate both imports and exports.
+
+;; RUN: wasm-opt %s.ttf --initial-fuzz=%s -all -ttf \
+;; RUN:          --metrics -S -o - | filecheck %s --check-prefix=NORMAL
+
+;; Rather than hardcode the number here, find two of each.
+;; NORMAL: (import
+;; NORMAL: (import
+;; NORMAL: (export
+;; NORMAL: (export
+
+(module
+  ;; Existing imports. Note that the fuzzer normally turns imported globals etc.
+  ;; into normal ones (as the fuzz harness does not know what to provide at
+  ;; compile time), so we also test that --fuzz-preserve-imports-exports leaves
+  ;; such imports alone.
+  (import "a" "b" (global $iglobal i32))
+  (import "a" "c" (tag $itag))
+  (import "a" "d" (memory $imemory 10 20))
+  (import "a" "e" (table $itable 10 20 funcref))
+  (import "a" "f" (func $ifunc))
+
+  ;; One existing export.
+  (func $foo (export "foo")
+  )
+)
+

test/lit/fuzz-preserve-imports-exports.wast.ttf

@@ -53,6 +53,9 @@
 ;; CHECK-NEXT:                                                 loads/stores/indirect calls when
 ;; CHECK-NEXT:                                                 fuzzing
 ;; CHECK-NEXT:
+;; CHECK-NEXT:   --fuzz-preserve-imports-exports               don't add imports and exports in
+;; CHECK-NEXT:                                                 -ttf mode
+;; CHECK-NEXT:
 ;; CHECK-NEXT:   --emit-spec-wrapper,-esw                      Emit a wasm spec interpreter
 ;; CHECK-NEXT:                                                 wrapper file that can run the
 ;; CHECK-NEXT:                                                 wasm with some test values,