ClusterFuzz: Add a --no_retry option for local debugging (#7187)

When we run ClusterFuzz's run.py locally, we need to investigate any
errors in creating fuzz testcases (as those are real bugs in wasm-opt).
Previously we just checked for a "retry" message in the output of run.py
(as the script retries automatically - we don't want a wasm-opt bug to
stop ClusterFuzz), but that makes it impossible to debug the problem,
since the retry tramples the data. The new option, used only locally,
stops at the error so it can be investigated.

Author: kripken

scripts/clusterfuzz/run.py

@@ -92,6 +92,12 @@ def get_file_name(prefix, index):
 # (We also use urandom below, which uses this under the hood.)
 system_random = random.SystemRandom()
 
+# In production ClusterFuzz we retry whenever we see a wasm-opt error. We are
+# not looking for wasm-opt issues there, and just use it to generate testcases
+# for VMs. For local testing, however, we may want to disable retrying, which
+# allows us to debug any such failures that we run into.
+retry = True
+
 
 # Generate a random wasm file, and return a string that creates a typed array of
 # those bytes, suitable for use in a JS file, in the form
@@ -117,6 +123,11 @@ def get_wasm_contents(i, output_dir):
         try:
             subprocess.check_call(cmd)
         except subprocess.CalledProcessError:
+            if not retry:
+                print('error in running wasm-opt')
+                print(' '.join(cmd))
+                raise
+
             # Try again.
             print('(oops, retrying wasm-opt)')
             attempt += 1
@@ -217,13 +228,16 @@ def main(argv):
     # https://google.github.io/clusterfuzz/setting-up-fuzzing/blackbox-fuzzing/#uploading-a-fuzzer
     output_dir = '.'
     num = 100
-    expected_flags = ['input_dir=', 'output_dir=', 'no_of_files=']
+    expected_flags = ['input_dir=', 'output_dir=', 'no_of_files=', 'no_retry']
     optlist, _ = getopt.getopt(argv[1:], '', expected_flags)
     for option, value in optlist:
         if option == '--output_dir':
             output_dir = value
         elif option == '--no_of_files':
             num = int(value)
+        elif option == '--no_retry':
+            global retry
+            retry = False
 
     for i in range(1, num + 1):
         testcase_file_path = os.path.join(output_dir,

scripts/fuzz_opt.py

@@ -1580,15 +1580,13 @@ def handle(self, wasm):
                 os.unlink(f)
 
         # Call run.py(), similarly to how ClusterFuzz does.
-        out = run([sys.executable,
-                   os.path.join(self.clusterfuzz_dir, 'run.py'),
-                   '--output_dir=' + os.getcwd(),
-                   '--no_of_files=1'])
-
-        # We should not see any mention of a wasm-opt error that caused a
-        # retry. On production ClusterFuzz this is not an error, but we do want
-        # to know about such issues, as they may be real bugs in wasm-opt.
-        assert 'retry' not in out, out
+        run([sys.executable,
+             os.path.join(self.clusterfuzz_dir, 'run.py'),
+             '--output_dir=' + os.getcwd(),
+             '--no_of_files=1',
+             # Do not retry on wasm-opt errors: we want to investigate
+             # them.
+             '--no_retry'])
 
         # We should see the two files.
         assert os.path.exists(fuzz_file)