Binary format code section offset tracking (#2515)

Optionally track the binary format code section offsets,
that is, when loading a binary, remember where each IR
node was read from. This is necessary for DWARF
debug info, as these are the offsets DWARF refers to.

(Note that eventually we may want to do something
else, like first read the DWARF and only then add
debug info annotations into the IR in a more LLVM-like
manner, but this is more straightforward and should be
enough to update debug lines and ranges).

This tracking adds noticeable overhead - every single
IR node adds an entry in a map - so avoid it unless
actually necessary. Specifically, if the user passes in
-g and there are actually DWARF sections in the
binary, and we are not about to remove those sections,
then we need it.

Print binary format code section offsets in text, when
printing with -g. This will help debug and test dwarf
support. It looks like

;; code offset: 0x7

as an annotation right before each node.

Also add support for -g in wasm-opt tests (unlike
a pass, it has just one - as a prefix).

Helps #2400

Author: kripken

auto_update_tests.py

@@ -103,7 +103,7 @@ def update_wasm_opt_tests():
         passes_file = os.path.join(shared.get_test_dir('passes'), passname + '.passes')
         if os.path.exists(passes_file):
             passname = open(passes_file).read().strip()
-        opts = [('--' + p if not p.startswith('O') else '-' + p) for p in passname.split('_')]
+        opts = [('--' + p if not p.startswith('O') and p != 'g' else '-' + p) for p in passname.split('_')]
         actual = ''
         for module, asserts in support.split_wast(t):
             assert len(asserts) == 0

check.py

@@ -99,7 +99,7 @@ def run_wasm_opt_tests():
         passes_file = os.path.join(shared.get_test_dir('passes'), passname + '.passes')
         if os.path.exists(passes_file):
             passname = open(passes_file).read().strip()
-        opts = [('--' + p if not p.startswith('O') else '-' + p) for p in passname.split('_')]
+        opts = [('--' + p if not p.startswith('O') and p != 'g' else '-' + p) for p in passname.split('_')]
         actual = ''
         for module, asserts in support.split_wast(t):
             assert len(asserts) == 0

src/passes/Print.cpp

@@ -1392,6 +1392,7 @@ struct PrintSExpression : public OverriddenVisitor<PrintSExpression> {
   Module* currModule = nullptr;
   Function* currFunction = nullptr;
   Function::DebugLocation lastPrintedLocation;
+  bool debugInfo;
 
   std::unordered_map<Name, Index> functionIndexes;
 
@@ -1421,6 +1422,16 @@ struct PrintSExpression : public OverriddenVisitor<PrintSExpression> {
       if (iter != debugLocations.end()) {
         printDebugLocation(iter->second);
       }
+      // show a binary position, if there is one
+      if (debugInfo) {
+        auto iter = currFunction->binaryLocations.find(curr);
+        if (iter != currFunction->binaryLocations.end()) {
+          Colors::grey(o);
+          o << ";; code offset: 0x" << iter->second << '\n';
+          restoreNormalColor(o);
+          doIndent(o, indent);
+        }
+      }
     }
   }
 
@@ -1437,6 +1448,10 @@ struct PrintSExpression : public OverriddenVisitor<PrintSExpression> {
 
   void setFull(bool full_) { full = full_; }
 
+  void setPrintStackIR(bool printStackIR_) { printStackIR = printStackIR_; }
+
+  void setDebugInfo(bool debugInfo_) { debugInfo = debugInfo_; }
+
   void incIndent() {
     if (minify) {
       return;
@@ -2321,6 +2336,7 @@ class Printer : public Pass {
 
   void run(PassRunner* runner, Module* module) override {
     PrintSExpression print(o);
+    print.setDebugInfo(runner->options.debugInfo);
     print.visitModule(module);
   }
 };
@@ -2337,6 +2353,7 @@ class MinifiedPrinter : public Printer {
   void run(PassRunner* runner, Module* module) override {
     PrintSExpression print(o);
     print.setMinify(true);
+    print.setDebugInfo(runner->options.debugInfo);
     print.visitModule(module);
   }
 };
@@ -2353,6 +2370,7 @@ class FullPrinter : public Printer {
   void run(PassRunner* runner, Module* module) override {
     PrintSExpression print(o);
     print.setFull(true);
+    print.setDebugInfo(runner->options.debugInfo);
     print.visitModule(module);
   }
 };
@@ -2368,7 +2386,8 @@ class PrintStackIR : public Printer {
 
   void run(PassRunner* runner, Module* module) override {
     PrintSExpression print(o);
-    print.printStackIR = true;
+    print.setDebugInfo(runner->options.debugInfo);
+    print.setPrintStackIR(true);
     print.visitModule(module);
   }
 };

src/tools/wasm-emscripten-finalize.cpp

@@ -49,6 +49,7 @@ int main(int argc, const char* argv[]) {
   std::string dataSegmentFile;
   bool emitBinary = true;
   bool debugInfo = false;
+  bool DWARF = false;
   bool isSideModule = false;
   bool legalizeJavaScriptFFI = true;
   bool checkStackOverflow = false;
@@ -71,6 +72,11 @@ int main(int argc, const char* argv[]) {
          "Emit names section in wasm binary (or full debuginfo in wast)",
          Options::Arguments::Zero,
          [&debugInfo](Options*, const std::string&) { debugInfo = true; })
+    .add("--dwarf",
+         "",
+         "Update DWARF debug info",
+         Options::Arguments::Zero,
+         [&DWARF](Options*, const std::string&) { DWARF = true; })
     .add("--emit-text",
          "-S",
          "Emit text instead of binary for the output file",
@@ -164,6 +170,7 @@ int main(int argc, const char* argv[]) {
 
   Module wasm;
   ModuleReader reader;
+  reader.setDWARF(DWARF);
   try {
     reader.read(infile, wasm, inputSourceMapFilename);
   } catch (ParseException& p) {

src/tools/wasm-metadce.cpp

@@ -498,6 +498,7 @@ int main(int argc, const char* argv[]) {
       std::cerr << "reading...\n";
     }
     ModuleReader reader;
+    reader.setDWARF(debugInfo);
     try {
       reader.read(options.extra["infile"], wasm);
     } catch (ParseException& p) {

src/tools/wasm-opt.cpp

@@ -43,7 +43,7 @@
 using namespace wasm;
 
 // runs a command and returns its output TODO: portability, return code checking
-std::string runCommand(std::string command) {
+static std::string runCommand(std::string command) {
 #ifdef __linux__
   std::string output;
   const int MAX_BUFFER = 1024;
@@ -59,6 +59,15 @@ std::string runCommand(std::string command) {
 #endif
 }
 
+static bool willRemoveDebugInfo(const std::vector<std::string>& passes) {
+  for (auto& pass : passes) {
+    if (pass == "strip" || pass == "strip-debug" || pass == "strip-dwarf") {
+      return true;
+    }
+  }
+  return false;
+}
+
 //
 // main
 //
@@ -210,6 +219,10 @@ int main(int argc, const char* argv[]) {
 
   if (!translateToFuzz) {
     ModuleReader reader;
+    // Enable DWARF parsing if we were asked for debug info, and were not
+    // asked to remove it.
+    reader.setDWARF(options.passOptions.debugInfo &&
+                    !willRemoveDebugInfo(options.passes));
     try {
       reader.read(options.extra["infile"], wasm, inputSourceMapFilename);
     } catch (ParseException& p) {

src/wasm-binary.h

@@ -1053,10 +1053,12 @@ class WasmBinaryBuilder {
   const std::vector<char>& input;
   std::istream* sourceMap;
   std::pair<uint32_t, Function::DebugLocation> nextDebugLocation;
+  bool DWARF = false;
 
   size_t pos = 0;
   Index startIndex = -1;
   std::set<Function::DebugLocation> debugLocation;
+  size_t codeSectionLocation;
 
   std::set<BinaryConsts::Section> seenSections;
 
@@ -1068,6 +1070,7 @@ class WasmBinaryBuilder {
     : wasm(wasm), allocator(wasm.allocator), input(input), sourceMap(nullptr),
       nextDebugLocation(0, {0, 0, 0}), debugLocation() {}
 
+  void setDWARF(bool value) { DWARF = value; }
   void read();
   void readUserSection(size_t payloadLen);
 
@@ -1275,6 +1278,9 @@ class WasmBinaryBuilder {
   void visitBrOnExn(BrOnExn* curr);
 
   void throwError(std::string text);
+
+private:
+  bool hasDWARFSections();
 };
 
 } // namespace wasm

src/wasm-io.h

@@ -29,6 +29,10 @@ namespace wasm {
 
 class ModuleReader {
 public:
+  // If DWARF support is enabled, we track the locations of all IR nodes in
+  // the binary, so that we can update DWARF sections later when writing.
+  void setDWARF(bool DWARF_) { DWARF = DWARF_; }
+
   // read text
   void readText(std::string filename, Module& wasm);
   // read binary
@@ -43,7 +47,13 @@ class ModuleReader {
   bool isBinaryFile(std::string filename);
 
 private:
+  bool DWARF = false;
+
   void readStdin(Module& wasm, std::string sourceMapFilename);
+
+  void readBinaryData(std::vector<char>& input,
+                      Module& wasm,
+                      std::string sourceMapFilename);
 };
 
 class ModuleWriter {

src/wasm.h

@@ -1154,6 +1154,7 @@ class Function : public Importable {
   std::map<Index, Name> localNames;
   std::map<Name, Index> localIndices;
 
+  // Source maps debugging info: map expression nodes to their file, line, col.
   struct DebugLocation {
     uint32_t fileIndex, lineNumber, columnNumber;
     bool operator==(const DebugLocation& other) const {
@@ -1175,6 +1176,10 @@ class Function : public Importable {
   std::set<DebugLocation> prologLocation;
   std::set<DebugLocation> epilogLocation;
 
+  // General debugging info: map every instruction to its original position in
+  // the binary, relative to the beginning of the code section.
+  std::unordered_map<Expression*, uint32_t> binaryLocations;
+
   size_t getNumParams();
   size_t getNumVars();
   size_t getNumLocals();
@@ -1344,6 +1349,8 @@ class Module {
   Name start;
 
   std::vector<UserSection> userSections;
+
+  // Source maps debug info.
   std::vector<std::string> debugInfoFileNames;
 
   // `features` are the features allowed to be used in this module and should be

src/wasm/CMakeLists.txt

@@ -2,6 +2,7 @@ set(wasm_SOURCES
   literal.cpp
   wasm.cpp
   wasm-binary.cpp
+  wasm-debug.cpp
   wasm-emscripten.cpp
   wasm-debug.cpp
   wasm-interpreter.cpp