Fix propagation bug in Unsubtyping (#8468)

Unsubtyping has to propagate a `subtypesExposedToJS` marker from
supertypes to subtypes. It previously did this by propagating from
supertype to subtype whenever a new subtyping relationship was found.
But subtyping trees are not always built from top to bottom. In cases
where the newly discovered subtype already had subtypes, the previous
propagation scheme would fail to propagate to those transitive subtypes.

Fix the problem by propagating to the full subtype tree rooted at a
newly discovered subtype. Use a DFS over the tree and avoid traversing
subtrees that have already been marked to avoid introducing possible
quadratic behavior.

Author: tlively

src/passes/Unsubtyping.cpp

@@ -421,8 +421,8 @@ struct TypeTree {
       for (auto child : node.children) {
         std::cerr << " " << ModuleHeapType(wasm, nodes[child].type);
       }
-      if (node.exposedToJS) {
-        std::cerr << ", exposed to JS";
+      if (node.subtypesExposedToJS) {
+        std::cerr << ", subtypes exposed to JS";
       }
       std::cerr << '\n';
     }
@@ -877,8 +877,22 @@ struct Unsubtyping : Pass, Noter<Unsubtyping> {
     types.setSupertype(sub, super);
 
     // If the supertype is exposed to JS, the subtype potentially is as well.
+    // `sub` may be the root of some existing subtype tree, and we have to
+    // propagate the exposure to JS to all those existing subtypes. We could
+    // just iterate over subtypes(), but manually traverse using
+    // immediateSubtypes() so we can avoid visiting subtrees that have already
+    // been marked.
     if (types.areSubtypesExposedToJS(super)) {
-      noteExposedToJS(sub);
+      std::vector<HeapType> work{{sub}};
+      while (!work.empty()) {
+        auto curr = work.back();
+        work.pop_back();
+        if (!types.areSubtypesExposedToJS(curr)) {
+          noteExposedToJS(curr);
+          auto subtypes = types.immediateSubtypes(curr);
+          work.insert(work.end(), subtypes.begin(), subtypes.end());
+        }
+      }
     }
 
     // Complete the descriptor squares to the left and right of the new

test/lit/passes/unsubtyping-jsinterop.wast

@@ -1027,3 +1027,53 @@
     (local.get $sub-out)
   )
 )
+
+(module
+  ;; Regression test for a bug where we were not fully propagating exposure to
+  ;; JS, resulting missing prototype-configuring descriptors depending on the
+  ;; order in which subtypes were processed. In this example, if $struct <:
+  ;; $super was processed before $super <: any, then exposure to JS would not be
+  ;; propagated down to $struct, so its descriptor would be removed.
+  (rec
+    ;; CHECK:      (rec
+    ;; CHECK-NEXT:  (type $super (sub (struct)))
+    (type $super (sub (struct)))
+    ;; CHECK:       (type $struct (sub $super (descriptor $desc) (struct)))
+    (type $struct (sub $super (descriptor $desc) (struct)))
+    ;; CHECK:       (type $desc (describes $struct) (struct (field externref)))
+    (type $desc (describes $struct) (struct (field externref)))
+  )
+
+  ;; CHECK:       (type $3 (func (result anyref)))
+
+  ;; CHECK:      (global $any (mut anyref) (ref.null none))
+  (global $any (mut anyref) (ref.null none))
+  ;; CHECK:      (global $super (mut (ref null $super)) (ref.null none))
+  (global $super (mut (ref null $super)) (ref.null none))
+  ;; CHECK:      (global $struct (ref null $struct) (ref.null none))
+  (global $struct (ref null $struct) (ref.null none))
+
+  ;; any exposed to JS.
+  ;; CHECK:      (@binaryen.js.called)
+  ;; CHECK-NEXT: (func $expose-anyref (type $3) (result anyref)
+  ;; CHECK-NEXT:  (global.set $any
+  ;; CHECK-NEXT:   (global.get $super)
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT:  (global.set $super
+  ;; CHECK-NEXT:   (global.get $struct)
+  ;; CHECK-NEXT:  )
+  ;; CHECK-NEXT:  (unreachable)
+  ;; CHECK-NEXT: )
+  (@binaryen.js.called)
+  (func $expose-anyref (result anyref)
+    ;; $super <: aby
+    (global.set $any
+      (global.get $super)
+    )
+    ;; $struct <: $super
+    (global.set $super
+      (global.get $struct)
+    )
+    (unreachable)
+  )
+)