Fix NaN handling for some instructions (#8472)

* Fix bit patterns for canonical and arithmetic NaNs
* Canonicalize NaNs for float instructions covered by spec tests
* Also remove equivalent tests in test/spec since they're the same as
the upstream tests but with some previously-failing parts removed, and
using older syntax for checking for NaNs
* Fixes 12 spec tests

Part of #8261.

Author: stevenfontanella

scripts/test/shared.py

@@ -418,10 +418,6 @@ def get_tests(test_dir, extensions=[], recursive=False):
     'conversions.wast',  # Promoted NaN should be canonical
     'data.wast',    # Fail to parse data segment offset abbreviation
     'elem.wast',    # Requires modeling empty declarative segments
-    'f32.wast',     # Adding -0 and -nan should give a canonical NaN
-    'f64.wast',     # Adding -0 and -nan should give a canonical NaN
-    'float_exprs.wast',  # Adding 0 and NaN should give canonical NaN
-    'float_misc.wast',   # Rounding wrong on f64.sqrt
     'func.wast',    # Duplicate parameter names not properly rejected
     'global.wast',  # Fail to parse table
     'if.wast',      # Requires more precise unreachable validation
@@ -455,15 +451,7 @@ def get_tests(test_dir, extensions=[], recursive=False):
     'imports3.wast',  # Requires better checking of exports from the special "spectest" module
     'relaxed_dot_product.wast',   # i16x8.relaxed_dot_i8x16_i7x16_s instruction not supported
     'relaxed_laneselect.wast',    # i8x16.relaxed_laneselect instruction not supported
-    'relaxed_min_max.wast',       # Non-canonical NaN from f32x4.relaxed_min
     'simd_const.wast',            # Hex float constant not recognized as out of range
-    'simd_conversions.wast',      # Promoted NaN should be canonical
-    'simd_f32x4.wast',            # Min of 0 and NaN should give a canonical NaN
-    'simd_f32x4_arith.wast',      # Adding inf and -inf should give a canonical NaN
-    'simd_f32x4_rounding.wast',   # Ceil of NaN should give a canonical NaN
-    'simd_f64x2.wast',            # Min of 0 and NaN should give a canonical NaN
-    'simd_f64x2_arith.wast',      # Adding inf and -inf should give a canonical NaN
-    'simd_f64x2_rounding.wast',   # Ceil of NaN should give a canonical NaN
     'token.wast',                 # Lexer should require spaces between strings and non-paren tokens
 ]
 

src/wasm/literal.cpp

@@ -543,16 +543,16 @@ bool Literal::isCanonicalNaN() {
   if (!isNaN()) {
     return false;
   }
-  return (type == Type::f32 && NaNPayload(getf32()) == (1u << 23) - 1) ||
-         (type == Type::f64 && NaNPayload(getf64()) == (1ull << 52) - 1);
+  return (type == Type::f32 && NaNPayload(getf32()) == (1u << 22)) ||
+         (type == Type::f64 && NaNPayload(getf64()) == (1ull << 51));
 }
 
 bool Literal::isArithmeticNaN() {
   if (!isNaN()) {
     return false;
   }
-  return (type == Type::f32 && NaNPayload(getf32()) > (1u << 23) - 1) ||
-         (type == Type::f64 && NaNPayload(getf64()) > (1ull << 52) - 1);
+  return (type == Type::f32 && NaNPayload(getf32()) >= (1u << 22)) ||
+         (type == Type::f64 && NaNPayload(getf64()) >= (1ull << 51));
 }
 
 uint32_t Literal::NaNPayload(float f) {
@@ -1104,9 +1104,9 @@ Literal Literal::abs() const {
 Literal Literal::ceil() const {
   switch (type.getBasic()) {
     case Type::f32:
-      return Literal(std::ceil(getf32()));
+      return standardizeNaN(Literal(std::ceil(getf32())));
     case Type::f64:
-      return Literal(std::ceil(getf64()));
+      return standardizeNaN(Literal(std::ceil(getf64())));
     default:
       WASM_UNREACHABLE("unexpected type");
   }
@@ -1115,9 +1115,9 @@ Literal Literal::ceil() const {
 Literal Literal::floor() const {
   switch (type.getBasic()) {
     case Type::f32:
-      return Literal(std::floor(getf32()));
+      return standardizeNaN(Literal(std::floor(getf32())));
     case Type::f64:
-      return Literal(std::floor(getf64()));
+      return standardizeNaN(Literal(std::floor(getf64())));
     default:
       WASM_UNREACHABLE("unexpected type");
   }
@@ -1126,9 +1126,9 @@ Literal Literal::floor() const {
 Literal Literal::trunc() const {
   switch (type.getBasic()) {
     case Type::f32:
-      return Literal(std::trunc(getf32()));
+      return standardizeNaN(Literal(std::trunc(getf32())));
     case Type::f64:
-      return Literal(std::trunc(getf64()));
+      return standardizeNaN(Literal(std::trunc(getf64())));
     default:
       WASM_UNREACHABLE("unexpected type");
   }