From 6a5978923032432c07ba9c8aa38a9b82041b0b44 Mon Sep 17 00:00:00 2001 From: Ralph Date: Sun, 5 Jul 2026 13:06:35 -0700 Subject: [PATCH] =?UTF-8?q?perf(codegen,runtime):=20#6011=20=E2=80=94=20pa?= =?UTF-8?q?cked-f64=20range-loop=20versioning:=20scalar-bound=20multi-arra?= =?UTF-8?q?y=20loops=20run=20inline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tight float loop (`for (i = 1; i < N; i++) ema[i] = prices[i]*a + ema[i-1]*b`) ran ~9x slower than Node: the existing packed-f64 loop versioning only matched `i < arr.length` bounds on a single array with exact-`i` indices, so every access went through generic runtime calls (reads probing four registries per element, stores paying the write barrier's thread-local preamble per element). Codegen: - new range-loop matcher + lowering (stmt/loops.rs): scalar loop-invariant bounds (`i < N`), multiple arrays per loop, affine `i±c` indices; one range guard per array at loop entry, hole-checked inline loads with a side exit to the slow loop - array-kind facts: seed params and `new Array(n)` locals (Generic{"Array"} type spelling + New{Array} initializers); facts stay versioning hints — every fast loop is runtime-guard validated - `arr[i±c]` offset support in the fast-loop index get/set lowerings Runtime: - js_typed_feedback_packed_f64_range_loop_guard: kind/shape/frozen checks + static index-range bounds proof + hole-tolerant slot canonicalization (rebuild_array_numeric_raw_f64_allow_holes) - GC_ARRAY_RAW_F64_HOLES header bit: `new Array(n)` is raw-f64-or-holes by construction; the invariant is maintained by the store choke points and makes the guard's verify walk O(1) (also O(1)-fails the per-store dense-layout probe while a fill is in flight — the walk there was O(N²) across a sequential fill) - write barrier: primitive-child early-exit before the incremental-mark TLS probe; decode_heap_addr fast-rejects f64 payload bit patterns before the page-map lookup EMA benchmark (100k elems × 100 iters): 55.6 → 5.9-6.1 ns/elem — at/below node v26 (6.0-6.1). Verification: 22-case loop matrix byte-identical to node (holes, guard-failure fallbacks, param arrays, offset reads); packed_f64_loop_versioning fixtures green; parity --filter array unchanged (49 pass, 1 pre-existing failure); cargo test -p perry-runtime 1155/1155 serial; the one perry-codegen failure (pod_field_read_after_dynamic_materialization_uses_number_coerce) is pre-existing on main (#6015 revert left tests/native_proof_regressions.rs uncompilable; this branch restores its compilation). Known follow-up: an array that later appears as an array-method receiver (`arr.filter(...)`) is currently rejected by the range matcher, so fill-then-chain code takes the per-access guarded path (~1.8x slower than the legacy call on that phase). Tracked for a follow-up relaxation. Closes #6011 Co-Authored-By: Claude Fable 5 --- crates/perry-codegen/src/codegen/closure.rs | 1 + crates/perry-codegen/src/codegen/entry.rs | 2 + crates/perry-codegen/src/codegen/function.rs | 1 + crates/perry-codegen/src/codegen/method.rs | 2 + .../perry-codegen/src/collectors/hir_facts.rs | 98 ++- crates/perry-codegen/src/expr/helpers.rs | 8 + crates/perry-codegen/src/expr/index_get.rs | 147 +++- crates/perry-codegen/src/expr/index_set.rs | 215 ++++- crates/perry-codegen/src/expr/mod.rs | 9 + .../src/runtime_decls/objects.rs | 7 + crates/perry-codegen/src/stmt/loops.rs | 830 ++++++++++++++++-- .../src/type_analysis/numeric.rs | 8 + crates/perry-codegen/src/type_analysis/pod.rs | 7 + .../tests/native_proof_regressions.rs | 6 + crates/perry-runtime/src/array/alloc.rs | 12 +- crates/perry-runtime/src/array/header.rs | 158 +++- crates/perry-runtime/src/array/mod.rs | 9 +- crates/perry-runtime/src/array/tests.rs | 78 ++ crates/perry-runtime/src/gc/barrier.rs | 27 +- crates/perry-runtime/src/gc/types.rs | 10 + crates/perry-runtime/src/typed_feedback.rs | 96 ++ .../perry-runtime/src/typed_feedback/tests.rs | 8 + 22 files changed, 1588 insertions(+), 151 deletions(-) diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index e85bc21a6..1905d1af0 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -705,6 +705,7 @@ pub(super) fn compile_closure( cross_module.flat_const_arrays.keys().copied().collect(); let native_facts = crate::collectors::collect_native_region_fact_graph( body, + &[], &flat_const_ids, &clamp_fn_ids, &cross_module.clamp3_functions, diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index ce6aebb35..e22b2d63f 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -544,6 +544,7 @@ pub(super) fn compile_module_entry( .collect(); let main_native_facts = crate::collectors::collect_native_region_fact_graph( &hir.init, + &[], &flat_const_ids, &clamp_fn_ids, &cross_module.clamp3_functions, @@ -1078,6 +1079,7 @@ pub(super) fn compile_module_entry( .collect(); let init_native_facts = crate::collectors::collect_native_region_fact_graph( &hir.init, + &[], &flat_const_ids, &clamp_fn_ids, &cross_module.clamp3_functions, diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index 9ab0c840c..94987492a 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -448,6 +448,7 @@ pub(super) fn compile_function( cross_module.flat_const_arrays.keys().copied().collect(); let native_facts = crate::collectors::collect_native_region_fact_graph( &f.body, + &f.params, &flat_const_ids, &clamp_fn_ids, &cross_module.clamp3_functions, diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index d1406c286..fa2f67b93 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -329,6 +329,7 @@ pub(super) fn compile_method( cross_module.flat_const_arrays.keys().copied().collect(); let native_facts = crate::collectors::collect_native_region_fact_graph( &method.body, + &[], &flat_const_ids, &clamp_fn_ids, &cross_module.clamp3_functions, @@ -1198,6 +1199,7 @@ pub(super) fn compile_static_method( cross_module.flat_const_arrays.keys().copied().collect(); let native_facts = crate::collectors::collect_native_region_fact_graph( &f.body, + &[], &flat_const_ids, &clamp_fn_ids, &cross_module.clamp3_functions, diff --git a/crates/perry-codegen/src/collectors/hir_facts.rs b/crates/perry-codegen/src/collectors/hir_facts.rs index 345873f54..3c16ecffb 100644 --- a/crates/perry-codegen/src/collectors/hir_facts.rs +++ b/crates/perry-codegen/src/collectors/hir_facts.rs @@ -138,6 +138,18 @@ impl TypeFacts { && !self.has_materialization_hazard(local_id) } + /// #6011: packed-f64 eligibility for arrays *written* by the range-guarded + /// versioned loop. Unlike [`Self::proves_packed_f64_array`] this does NOT + /// require length stability — an `arr[i] = …` in the loop body marks the + /// local as length-mutating even though the range guard proves every such + /// store is in-bounds (and therefore length-preserving) at loop entry. + /// Kind + noalias + no-hazard is what the guarded store path needs. + pub(crate) fn packed_f64_eligible_for_guarded_store(&self, local_id: u32) -> bool { + self.array_kind(local_id) == ArrayKindFact::PackedF64 + && self.proves_noalias_array(local_id) + && !self.has_materialization_hazard(local_id) + } + pub(crate) fn proves_packed_i32_array(&self, local_id: u32) -> bool { self.array_kind(local_id) == ArrayKindFact::PackedI32 && self.proves_noalias_array(local_id) @@ -276,6 +288,7 @@ impl TypeFacts { #[allow(clippy::too_many_arguments)] pub(crate) fn collect_type_facts( stmts: &[Stmt], + params: &[perry_hir::Param], flat_const_ids: &HashSet, clamp_fn_ids: &HashSet, arg_dependent_clamp_fn_ids: &HashSet, @@ -291,7 +304,7 @@ pub(crate) fn collect_type_facts( arg_dependent_clamp_fn_ids, ); let unsigned_i32_locals = super::i32_locals::collect_unsigned_i32_locals(stmts); - let (array_facts, effect_facts, materialization_hazards) = collect_array_facts(stmts); + let (array_facts, effect_facts, materialization_hazards) = collect_array_facts(stmts, params); let index_used_locals = super::index_uses::collect_index_used_locals(stmts); let strictly_i32_bounded_locals = super::i32_locals::collect_strictly_i32_bounded_locals( stmts, @@ -371,6 +384,7 @@ pub(crate) fn collect_type_facts( #[allow(clippy::too_many_arguments)] pub(crate) fn collect_native_region_fact_graph( stmts: &[Stmt], + params: &[perry_hir::Param], flat_const_ids: &HashSet, clamp_fn_ids: &HashSet, arg_dependent_clamp_fn_ids: &HashSet, @@ -381,6 +395,7 @@ pub(crate) fn collect_native_region_fact_graph( ) -> NativeRegionFactGraph { collect_type_facts( stmts, + params, flat_const_ids, clamp_fn_ids, arg_dependent_clamp_fn_ids, @@ -401,6 +416,7 @@ pub(crate) fn collect_hir_facts( ) -> TypeFacts { collect_type_facts( stmts, + &[], flat_const_ids, clamp_fn_ids, &HashSet::new(), @@ -548,8 +564,12 @@ fn is_fresh_uint8array_length_literal(expr: &Expr) -> bool { } } -fn collect_array_facts(stmts: &[Stmt]) -> (ArrayFacts, EffectFacts, MaterializationHazardFacts) { +fn collect_array_facts( + stmts: &[Stmt], + params: &[perry_hir::Param], +) -> (ArrayFacts, EffectFacts, MaterializationHazardFacts) { let mut collector = ArrayFactCollector::default(); + collector.seed_params(params); collector.collect_stmts(stmts); collector.finish() } @@ -561,11 +581,39 @@ struct ArrayFactCollector { aliased_locals: HashSet, length_mutation_locals: HashSet, materialization_hazard_locals: HashSet, + /// #6011: param ids seeded purely from a declared `Packed*` array type. + /// A param can receive ANY array at runtime, so these seeds are only + /// versioning hints for the runtime-guard-validated packed loop matcher; + /// body-observed mutations still downgrade them like any tracked local. + param_seeded_locals: HashSet, unknown_call_escape: bool, async_microtask_escape: bool, } impl ArrayFactCollector { + /// #6011: seed `local_kinds` for function params whose *declared* type is + /// a packed numeric array (e.g. `prices: number[]`). Only the numeric + /// `Packed*` kinds are seeded — a `PackedValue`/`HoleyValue` param fact + /// enables no consumer and would only widen the conservative-downgrade + /// surface. Seeding happens before the body walk so every mutation the + /// walk observes (push, aliasing, length writes, …) downgrades the param + /// exactly like a `Stmt::Let`-declared array. + fn seed_params(&mut self, params: &[perry_hir::Param]) { + for param in params { + if param.is_rest { + continue; + } + let kind = array_kind_from_declared_type(¶m.ty); + if matches!( + kind, + ArrayKindFact::PackedI32 | ArrayKindFact::PackedU32 | ArrayKindFact::PackedF64 + ) { + self.local_kinds.insert(param.id, kind); + self.param_seeded_locals.insert(param.id); + } + } + } + fn collect_stmts(&mut self, stmts: &[Stmt]) { for stmt in stmts { self.collect_stmt(stmt); @@ -1095,7 +1143,17 @@ impl ArrayFactCollector { self.unknown_call_escape = true; let ids: Vec = self.local_kinds.keys().copied().collect(); for id in ids { - self.mark_array_materialization_hazard(id); + // #6011: param-seeded facts still lose their packed kind on an + // unknown call (conservative for every fact consumer), but do NOT + // gain a materialization hazard — params were never hazard-tracked + // before seeding existed, and hazards feed non-fact consumers + // (`array_length_receiver_is_loop_local`'s length-hoist gate) that + // must not regress for `i < param.length` loops in call-bearing + // bodies. Explicit hazards (freeze/defineProperty/identity escape + // on the param itself) still mark normally. + if !self.param_seeded_locals.contains(&id) { + self.mark_array_materialization_hazard(id); + } self.update_array_kind_for_local(id, ArrayKindFact::Unknown); } } @@ -1129,12 +1187,44 @@ fn array_kind_from_declared_type(ty: &perry_types::Type) -> ArrayKindFact { ArrayKindFact::PackedF64 } perry_types::Type::Array(_) => ArrayKindFact::PackedValue, + // #6011: `new Array(n)` declares/infers as + // `Generic { base: "Array", type_args: [Number] }` rather than + // `Array(Number)` — classify the generic spelling identically. + perry_types::Type::Generic { base, type_args } + if base == "Array" && type_args.len() == 1 => + { + match &type_args[0] { + perry_types::Type::Int32 => ArrayKindFact::PackedI32, + perry_types::Type::Named(name) if name == "PerryU32" => ArrayKindFact::PackedU32, + perry_types::Type::Number => ArrayKindFact::PackedF64, + _ => ArrayKindFact::PackedValue, + } + } + perry_types::Type::Generic { base, .. } if base == "Array" => ArrayKindFact::PackedValue, _ => ArrayKindFact::Unknown, } } fn array_kind_from_initializer(expr: &Expr) -> ArrayKindFact { match expr { + // #6011: `new Array()` / `new Array(n)` allocations. Zero args is an + // empty array; one arg is (almost always) a length — all slots start + // as TAG_HOLE, which the hole-tolerant packed-f64 range guard accepts. + // A single NON-numeric arg (`new Array("x")`) actually stores the arg + // at element 0, making the array non-numeric — that is still fine to + // seed as PackedF64 because this fact is only a versioning hint: the + // runtime guard revalidates every slot at loop entry and falls back + // to the slow loop. Multiple args become elements, so they must all + // be literal numbers for the packed-f64 seed. + Expr::New { + class_name, args, .. + } if class_name == "Array" => { + if args.len() <= 1 || args.iter().all(expr_is_literal_number) { + ArrayKindFact::PackedF64 + } else { + ArrayKindFact::PackedValue + } + } Expr::Array(elements) if elements.iter().all(expr_is_literal_i32) => { ArrayKindFact::PackedI32 } @@ -1530,6 +1620,7 @@ mod tests { 1, Expr::Uint8ArrayNew(Some(Box::new(Expr::Integer(8)))), )], + &[], &HashSet::new(), &pure_helpers, &HashSet::new(), @@ -1618,6 +1709,7 @@ mod tests { let graph = collect_native_region_fact_graph( &stmts, + &[], &HashSet::new(), &HashSet::new(), &HashSet::new(), diff --git a/crates/perry-codegen/src/expr/helpers.rs b/crates/perry-codegen/src/expr/helpers.rs index 705fd46ec..ecb8a3757 100644 --- a/crates/perry-codegen/src/expr/helpers.rs +++ b/crates/perry-codegen/src/expr/helpers.rs @@ -18,6 +18,14 @@ use crate::types::{DOUBLE, I32, I64}; pub(crate) fn type_has_numeric_pointer_free_array_layout(ty: &HirType) -> bool { match ty { HirType::Array(elem) => matches!(elem.as_ref(), HirType::Number | HirType::Int32), + // #6011: `new Array(n)` declarations carry the generic + // spelling; treat it exactly like `Array(Number)` so number-context + // element reads keep their coerced fallback (a raw arithmetic op on + // an uncoerced boxed `undefined` would propagate the NaN-box payload + // verbatim) and stores take the guarded numeric-layout path. + HirType::Generic { base, type_args } if base == "Array" && type_args.len() == 1 => { + matches!(type_args[0], HirType::Number | HirType::Int32) + } HirType::Tuple(elems) => elems .iter() .all(|elem| matches!(elem, HirType::Number | HirType::Int32)), diff --git a/crates/perry-codegen/src/expr/index_get.rs b/crates/perry-codegen/src/expr/index_get.rs index 13969ae36..eb70fda12 100644 --- a/crates/perry-codegen/src/expr/index_get.rs +++ b/crates/perry-codegen/src/expr/index_get.rs @@ -115,16 +115,84 @@ fn bitand_has_nonnegative_i32_mask(left: &Expr, right: &Expr) -> bool { .is_some_and(|mask| (0..=i32::MAX as i64).contains(&mask)) } +/// #6011: decompose a packed-loop index expression into `(counter_local_id, +/// constant_offset)`. Matches `i`, `i + c`, `c + i`, and `i - c` with a small +/// |c| — exactly the shapes the packed-f64 range loop matcher admits, so any +/// offset seen here on a fact-carrying (array, counter) pair is inside the +/// range guard's validated window. +pub(crate) fn packed_f64_loop_index_parts(index: &Expr) -> Option<(u32, i32)> { + use perry_hir::BinaryOp; + match index { + Expr::LocalGet(id) => Some((*id, 0)), + Expr::Binary { op, left, right } if matches!(op, BinaryOp::Add | BinaryOp::Sub) => { + let (id, offset) = match (left.as_ref(), right.as_ref()) { + (Expr::LocalGet(id), Expr::Integer(c)) => { + let offset = if matches!(op, BinaryOp::Sub) { + c.checked_neg()? + } else { + *c + }; + (*id, offset) + } + (Expr::Integer(c), Expr::LocalGet(id)) if matches!(op, BinaryOp::Add) => (*id, *c), + _ => return None, + }; + let offset = i32::try_from(offset).ok()?; + if offset.unsigned_abs() > 64 { + return None; + } + Some((id, offset)) + } + _ => None, + } +} + +/// Look up a packed-f64 loop fact for `(arr, index-expr)`. Zero-offset +/// indices match any fact; non-zero offsets only match hole-tolerant facts +/// (established by the range guard, which validated the whole offset window — +/// the length-bound guard of the classic matcher only proves `i` itself). +fn packed_f64_loop_fact_for_index( + ctx: &FnCtx<'_>, + arr_id: u32, + index: &Expr, +) -> Option<(PackedF64LoopFact, u32, i32)> { + let (idx_id, offset) = packed_f64_loop_index_parts(index)?; + let fact = packed_f64_loop_fact(ctx, arr_id, idx_id)?; + if offset != 0 && !fact.allow_holes { + return None; + } + Some((fact, idx_id, offset)) +} + +/// Load the packed-loop counter's i32 shadow slot and apply the constant +/// index offset. +fn load_packed_loop_index_i32(ctx: &mut FnCtx<'_>, i32_slot: &str, offset: i32) -> String { + let idx_i32 = ctx.block().load(I32, i32_slot); + match offset.cmp(&0) { + std::cmp::Ordering::Equal => idx_i32, + std::cmp::Ordering::Greater => ctx.block().add(I32, &idx_i32, &offset.to_string()), + std::cmp::Ordering::Less => ctx.block().sub(I32, &idx_i32, &(-offset).to_string()), + } +} + fn numeric_index_has_loop_array_index_proof(ctx: &FnCtx<'_>, object: &Expr, index: &Expr) -> bool { - let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = (object, index) else { + let Expr::LocalGet(arr_id) = object else { + return false; + }; + let Some((idx_id, offset)) = packed_f64_loop_index_parts(index) else { return false; }; - ctx.i32_counter_slots.contains_key(idx_id) - && (packed_f64_loop_fact(ctx, *arr_id, *idx_id).is_some() - || ctx - .bounded_index_pairs - .iter() - .any(|fact| fact.array_local_id == *arr_id && fact.index_local_id == *idx_id)) + if !ctx.i32_counter_slots.contains_key(&idx_id) { + return false; + } + if packed_f64_loop_fact_for_index(ctx, *arr_id, index).is_some() { + return true; + } + offset == 0 + && ctx + .bounded_index_pairs + .iter() + .any(|fact| fact.array_local_id == *arr_id && fact.index_local_id == idx_id) } fn numeric_index_needs_runtime_key(ctx: &FnCtx<'_>, object: &Expr, index: &Expr) -> bool { @@ -471,9 +539,10 @@ fn lower_packed_f64_loop_index_get( arr_id: u32, arr_box: &str, idx_i32: &str, - guard_id: &str, - array_kind: PackedNumericLoopKind, + fact: &PackedF64LoopFact, ) -> String { + let guard_id = fact.guard_id.as_str(); + let array_kind = fact.array_kind; let value = { let blk = ctx.block(); let arr_bits = blk.bitcast_double_to_i64(arr_box); @@ -485,6 +554,24 @@ fn lower_packed_f64_loop_index_get( let element_ptr = blk.inttoptr(I64, &element_addr); blk.load(DOUBLE, &element_ptr) }; + if fact.allow_holes { + // #6011: hole-tolerant range-guarded loop — the guard proved every + // slot in the window is a raw-f64 number OR TAG_HOLE. Reading a hole + // must observe `undefined` (or a polluted prototype), so side-exit to + // the slow preheader, which re-executes the current iteration through + // the generic read path. The side exit fires before any effect of the + // iteration (matcher invariant), so the re-run cannot double-apply. + let is_hole = { + let blk = ctx.block(); + let raw_bits = blk.bitcast_double_to_i64(&value); + blk.icmp_eq(I64, &raw_bits, crate::nanbox::TAG_HOLE_I64) + }; + let cont_idx = ctx.new_block("packed_f64_range.load.cont"); + let cont_label = ctx.block_label(cont_idx); + ctx.block() + .cond_br(&is_hole, &fact.store_side_exit_label, &cont_label); + ctx.current_block = cont_idx; + } let lowered = LoweredValue { semantic: SemanticKind::JsNumber, rep: NativeRep::F64, @@ -898,21 +985,20 @@ pub(crate) fn lower_numeric_index_get_for_number_context( return Ok(None); } - if let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = (object.as_ref(), index.as_ref()) { - if let Some(fact) = packed_f64_loop_fact(ctx, *arr_id, *idx_id) { - if let Some(i32_slot) = ctx.i32_counter_slots.get(idx_id).cloned() { + if let Expr::LocalGet(arr_id) = object.as_ref() { + if let Some((fact, idx_id, offset)) = + packed_f64_loop_fact_for_index(ctx, *arr_id, index.as_ref()) + { + if let Some(i32_slot) = ctx.i32_counter_slots.get(&idx_id).cloned() { let arr_box = lower_expr(ctx, object)?; - let idx_i32 = ctx.block().load(I32, &i32_slot); + let idx_i32 = load_packed_loop_index_i32(ctx, &i32_slot, offset); return Ok(Some(lower_packed_f64_loop_index_get( - ctx, - *arr_id, - &arr_box, - &idx_i32, - &fact.guard_id, - fact.array_kind, + ctx, *arr_id, &arr_box, &idx_i32, &fact, ))); } } + } + if let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = (object.as_ref(), index.as_ref()) { if ctx .bounded_index_pairs .iter() @@ -1515,23 +1601,22 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // we can skip the bound check + OOB phi entirely. // The loop already proved `i < arr.length` and the // body provably can't change `arr.length`. - if let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = - (object.as_ref(), index.as_ref()) - { - if let Some(fact) = packed_f64_loop_fact(ctx, *arr_id, *idx_id) { - if let Some(i32_slot) = ctx.i32_counter_slots.get(idx_id).cloned() { + if let Expr::LocalGet(arr_id) = object.as_ref() { + if let Some((fact, idx_id, offset)) = + packed_f64_loop_fact_for_index(ctx, *arr_id, index.as_ref()) + { + if let Some(i32_slot) = ctx.i32_counter_slots.get(&idx_id).cloned() { let arr_box = lower_expr(ctx, object)?; - let idx_i32 = ctx.block().load(I32, &i32_slot); + let idx_i32 = load_packed_loop_index_i32(ctx, &i32_slot, offset); return Ok(lower_packed_f64_loop_index_get( - ctx, - *arr_id, - &arr_box, - &idx_i32, - &fact.guard_id, - fact.array_kind, + ctx, *arr_id, &arr_box, &idx_i32, &fact, )); } } + } + if let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = + (object.as_ref(), index.as_ref()) + { if ctx.bounded_index_pairs.iter().any(|fact| { fact.index_local_id == *idx_id && fact.array_local_id == *arr_id }) { diff --git a/crates/perry-codegen/src/expr/index_set.rs b/crates/perry-codegen/src/expr/index_set.rs index 63b084755..07eb131ff 100644 --- a/crates/perry-codegen/src/expr/index_set.rs +++ b/crates/perry-codegen/src/expr/index_set.rs @@ -189,16 +189,49 @@ fn packed_f64_loop_fact(ctx: &FnCtx<'_>, arr_id: u32, idx_id: u32) -> Option, + arr_id: u32, + index: &Expr, +) -> Option<(PackedF64LoopFact, u32, i32)> { + let (idx_id, offset) = super::packed_f64_loop_index_parts(index)?; + let fact = packed_f64_loop_fact(ctx, arr_id, idx_id)?; + if offset != 0 && !fact.allow_holes { + return None; + } + Some((fact, idx_id, offset)) +} + +fn load_packed_loop_index_i32(ctx: &mut FnCtx<'_>, i32_slot: &str, offset: i32) -> String { + let idx_i32 = ctx.block().load(I32, i32_slot); + match offset.cmp(&0) { + std::cmp::Ordering::Equal => idx_i32, + std::cmp::Ordering::Greater => ctx.block().add(I32, &idx_i32, &offset.to_string()), + std::cmp::Ordering::Less => ctx.block().sub(I32, &idx_i32, &(-offset).to_string()), + } +} + fn numeric_index_has_loop_array_index_proof(ctx: &FnCtx<'_>, object: &Expr, index: &Expr) -> bool { - let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = (object, index) else { + let Expr::LocalGet(arr_id) = object else { + return false; + }; + let Some((idx_id, offset)) = super::packed_f64_loop_index_parts(index) else { return false; }; - ctx.i32_counter_slots.contains_key(idx_id) - && (packed_f64_loop_fact(ctx, *arr_id, *idx_id).is_some() - || ctx - .bounded_index_pairs - .iter() - .any(|fact| fact.array_local_id == *arr_id && fact.index_local_id == *idx_id)) + if !ctx.i32_counter_slots.contains_key(&idx_id) { + return false; + } + if packed_f64_loop_fact_for_index(ctx, *arr_id, index).is_some() { + return true; + } + offset == 0 + && ctx + .bounded_index_pairs + .iter() + .any(|fact| fact.array_local_id == *arr_id && fact.index_local_id == idx_id) } fn numeric_index_needs_runtime_key(ctx: &FnCtx<'_>, object: &Expr, index: &Expr) -> bool { @@ -329,6 +362,146 @@ fn lower_packed_numeric_loop_store_value( } } +/// #6011: inline store for the hole-tolerant *range-guarded* packed-f64 loop. +/// +/// The range guard already proved at loop entry that every index this loop +/// can touch is in bounds, that the receiver is a plain, mutable (not +/// frozen/sealed), descriptor-free array, and that its slots are raw-f64 +/// numbers or `TAG_HOLE` — and the matcher proved the body cannot invalidate +/// any of that mid-loop (no calls/closures/awaits, stores only through this +/// path). The only per-iteration check left is on the RHS *value*: a NaN-boxed +/// non-double (string/object/undefined/INT32-boxed int/…) side-exits to the +/// slow loop, which re-executes the current iteration through the generic +/// store (the side exit fires before the store, so nothing double-applies). +/// The store itself is a raw f64 write; overwriting `TAG_HOLE` with a number +/// is exactly JS element definition on an in-bounds index, and a number never +/// carries a heap edge, so no barrier / layout note is needed (the guard +/// (re)asserted the pointer-free GC layout). +fn lower_packed_f64_range_loop_index_set( + ctx: &mut FnCtx<'_>, + arr_id: u32, + idx_i32: &str, + value: &Expr, + guard_id: &str, + side_exit_label: &str, +) -> Result { + let (val_double, rhs_notes) = lower_packed_f64_loop_store_value(ctx, arr_id, value)?; + + let fast_idx = ctx.new_block("packed_f64_range_store.fast"); + let exit_idx = ctx.new_block("packed_f64_range_store.side_exit"); + let fast_label = ctx.block_label(fast_idx); + let exit_label = ctx.block_label(exit_idx); + + // Numeric-bits check: (bits >> 48) - 0x7FF9 , arr_id: u32, @@ -337,7 +510,18 @@ fn lower_packed_numeric_loop_index_set( guard_id: &str, side_exit_label: &str, array_kind: PackedNumericLoopKind, + allow_holes: bool, ) -> Result { + if allow_holes && matches!(array_kind, PackedNumericLoopKind::F64) { + return lower_packed_f64_range_loop_index_set( + ctx, + arr_id, + idx_i32, + value, + guard_id, + side_exit_label, + ); + } let (val_double, native_value, rhs_notes) = lower_packed_numeric_loop_store_value(ctx, arr_id, value, array_kind)?; let arr_expr = Expr::LocalGet(arr_id); @@ -863,17 +1047,17 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // for-loop already proved `i < arr.length` and the // body provably can't change `arr.length`, so the // IndexSet at `arr[i]` is statically inbounds. - if let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = - (object.as_ref(), index.as_ref()) - { - if let Some(fact) = packed_f64_loop_fact(ctx, *arr_id, *idx_id) { + if let Expr::LocalGet(arr_id) = object.as_ref() { + if let Some((fact, idx_id, offset)) = + packed_f64_loop_fact_for_index(ctx, *arr_id, index.as_ref()) + { // Packed-U32 typed-slot stores are not implemented; rather // than abort codegen, let U32 facts fall through to the // generic/bounded array-store path below (correct, just // not the packed fast path). if !matches!(fact.array_kind, PackedNumericLoopKind::U32) { - if let Some(i32_slot) = ctx.i32_counter_slots.get(idx_id).cloned() { - let idx_i32 = ctx.block().load(I32, &i32_slot); + if let Some(i32_slot) = ctx.i32_counter_slots.get(&idx_id).cloned() { + let idx_i32 = load_packed_loop_index_i32(ctx, &i32_slot, offset); return lower_packed_numeric_loop_index_set( ctx, *arr_id, @@ -882,10 +1066,15 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { &fact.guard_id, &fact.store_side_exit_label, fact.array_kind, + fact.allow_holes, ); } } } + } + if let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = + (object.as_ref(), index.as_ref()) + { if ctx.bounded_index_pairs.iter().any(|fact| { fact.index_local_id == *idx_id && fact.array_local_id == *arr_id }) { diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index d4b0ddaba..937925fdc 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -1140,6 +1140,14 @@ pub(crate) struct PackedF64LoopFact { pub guard_id: String, pub store_side_exit_label: String, pub array_kind: PackedNumericLoopKind, + /// #6011: true when the guard that established this fact was the + /// hole-tolerant *range* guard (`js_typed_feedback_packed_f64_range_loop_ + /// guard`): slots inside the guarded index window are raw-f64 numbers OR + /// `TAG_HOLE`. Inline loads must hole-check and side-exit to + /// `store_side_exit_label` on a hole; inline stores must runtime-check the + /// RHS is numeric bits (side-exiting otherwise) and skip the per-iteration + /// store guard — the range guard already proved bounds and mutability. + pub allow_holes: bool, } impl<'a> FnCtx<'a> { @@ -1216,6 +1224,7 @@ mod dyn_extern_i18n; mod env_clones; mod fs_await; mod index_get; +pub(crate) use index_get::packed_f64_loop_index_parts; mod index_set; mod instance_misc1; pub(crate) use instance_misc1::builtin_parent_reserved_class_id; diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index 1077ea8c5..87162e9e9 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -317,6 +317,13 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { I32, &[I64, DOUBLE], ); + // #6011: range-preguarded packed-f64 loop — validates a whole + // [min_idx, max_idx_exclusive) index window (hole-tolerant) at loop entry. + module.declare_function( + "js_typed_feedback_packed_f64_range_loop_guard", + I32, + &[I64, DOUBLE, I32, I32], + ); module.declare_function( "js_typed_feedback_packed_u32_array_loop_guard", I32, diff --git a/crates/perry-codegen/src/stmt/loops.rs b/crates/perry-codegen/src/stmt/loops.rs index b8eb5d3e0..bdfd0df4d 100644 --- a/crates/perry-codegen/src/stmt/loops.rs +++ b/crates/perry-codegen/src/stmt/loops.rs @@ -358,6 +358,7 @@ fn lower_packed_f64_versioned_for( guard_id: guard_id.to_string(), store_side_exit_label: slow_pre_label.clone(), array_kind: matched.array_kind, + allow_holes: false, }); lower_for_after_init( ctx, @@ -390,6 +391,601 @@ fn lower_packed_f64_versioned_for( Ok(true) } +/// #6011: cap on the |constant offset| accepted in `arr[i ± c]` accesses by +/// the range-preguarded packed-f64 loop matcher. +const PACKED_F64_RANGE_LOOP_MAX_OFFSET: i64 = 64; + +#[derive(Clone, Copy)] +enum PackedF64RangeLoopBound { + /// `i < `. + Constant(i64), + /// `i < b` where `b` is a loop-invariant plain local or module global. + Local(u32), +} + +#[derive(Clone, Copy)] +struct PackedF64RangeArrayAccess { + array_id: u32, + /// Smallest / largest constant offset `c` over all `arr[i + c]` accesses. + min_offset: i32, + max_offset: i32, + written: bool, +} + +struct PackedF64RangeLoop { + counter_id: u32, + /// Loop-entry counter value (`let i = `), proven in `0..=i32::MAX`. + start: i64, + bound: PackedF64RangeLoopBound, + /// Per-array access windows, ordered by array local id (deterministic). + arrays: Vec, +} + +/// #6011: range-preguarded packed-f64 versioned loop. +/// +/// Matches `for (let i = k0; i < B; i++) ` where `B` is an +/// integer literal or a loop-invariant local/module-global, and every array +/// access in the body is `a[i]` / `a[i ± c]` (|c| ≤ 64) on eligible +/// number-array locals. Unlike [`match_packed_f64_versioned_loop`] the bound +/// is NOT `arr.length`, so bounds cannot be proven per-array statically — +/// instead a runtime guard validates the whole static index window +/// `[k0 + min_offset, B + max_offset)` against each array's length at loop +/// entry (hole-tolerantly: `new Array(n)` slots start as TAG_HOLE). +/// +/// The body is restricted to ONE statement whose only side effect (a tracked +/// array store, or a scalar `LocalSet`/`Update`) completes after every +/// potential side exit (hole-checked loads / the store's numeric-RHS check), +/// so a side exit into the slow loop re-executes the current iteration +/// without duplicating effects. +fn match_packed_f64_range_loop( + ctx: &FnCtx<'_>, + init: Option<&Stmt>, + condition: Option<&perry_hir::Expr>, + update: Option<&perry_hir::Expr>, + body: &[Stmt], +) -> Option { + use perry_hir::{CompareOp, Expr, UpdateOp}; + if !ctx.pending_labels.is_empty() { + return None; + } + let (counter_id, start) = match init? { + Stmt::Let { + id, + init: Some(init_expr), + .. + } => { + let start = match init_expr { + Expr::Integer(n) => *n, + Expr::Number(n) if n.is_finite() && n.fract() == 0.0 => *n as i64, + _ => return None, + }; + (*id, start) + } + _ => return None, + }; + if !(0..=i64::from(i32::MAX)).contains(&start) { + return None; + } + let (op, left, right) = match condition? { + Expr::Compare { op, left, right } => (*op, left.as_ref(), right.as_ref()), + _ => return None, + }; + if !matches!(op, CompareOp::Lt) || !matches!(left, Expr::LocalGet(id) if *id == counter_id) { + return None; + } + let bound = match right { + // Cap constants at i32::MAX - 64 so `bound + max_offset` cannot + // overflow the guard's i32 argument. + Expr::Integer(k) + if (0..=i64::from(i32::MAX) - PACKED_F64_RANGE_LOOP_MAX_OFFSET).contains(k) => + { + PackedF64RangeLoopBound::Constant(*k) + } + Expr::LocalGet(bound_id) if *bound_id != counter_id => { + // Boxed bounds live behind a closure cell the once-per-entry load + // below does not model. Plain locals AND module globals are fine: + // the body walk rejects every call/await/closure, so nothing can + // mutate the global mid-loop, and direct writes to `bound_id` in + // cond/update/body are rejected by the invariance walker. + if ctx.boxed_vars.contains(bound_id) { + return None; + } + if !ctx.locals.contains_key(bound_id) && !ctx.module_globals.contains_key(bound_id) { + return None; + } + if !local_bound_is_loop_invariant(condition?, update, body, *bound_id) { + return None; + } + PackedF64RangeLoopBound::Local(*bound_id) + } + _ => return None, + }; + if !matches!( + update?, + Expr::Update { + id, + op: UpdateOp::Increment, + .. + } if *id == counter_id + ) { + return None; + } + if !ctx.locals.contains_key(&counter_id) + || ctx.boxed_vars.contains(&counter_id) + || !ctx.integer_locals.contains(&counter_id) + || !loop_counter_bounds_are_safe(ctx, counter_id, update, body) + || !loop_counter_entry_i32_range_is_safe(init, counter_id) + { + return None; + } + + let bound_local = match bound { + PackedF64RangeLoopBound::Local(b) => Some(b), + PackedF64RangeLoopBound::Constant(_) => None, + }; + let mut accesses: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + if !packed_f64_range_loop_body_collect(body, counter_id, bound_local, &mut accesses) { + return None; + } + if accesses.is_empty() { + // No tracked array access — nothing for the versioned loop to win. + return None; + } + for access in accesses.values() { + let arr_id = access.array_id; + if !ctx.locals.contains_key(&arr_id) + || ctx.boxed_vars.contains(&arr_id) + || ctx.module_globals.contains_key(&arr_id) + || ctx.scalar_replaced_arrays.contains_key(&arr_id) + || ctx.native_facts.has_materialization_hazard(arr_id) + { + return None; + } + // The guard takes i32 window endpoints; make sure `start + offset` + // still fits (bound-side overflow is prevented by the constant cap / + // runtime bound range check). + let min_idx = start + i64::from(access.min_offset); + let max_base = start + i64::from(access.max_offset); + if !(i64::from(i32::MIN)..=i64::from(i32::MAX)).contains(&min_idx) + || !(i64::from(i32::MIN)..=i64::from(i32::MAX)).contains(&max_base) + { + return None; + } + if access.written { + if !local_allows_packed_f64_loop_store(ctx, arr_id) + || !ctx + .native_facts + .packed_f64_eligible_for_guarded_store(arr_id) + { + return None; + } + } else if !local_is_number_array(ctx, arr_id) + || !ctx.native_facts.proves_packed_f64_array(arr_id) + { + return None; + } + } + Some(PackedF64RangeLoop { + counter_id, + start, + bound, + arrays: accesses.into_values().collect(), + }) +} + +fn record_packed_f64_range_access( + accesses: &mut std::collections::BTreeMap, + array_id: u32, + offset: i32, + written: bool, +) { + let entry = accesses + .entry(array_id) + .or_insert(PackedF64RangeArrayAccess { + array_id, + min_offset: offset, + max_offset: offset, + written, + }); + entry.min_offset = entry.min_offset.min(offset); + entry.max_offset = entry.max_offset.max(offset); + entry.written |= written; +} + +/// `i` → 0, `i + c` / `c + i` → c, `i - c` → -c, with |result| ≤ 64. +fn packed_f64_range_loop_index_offset(index: &perry_hir::Expr, counter_id: u32) -> Option { + use perry_hir::{BinaryOp, Expr}; + let offset = match index { + Expr::LocalGet(id) if *id == counter_id => Some(0i64), + Expr::Binary { op, left, right } if matches!(op, BinaryOp::Add | BinaryOp::Sub) => { + match (left.as_ref(), right.as_ref()) { + (Expr::LocalGet(id), Expr::Integer(c)) if *id == counter_id => { + if matches!(op, BinaryOp::Sub) { + c.checked_neg() + } else { + Some(*c) + } + } + (Expr::Integer(c), Expr::LocalGet(id)) + if *id == counter_id && matches!(op, BinaryOp::Add) => + { + Some(*c) + } + _ => None, + } + } + _ => None, + }?; + if offset.unsigned_abs() > PACKED_F64_RANGE_LOOP_MAX_OFFSET as u64 { + return None; + } + i32::try_from(offset).ok() +} + +/// Body walk for [`match_packed_f64_range_loop`]: exactly one expression +/// statement whose single side effect happens after all potential side exits. +fn packed_f64_range_loop_body_collect( + body: &[Stmt], + counter_id: u32, + bound_local: Option, + accesses: &mut std::collections::BTreeMap, +) -> bool { + use perry_hir::Expr; + let [Stmt::Expr(expr)] = body else { + return false; + }; + match expr { + Expr::IndexSet { + object, + index, + value, + } => packed_f64_range_loop_store_collect(object, index, value, counter_id, accesses), + Expr::PutValueSet { + target, + key, + value, + receiver, + .. + } if matches!( + (target.as_ref(), receiver.as_ref()), + (Expr::LocalGet(a), Expr::LocalGet(b)) if a == b + ) => + { + packed_f64_range_loop_store_collect(target, key, value, counter_id, accesses) + } + // Scalar accumulator: `sum = ` / `sum += a[i]`. The LocalSet + // completes after its RHS fully evaluates, so a hole-read side exit + // in the RHS re-executes the iteration without a double-update. The + // counter/bound are already proven unwritten by the walkers above; + // the "target is not a tracked array" half is validated by the + // caller once the access map is complete. + Expr::LocalSet(id, value) => { + *id != counter_id + && Some(*id) != bound_local + && packed_f64_range_loop_pure_expr_collect(value, counter_id, accesses) + && !accesses.contains_key(id) + } + _ => packed_f64_range_loop_pure_expr_collect(expr, counter_id, accesses), + } +} + +/// #6011: module globals READ (and provably never written — the matched +/// body's only possible write target is the top-level `LocalSet`, which the +/// caller passes as `written_local`) inside the matched loop body. The +/// versioned lowering caches these into entry stack slots so LLVM can keep +/// them in registers: the raw inttoptr element stores in the fast loop +/// otherwise force a re-load of every `@perry_global_*` each iteration. +fn packed_f64_range_loop_invariant_global_reads( + ctx: &FnCtx<'_>, + body: &[Stmt], + written_local: Option, +) -> Vec { + use perry_hir::Expr; + let [Stmt::Expr(expr)] = body else { + return Vec::new(); + }; + let mut globals = std::collections::BTreeSet::new(); + fn walk( + ctx: &FnCtx<'_>, + expr: &perry_hir::Expr, + written_local: Option, + globals: &mut std::collections::BTreeSet, + ) { + if let Expr::LocalGet(id) = expr { + if Some(*id) != written_local + && !ctx.locals.contains_key(id) + && ctx.module_globals.contains_key(id) + { + globals.insert(*id); + } + } + perry_hir::walker::walk_expr_children(expr, &mut |child| { + walk(ctx, child, written_local, globals); + }); + } + walk(ctx, expr, written_local, &mut globals); + globals.into_iter().collect() +} + +fn packed_f64_range_loop_store_collect( + object: &perry_hir::Expr, + index: &perry_hir::Expr, + value: &perry_hir::Expr, + counter_id: u32, + accesses: &mut std::collections::BTreeMap, +) -> bool { + use perry_hir::Expr; + let Expr::LocalGet(arr_id) = object else { + return false; + }; + let Some(offset) = packed_f64_range_loop_index_offset(index, counter_id) else { + return false; + }; + if !packed_f64_range_loop_pure_expr_collect(value, counter_id, accesses) { + return false; + } + record_packed_f64_range_access(accesses, *arr_id, offset, true); + true +} + +/// Effect-free expression walk: tracked `a[i ± c]` reads, locals, literals and +/// pure arithmetic/Math only. Any store, call, update, closure, or index read +/// with an unrecognized receiver/index shape bails the whole match. +fn packed_f64_range_loop_pure_expr_collect( + expr: &perry_hir::Expr, + counter_id: u32, + accesses: &mut std::collections::BTreeMap, +) -> bool { + use perry_hir::Expr; + match expr { + Expr::IndexGet { object, index } => { + let Expr::LocalGet(arr_id) = object.as_ref() else { + return false; + }; + let Some(offset) = packed_f64_range_loop_index_offset(index, counter_id) else { + return false; + }; + record_packed_f64_range_access(accesses, *arr_id, offset, false); + true + } + Expr::LocalGet(_) + | Expr::Number(_) + | Expr::Integer(_) + | Expr::Bool(_) + | Expr::Null + | Expr::Undefined => true, + Expr::Binary { left, right, .. } + | Expr::Compare { left, right, .. } + | Expr::Logical { left, right, .. } => { + packed_f64_range_loop_pure_expr_collect(left, counter_id, accesses) + && packed_f64_range_loop_pure_expr_collect(right, counter_id, accesses) + } + Expr::Unary { operand, .. } + | Expr::Void(operand) + | Expr::TypeOf(operand) + | Expr::NumberCoerce(operand) + | Expr::BooleanCoerce(operand) => { + packed_f64_range_loop_pure_expr_collect(operand, counter_id, accesses) + } + Expr::Conditional { + condition, + then_expr, + else_expr, + } => { + packed_f64_range_loop_pure_expr_collect(condition, counter_id, accesses) + && packed_f64_range_loop_pure_expr_collect(then_expr, counter_id, accesses) + && packed_f64_range_loop_pure_expr_collect(else_expr, counter_id, accesses) + } + Expr::MathImul(left, right) | Expr::MathPow(left, right) => { + packed_f64_range_loop_pure_expr_collect(left, counter_id, accesses) + && packed_f64_range_loop_pure_expr_collect(right, counter_id, accesses) + } + Expr::MathMin(values) | Expr::MathMax(values) => values + .iter() + .all(|expr| packed_f64_range_loop_pure_expr_collect(expr, counter_id, accesses)), + Expr::MathAbs(value) + | Expr::MathSqrt(value) + | Expr::MathFloor(value) + | Expr::MathCeil(value) + | Expr::MathRound(value) + | Expr::MathTrunc(value) + | Expr::MathSign(value) + | Expr::MathF16round(value) => { + packed_f64_range_loop_pure_expr_collect(value, counter_id, accesses) + } + _ => false, + } +} + +/// #6011: lowering for [`match_packed_f64_range_loop`], modeled on +/// [`lower_packed_f64_versioned_for`]. The bound is materialized to i32 once +/// (with a runtime finite-integral check for local/global bounds), one range +/// guard runs per accessed array, and the AND of the guards picks the fast +/// loop (hole-tolerant `PackedF64LoopFact` per array; side exits resume at +/// the current `i` in the slow copy) or the slow loop. +fn lower_packed_f64_range_versioned_for( + ctx: &mut FnCtx<'_>, + init: Option<&Stmt>, + condition: Option<&perry_hir::Expr>, + update: Option<&perry_hir::Expr>, + body: &[Stmt], +) -> Result { + let Some(matched) = match_packed_f64_range_loop(ctx, init, condition, update, body) else { + return Ok(false); + }; + // The inline load/store fast paths read the counter through its i32 + // shadow slot; without one the versioned copy would win nothing. + if !ctx.i32_counter_slots.contains_key(&matched.counter_id) { + return Ok(false); + } + + // Cache loop-invariant module-global reads (e.g. `alpha` in the EMA + // recurrence) into entry stack slots and alias them into `ctx.locals` + // for the duration of both loop copies. The matched body cannot write + // them (its only writable target is the top-level LocalSet, which is + // excluded) and contains no calls/closures/awaits, so the cached value + // is exact for the whole loop — and, unlike a `@perry_global_*` load, + // a non-escaping alloca is promotable to a register even with the fast + // loop's raw inttoptr element stores in the way. + let written_local = match body { + [Stmt::Expr(perry_hir::Expr::LocalSet(id, _))] => Some(*id), + _ => None, + }; + let mut global_override_ids: Vec = Vec::new(); + for gid in packed_f64_range_loop_invariant_global_reads(ctx, body, written_local) { + let Some(global_name) = ctx.module_globals.get(&gid).cloned() else { + continue; + }; + let slot = ctx.func.alloca_entry(DOUBLE); + let g_ref = format!("@{global_name}"); + let val = ctx.block().load(DOUBLE, &g_ref); + ctx.block().store(DOUBLE, &val, &slot); + ctx.locals.insert(gid, slot); + global_override_ids.push(gid); + } + + let fast_pre_idx = ctx.new_block("packed_f64_range.loop.fast.preheader"); + let slow_pre_idx = ctx.new_block("packed_f64_range.loop.slow.preheader"); + let merge_idx = ctx.new_block("packed_f64_range.loop.merge"); + let fast_pre_label = ctx.block_label(fast_pre_idx); + let slow_pre_label = ctx.block_label(slow_pre_idx); + let merge_label = ctx.block_label(merge_idx); + + let bound_i32: String = match matched.bound { + PackedF64RangeLoopBound::Constant(k) => k.to_string(), + PackedF64RangeLoopBound::Local(bound_id) => { + // One-time finite-integral-i32 materialization of the bound. + // Non-number / NaN / fractional / out-of-range bounds keep full + // JS trip-count semantics in the slow loop. The upper cap leaves + // room for `bound + max_offset` in i32. The fptosi lives in its + // own guarded block so its result is never poison when used. + let bound_d = lower_expr(ctx, &perry_hir::Expr::LocalGet(bound_id))?; + let is_number = emit_js_value_is_number(ctx, &bound_d); + let range_idx = ctx.new_block("packed_f64_range.bound.range"); + let convert_idx = ctx.new_block("packed_f64_range.bound.convert"); + let guards_idx = ctx.new_block("packed_f64_range.guards"); + let range_label = ctx.block_label(range_idx); + let convert_label = ctx.block_label(convert_idx); + let guards_label = ctx.block_label(guards_idx); + ctx.block() + .cond_br(&is_number, &range_label, &slow_pre_label); + + ctx.current_block = range_idx; + let ge_zero = ctx.block().fcmp("oge", &bound_d, "0.0"); + let le_max = { + let max_literal = format!( + "{:.1}", + (i64::from(i32::MAX) - PACKED_F64_RANGE_LOOP_MAX_OFFSET) as f64 + ); + ctx.block().fcmp("ole", &bound_d, &max_literal) + }; + let in_range = ctx.block().and(I1, &ge_zero, &le_max); + ctx.block() + .cond_br(&in_range, &convert_label, &slow_pre_label); + + ctx.current_block = convert_idx; + let bound_i32 = ctx.block().fptosi(DOUBLE, &bound_d, I32); + let roundtrip = ctx.block().sitofp(I32, &bound_i32, DOUBLE); + let is_integral = ctx.block().fcmp("oeq", &roundtrip, &bound_d); + ctx.block() + .cond_br(&is_integral, &guards_label, &slow_pre_label); + + ctx.current_block = guards_idx; + bound_i32 + } + }; + + let guard_id = "packed_f64_range_loop_guard"; + let mut all_guards_ok: Option = None; + for access in &matched.arrays { + let arr_box = lower_expr(ctx, &perry_hir::Expr::LocalGet(access.array_id))?; + let feedback_site_id = emit_typed_feedback_register_site( + ctx, + TypedFeedbackKind::ArrayElement, + "array[packed_f64_range_loop]", + TypedFeedbackContract::packed_f64_array_loop(), + ); + let min_idx = (matched.start + i64::from(access.min_offset)).to_string(); + let max_idx = ctx + .block() + .add(I32, &bound_i32, &access.max_offset.to_string()); + let guard_i32 = ctx.block().call( + I32, + "js_typed_feedback_packed_f64_range_loop_guard", + &[ + (I64, &feedback_site_id), + (DOUBLE, &arr_box), + (I32, &min_idx), + (I32, &max_idx), + ], + ); + let guard_ok = ctx.block().icmp_ne(I32, &guard_i32, "0"); + all_guards_ok = Some(match all_guards_ok { + None => guard_ok, + Some(prev) => ctx.block().and(I1, &prev, &guard_ok), + }); + record_packed_f64_loop_guard_artifacts( + ctx, + access.array_id, + &arr_box, + guard_id, + PackedNumericLoopKind::F64, + ); + } + let all_guards_ok = all_guards_ok.expect("range loop matcher requires >= 1 array"); + ctx.block() + .cond_br(&all_guards_ok, &fast_pre_label, &slow_pre_label); + + let packed_scope_id = ctx.next_loop_proof_scope_id(); + + ctx.current_block = fast_pre_idx; + for access in &matched.arrays { + ctx.packed_f64_loop_facts.push(PackedF64LoopFact { + index_local_id: matched.counter_id, + array_local_id: access.array_id, + scope_id: packed_scope_id, + guard_id: guard_id.to_string(), + store_side_exit_label: slow_pre_label.clone(), + array_kind: PackedNumericLoopKind::F64, + allow_holes: true, + }); + } + lower_for_after_init_with_i32_bound( + ctx, + init, + condition, + update, + body, + "for.packed_f64_range_fast", + Some((matched.counter_id, bound_i32.clone())), + )?; + ctx.packed_f64_loop_facts + .retain(|fact| fact.scope_id != packed_scope_id); + if !ctx.block().is_terminated() { + ctx.block().br(&merge_label); + } + + ctx.current_block = slow_pre_idx; + lower_for_after_init( + ctx, + init, + condition, + update, + body, + "for.packed_f64_range_slow", + )?; + if !ctx.block().is_terminated() { + ctx.block().br(&merge_label); + } + + for gid in &global_override_ids { + ctx.locals.remove(gid); + } + ctx.current_block = merge_idx; + Ok(true) +} + fn record_packed_f64_loop_guard_artifacts( ctx: &mut FnCtx<'_>, arr_id: u32, @@ -583,34 +1179,53 @@ fn match_packed_f64_versioned_loop( }) } +/// #6011: element type of an array-typed local, accepting BOTH the +/// `Type::Array(elem)` spelling (`prices: number[]`) and the generic spelling +/// `Type::Generic { base: "Array", type_args: [elem] }` that `new +/// Array(n)` declarations carry. +fn local_array_element_type<'t>( + ctx: &'t FnCtx<'_>, + local_id: u32, +) -> Option<&'t perry_types::Type> { + match ctx.local_types.get(&local_id) { + Some(perry_types::Type::Array(elem)) => Some(elem.as_ref()), + Some(perry_types::Type::Generic { base, type_args }) + if base == "Array" && type_args.len() == 1 => + { + Some(&type_args[0]) + } + _ => None, + } +} + fn local_is_number_array(ctx: &FnCtx<'_>, local_id: u32) -> bool { matches!( - ctx.local_types.get(&local_id), - Some(perry_types::Type::Array(elem)) - if matches!(elem.as_ref(), perry_types::Type::Number | perry_types::Type::Int32) - || matches!(elem.as_ref(), perry_types::Type::Named(name) if name == "PerryU32") + local_array_element_type(ctx, local_id), + Some(perry_types::Type::Number | perry_types::Type::Int32) + ) || matches!( + local_array_element_type(ctx, local_id), + Some(perry_types::Type::Named(name)) if name == "PerryU32" ) } fn local_allows_packed_f64_loop_store(ctx: &FnCtx<'_>, local_id: u32) -> bool { matches!( - ctx.local_types.get(&local_id), - Some(perry_types::Type::Array(elem)) if matches!(elem.as_ref(), perry_types::Type::Number) + local_array_element_type(ctx, local_id), + Some(perry_types::Type::Number) ) } fn local_is_int32_array(ctx: &FnCtx<'_>, local_id: u32) -> bool { matches!( - ctx.local_types.get(&local_id), - Some(perry_types::Type::Array(elem)) if matches!(elem.as_ref(), perry_types::Type::Int32) + local_array_element_type(ctx, local_id), + Some(perry_types::Type::Int32) ) } fn local_is_u32_array(ctx: &FnCtx<'_>, local_id: u32) -> bool { matches!( - ctx.local_types.get(&local_id), - Some(perry_types::Type::Array(elem)) - if matches!(elem.as_ref(), perry_types::Type::Named(name) if name == "PerryU32") + local_array_element_type(ctx, local_id), + Some(perry_types::Type::Named(name)) if name == "PerryU32" ) } @@ -1028,6 +1643,13 @@ pub(crate) fn lower_for( return Ok(()); } + // #6011: `i < N`-bounded loops (N an integer literal or loop-invariant + // local/module-global) with `a[i ± c]` accesses — EMA-style recurrences. + // Tried only after the `i < arr.length` matcher above declined. + if lower_packed_f64_range_versioned_for(ctx, init, condition, update, body)? { + return Ok(()); + } + lower_for_after_init(ctx, init, condition, update, body, "for") } @@ -1038,6 +1660,26 @@ fn lower_for_after_init( update: Option<&perry_hir::Expr>, body: &[Stmt], label_prefix: &str, +) -> Result<()> { + lower_for_after_init_with_i32_bound(ctx, init, condition, update, body, label_prefix, None) +} + +/// #6011: like [`lower_for_after_init`], but the range-versioned fast copy can +/// hand down its already-materialized (finite-integral-validated) i32 loop +/// bound so the condition block emits `icmp slt i32` instead of re-lowering +/// the generic `i < N` comparison (a module-global load + `fcmp` per +/// iteration that LLVM cannot hoist past the loop's raw element stores). The +/// value must dominate the block this is emitted from — only the fast +/// preheader of the range-versioned loop qualifies. +#[allow(clippy::too_many_arguments)] +fn lower_for_after_init_with_i32_bound( + ctx: &mut FnCtx<'_>, + init: Option<&Stmt>, + condition: Option<&perry_hir::Expr>, + update: Option<&perry_hir::Expr>, + body: &[Stmt], + label_prefix: &str, + precomputed_i32_bound: Option<(u32, String)>, ) -> Result<()> { let loop_proof_scope_id = ctx.next_loop_proof_scope_id(); @@ -1310,85 +1952,103 @@ fn lower_for_after_init( // Cond block — fast i32 path when both counter and length are i32. ctx.current_block = cond_idx; - let used_i32_cond = - if let (Some(hoist), Some(ref len_i32_slot)) = (hoist_classification, &i32_length_slot) { - // Existing path: `i < arr.length` / `i <= arr.length` with - // hoisted i32 length. - if let Some(ctr_i32_slot) = ctx.i32_counter_slots.get(&hoist.counter_id).cloned() { - let mut ctr = ctx.block().load(I32, &ctr_i32_slot); - if hoist.lhs_addend != 0 { - ctr = ctx.block().add(I32, &ctr, &hoist.lhs_addend.to_string()); - } - let len = ctx.block().load(I32, len_i32_slot); - let cmp = match hoist.op { - perry_hir::CompareOp::Le => ctx.block().icmp_sle(I32, &ctr, &len), - _ => ctx.block().icmp_slt(I32, &ctr, &len), - }; - ctx.block().cond_br(&cmp, &body_label, &exit_label); - true - } else { - false - } - } else if let (Some((counter_id, _, op)), Some(ref bound_i32_slot)) = - (local_bound_classification, &i32_local_bound_slot) - { - // Issue #168: `i < n` / `i <= n` where `n` is statically proven - // safe for unguarded i32 materialization. The fptosi(n) was - // hoisted above; use icmp i32. - if let Some(ctr_i32_slot) = ctx.i32_counter_slots.get(&counter_id).cloned() { - let ctr = ctx.block().load(I32, &ctr_i32_slot); - let bound = ctx.block().load(I32, bound_i32_slot); - let cmp = match op { - perry_hir::CompareOp::Le => ctx.block().icmp_sle(I32, &ctr, &bound), - _ => ctx.block().icmp_slt(I32, &ctr, &bound), - }; - ctx.block().cond_br(&cmp, &body_label, &exit_label); - true - } else { - false + let used_precomputed_i32_cond = if let Some((counter_id, bound_i32)) = &precomputed_i32_bound { + // #6011: range-versioned fast copy — the caller already materialized + // and validated the loop bound as i32 (finite, integral, in range), + // and the matcher proved the strict `i < bound` shape with an + // increment-only integer counter, so `icmp slt i32` is trip-count + // exact. + if let Some(ctr_i32_slot) = ctx.i32_counter_slots.get(counter_id).cloned() { + let ctr = ctx.block().load(I32, &ctr_i32_slot); + let cmp = ctx.block().icmp_slt(I32, &ctr, bound_i32); + ctx.block().cond_br(&cmp, &body_label, &exit_label); + true + } else { + false + } + } else { + false + }; + let used_i32_cond = if used_precomputed_i32_cond { + true + } else if let (Some(hoist), Some(ref len_i32_slot)) = (hoist_classification, &i32_length_slot) { + // Existing path: `i < arr.length` / `i <= arr.length` with + // hoisted i32 length. + if let Some(ctr_i32_slot) = ctx.i32_counter_slots.get(&hoist.counter_id).cloned() { + let mut ctr = ctx.block().load(I32, &ctr_i32_slot); + if hoist.lhs_addend != 0 { + ctr = ctx.block().add(I32, &ctr, &hoist.lhs_addend.to_string()); } - } else if let Some(ref dyn_bound) = dynamic_i32_bound { - // Issue #168 follow-up: `i < n` / `i <= n` with a runtime-guarded - // local bound. Branch on the one-time finite-integral-i32 flag - // hoisted above: the fast loop uses `icmp`, and the slow loop keeps - // full JS comparison semantics. The branch is loop-invariant, so - // LLVM's LoopUnswitch peels it into two loops at -O2+; even - // unswitched, the hot path executes pure integer compares with no - // per-iteration `sitofp` / call. - if let Some(ctr_i32_slot) = ctx.i32_counter_slots.get(&dyn_bound.counter_id).cloned() { - let fast_idx = ctx.new_block(&format!("{label_prefix}.cond.fast")); - let slow_idx = ctx.new_block(&format!("{label_prefix}.cond.slow")); - let fast_label = ctx.block_label(fast_idx); - let slow_label = ctx.block_label(slow_idx); - let flag = ctx.block().load(I1, &dyn_bound.flag_slot); - ctx.block().cond_br(&flag, &fast_label, &slow_label); - - // Fast path: integer induction variable + `icmp`. - ctx.current_block = fast_idx; - let ctr = ctx.block().load(I32, &ctr_i32_slot); - let bound = ctx.block().load(I32, &dyn_bound.bound_i32_slot); - let cmp = match dyn_bound.op { - perry_hir::CompareOp::Le => ctx.block().icmp_sle(I32, &ctr, &bound), - _ => ctx.block().icmp_slt(I32, &ctr, &bound), - }; - ctx.block().cond_br(&cmp, &body_label, &exit_label); - - // Slow path: generic per-iteration comparison (full coercion). - ctx.current_block = slow_idx; - if let Some(cond_expr) = condition { - let cv = lower_expr(ctx, cond_expr)?; - let i1 = lower_truthy(ctx, &cv, cond_expr); - ctx.block().cond_br(&i1, &body_label, &exit_label); - } else { - ctx.block().br(&body_label); - } - true + let len = ctx.block().load(I32, len_i32_slot); + let cmp = match hoist.op { + perry_hir::CompareOp::Le => ctx.block().icmp_sle(I32, &ctr, &len), + _ => ctx.block().icmp_slt(I32, &ctr, &len), + }; + ctx.block().cond_br(&cmp, &body_label, &exit_label); + true + } else { + false + } + } else if let (Some((counter_id, _, op)), Some(ref bound_i32_slot)) = + (local_bound_classification, &i32_local_bound_slot) + { + // Issue #168: `i < n` / `i <= n` where `n` is statically proven + // safe for unguarded i32 materialization. The fptosi(n) was + // hoisted above; use icmp i32. + if let Some(ctr_i32_slot) = ctx.i32_counter_slots.get(&counter_id).cloned() { + let ctr = ctx.block().load(I32, &ctr_i32_slot); + let bound = ctx.block().load(I32, bound_i32_slot); + let cmp = match op { + perry_hir::CompareOp::Le => ctx.block().icmp_sle(I32, &ctr, &bound), + _ => ctx.block().icmp_slt(I32, &ctr, &bound), + }; + ctx.block().cond_br(&cmp, &body_label, &exit_label); + true + } else { + false + } + } else if let Some(ref dyn_bound) = dynamic_i32_bound { + // Issue #168 follow-up: `i < n` / `i <= n` with a runtime-guarded + // local bound. Branch on the one-time finite-integral-i32 flag + // hoisted above: the fast loop uses `icmp`, and the slow loop keeps + // full JS comparison semantics. The branch is loop-invariant, so + // LLVM's LoopUnswitch peels it into two loops at -O2+; even + // unswitched, the hot path executes pure integer compares with no + // per-iteration `sitofp` / call. + if let Some(ctr_i32_slot) = ctx.i32_counter_slots.get(&dyn_bound.counter_id).cloned() { + let fast_idx = ctx.new_block(&format!("{label_prefix}.cond.fast")); + let slow_idx = ctx.new_block(&format!("{label_prefix}.cond.slow")); + let fast_label = ctx.block_label(fast_idx); + let slow_label = ctx.block_label(slow_idx); + let flag = ctx.block().load(I1, &dyn_bound.flag_slot); + ctx.block().cond_br(&flag, &fast_label, &slow_label); + + // Fast path: integer induction variable + `icmp`. + ctx.current_block = fast_idx; + let ctr = ctx.block().load(I32, &ctr_i32_slot); + let bound = ctx.block().load(I32, &dyn_bound.bound_i32_slot); + let cmp = match dyn_bound.op { + perry_hir::CompareOp::Le => ctx.block().icmp_sle(I32, &ctr, &bound), + _ => ctx.block().icmp_slt(I32, &ctr, &bound), + }; + ctx.block().cond_br(&cmp, &body_label, &exit_label); + + // Slow path: generic per-iteration comparison (full coercion). + ctx.current_block = slow_idx; + if let Some(cond_expr) = condition { + let cv = lower_expr(ctx, cond_expr)?; + let i1 = lower_truthy(ctx, &cv, cond_expr); + ctx.block().cond_br(&i1, &body_label, &exit_label); } else { - false + ctx.block().br(&body_label); } + true } else { false - }; + } + } else { + false + }; if !used_i32_cond { if let Some(cond_expr) = condition { let cv = lower_expr(ctx, cond_expr)?; diff --git a/crates/perry-codegen/src/type_analysis/numeric.rs b/crates/perry-codegen/src/type_analysis/numeric.rs index 94173700d..528d295b8 100644 --- a/crates/perry-codegen/src/type_analysis/numeric.rs +++ b/crates/perry-codegen/src/type_analysis/numeric.rs @@ -221,6 +221,14 @@ pub(crate) fn is_numeric_expr(ctx: &FnCtx<'_>, e: &Expr) -> bool { Some(HirType::Array(elem)) => { matches!(**elem, HirType::Number | HirType::Int32) } + // #6011: `new Array(n)` locals carry the generic + // spelling `Generic { base: "Array", type_args: [Number] }`; + // element reads are numeric exactly like `Array(Number)`. + Some(HirType::Generic { base, type_args }) + if base == "Array" && type_args.len() == 1 => + { + matches!(type_args[0], HirType::Number | HirType::Int32) + } Some(HirType::Named(name)) => is_numeric_typed_array_class(name), _ => false, } diff --git a/crates/perry-codegen/src/type_analysis/pod.rs b/crates/perry-codegen/src/type_analysis/pod.rs index 59a515d0b..3752d8e38 100644 --- a/crates/perry-codegen/src/type_analysis/pod.rs +++ b/crates/perry-codegen/src/type_analysis/pod.rs @@ -388,6 +388,13 @@ pub(crate) fn scalar_replaced_array_element_is_raw_f64( fn type_has_numeric_pointer_free_array_layout_for_fallback(ty: &HirType) -> bool { match ty { HirType::Array(elem) => matches!(elem.as_ref(), HirType::Number | HirType::Int32), + // #6011: `new Array(n)` carries the generic spelling; its + // element reads have the same boxed-fallback hazard as `Array(Number)` + // (a hole surfaces `undefined` from the guarded read's boxed fallback, + // which must be coerced before raw f64 arithmetic). + HirType::Generic { base, type_args } if base == "Array" && type_args.len() == 1 => { + matches!(type_args[0], HirType::Number | HirType::Int32) + } HirType::Tuple(elems) => elems .iter() .all(|elem| matches!(elem, HirType::Number | HirType::Int32)), diff --git a/crates/perry-codegen/tests/native_proof_regressions.rs b/crates/perry-codegen/tests/native_proof_regressions.rs index 37d763c4a..409996bda 100644 --- a/crates/perry-codegen/tests/native_proof_regressions.rs +++ b/crates/perry-codegen/tests/native_proof_regressions.rs @@ -7425,6 +7425,7 @@ fn typed_f64_clone_test_module(use_any_param: bool) -> Module { exported_functions: Vec::new(), script_global_functions: Vec::new(), references_global_this: false, + annexb_global_undefined_names: Vec::new(), widgets: Vec::new(), uses_fetch: false, uses_webassembly: false, @@ -7596,6 +7597,7 @@ fn typed_i1_clone_test_module_named(name: &str) -> Module { exported_functions: Vec::new(), script_global_functions: Vec::new(), references_global_this: false, + annexb_global_undefined_names: Vec::new(), widgets: Vec::new(), uses_fetch: false, uses_webassembly: false, @@ -7687,6 +7689,7 @@ fn typed_string_clone_test_module(case: &str) -> Module { exported_functions: Vec::new(), script_global_functions: Vec::new(), references_global_this: false, + annexb_global_undefined_names: Vec::new(), widgets: Vec::new(), uses_fetch: false, uses_webassembly: false, @@ -7800,6 +7803,7 @@ fn typed_i1_numeric_predicate_module() -> Module { exported_functions: Vec::new(), script_global_functions: Vec::new(), references_global_this: false, + annexb_global_undefined_names: Vec::new(), widgets: Vec::new(), uses_fetch: false, uses_webassembly: false, @@ -7876,6 +7880,7 @@ fn typed_i1_i32_predicate_module() -> Module { exported_functions: Vec::new(), script_global_functions: Vec::new(), references_global_this: false, + annexb_global_undefined_names: Vec::new(), widgets: Vec::new(), uses_fetch: false, uses_webassembly: false, @@ -8001,6 +8006,7 @@ fn typed_i32_return_module(case: &str) -> Module { exported_functions: Vec::new(), script_global_functions: Vec::new(), references_global_this: false, + annexb_global_undefined_names: Vec::new(), widgets: Vec::new(), uses_fetch: false, uses_webassembly: false, diff --git a/crates/perry-runtime/src/array/alloc.rs b/crates/perry-runtime/src/array/alloc.rs index b595ea9f5..112d31f1b 100644 --- a/crates/perry-runtime/src/array/alloc.rs +++ b/crates/perry-runtime/src/array/alloc.rs @@ -108,7 +108,17 @@ pub extern "C" fn js_array_alloc_with_length(capacity: u32) -> *mut ArrayHeader pub extern "C" fn js_array_constructor_single(value: f64) -> *mut ArrayHeader { if let Some(number) = value_bits_to_number(value.to_bits()) { let length = array_length_from_number_or_throw(number); - return js_array_alloc_with_length(length); + let arr = js_array_alloc_with_length(length); + if length > 0 { + // #6011: user-facing `new Array(n)` — every slot is TAG_HOLE, so + // the raw-f64-or-holes invariant holds by construction and the + // packed-f64 range-loop guard can skip its verify walk. This is + // the ONLY `js_array_alloc_with_length` caller that may mark: + // internal callers (shape keys arrays, sort scratch, …) + // direct-write slots without the layout-noting store helpers. + unsafe { mark_array_raw_f64_holes_fresh(arr) }; + } + return arr; } let scope = crate::gc::RuntimeHandleScope::new(); diff --git a/crates/perry-runtime/src/array/header.rs b/crates/perry-runtime/src/array/header.rs index 804100d4a..b4347ebf2 100644 --- a/crates/perry-runtime/src/array/header.rs +++ b/crates/perry-runtime/src/array/header.rs @@ -790,7 +790,12 @@ pub(crate) unsafe fn canonicalize_array_numeric_store_bits( arr: *mut ArrayHeader, value_bits: u64, ) -> u64 { - if array_numeric_layout(arr) == Some(NumericArrayLayout::RawF64) { + // #6011: the raw-f64-or-holes invariant needs numeric stores canonicalized + // exactly like the dense layout — an INT32-boxed store left verbatim would + // read as NaN payload through the guard's raw-f64 loads. + if array_numeric_layout(arr) == Some(NumericArrayLayout::RawF64) + || array_has_raw_f64_holes_flag(arr) + { if let Some(number) = value_bits_to_number(value_bits) { return number.to_bits(); } @@ -858,14 +863,43 @@ unsafe fn set_array_raw_f64_layout_flag(arr: *const ArrayHeader) { #[inline] unsafe fn clear_array_raw_f64_layout_flag(arr: *const ArrayHeader) { if let Some(header) = array_gc_header(arr) { - let had_raw_layout = (*header)._reserved & crate::gc::GC_ARRAY_RAW_F64_LAYOUT != 0; - (*header)._reserved &= !crate::gc::GC_ARRAY_RAW_F64_LAYOUT; + let raw_bits = crate::gc::GC_ARRAY_RAW_F64_LAYOUT | crate::gc::GC_ARRAY_RAW_F64_HOLES; + let had_raw_layout = (*header)._reserved & raw_bits != 0; + (*header)._reserved &= !raw_bits; if had_raw_layout { crate::typed_feedback::invalidate_representation_change(arr as usize); } } } +/// #6011: hole-tolerant sibling of the dense raw-f64 flag — see +/// [`crate::gc::GC_ARRAY_RAW_F64_HOLES`]. Queried by the packed-f64 +/// range-loop guard's verify pass; cleared alongside the dense flag by the +/// same `clear_array_numeric_layout` choke point. +#[inline] +unsafe fn array_has_raw_f64_holes_flag(arr: *const ArrayHeader) -> bool { + array_gc_header(arr) + .is_some_and(|header| (*header)._reserved & crate::gc::GC_ARRAY_RAW_F64_HOLES != 0) +} + +#[inline] +unsafe fn set_array_raw_f64_holes_flag(arr: *const ArrayHeader) { + if let Some(header) = array_gc_header(arr) { + (*header)._reserved |= crate::gc::GC_ARRAY_RAW_F64_HOLES; + } +} + +/// #6011: mark a freshly hole-initialized user-facing array (`new Array(n)`): +/// every slot in `[0, length)` is `TAG_HOLE`, so the raw-f64-or-holes +/// invariant holds by construction. Callers must guarantee no slot has been +/// written since the hole fill. Internal scratch arrays that direct-write +/// slots afterwards (shape keys arrays, sort temporaries, …) must NOT be +/// marked — they bypass the layout-noting store helpers by contract. +#[inline] +pub(crate) unsafe fn mark_array_raw_f64_holes_fresh(arr: *const ArrayHeader) { + set_array_raw_f64_holes_flag(arr); +} + pub(crate) unsafe fn mark_array_as_arguments_object(arr: *const ArrayHeader) { if let Some(header) = array_gc_header(arr) { (*header)._reserved |= crate::gc::GC_ARRAY_ARGUMENTS_OBJECT; @@ -899,12 +933,26 @@ unsafe fn rebuild_array_numeric_raw_f64(arr: *mut ArrayHeader) -> bool { let elements = array_elements_ptr(arr); for i in 0..length { let slot_bits = array_slot_bits(arr, i); + if slot_bits == crate::value::TAG_HOLE { + // #6011: a hole disproves the DENSE raw-f64 layout, but leaves + // the raw-f64-or-holes invariant intact — the dense flag cannot + // be set here (dense-flagged arrays have no holes), so there is + // nothing to clear, and clearing would wrongly drop the holes + // flag a fresh `new Array(n)` carries. + return false; + } let Some(number) = value_bits_to_number(slot_bits) else { clear_array_numeric_layout(arr); return false; }; - // GC_STORE_AUDIT(POINTER_FREE): raw-f64 layout rewrite stores numeric payloads only. - std::ptr::write(elements.add(i) as *mut f64, number); + // #6011: skip the write-back when the slot already holds the raw-f64 + // bits (the overwhelmingly common case when a packed loop produced + // the values). Halves the memory traffic of the first layout probe + // of a large freshly-filled array. + if number.to_bits() != slot_bits { + // GC_STORE_AUDIT(POINTER_FREE): raw-f64 layout rewrite stores numeric payloads only. + std::ptr::write(elements.add(i) as *mut f64, number); + } } set_array_raw_f64_layout_flag(arr); @@ -912,6 +960,74 @@ unsafe fn rebuild_array_numeric_raw_f64(arr: *mut ArrayHeader) -> bool { true } +/// #6011: hole-tolerant variant of [`rebuild_array_numeric_raw_f64`] for the +/// packed-f64 *range* loop guard. Every non-hole slot is rewritten to a raw +/// f64 (INT32-boxed integers included); `TAG_HOLE` slots are left in place — +/// they never contain a heap pointer, and the guarded loop's inline loads +/// hole-check each slot before use. A slot that is neither numeric nor a hole +/// (string / object / bool / …) clears the layout flag and fails the guard. +/// +/// When no holes were seen the array is uniformly numeric, so the RawF64 +/// layout flag is set exactly like the strict rebuild. When holes remain the +/// dense flag stays clear but `GC_ARRAY_RAW_F64_HOLES` records the verified +/// raw-f64-or-holes invariant (also pre-set by `new Array(n)` allocation), so +/// later guard entries — and the guard on a fresh hole-filled array — skip +/// the walk entirely. Either way the array is pointer-free and the GC layout +/// note reflects it. +pub(crate) unsafe fn rebuild_array_numeric_raw_f64_allow_holes(arr: *mut ArrayHeader) -> bool { + if arr.is_null() { + return false; + } + let length = (*arr).length as usize; + let capacity = (*arr).capacity as usize; + if length > capacity || length > 16_000_000 { + clear_array_numeric_layout(arr); + return false; + } + if array_has_raw_f64_layout_flag(arr) { + // Already proven dense raw-f64 numeric — nothing to rewrite. + return true; + } + if array_has_raw_f64_holes_flag(arr) { + // #6011: invariant flag — every slot in `[0, length)` is already raw + // f64 bits or TAG_HOLE (established at `new Array(n)` allocation or by + // a previous verify walk; every non-numeric store clears the flag via + // `clear_array_numeric_layout`, and numeric stores canonicalize to raw + // bits via `canonicalize_array_numeric_store_bits`). Nothing to verify + // or rewrite — this turns the per-loop-entry guard walk over a fresh + // 100k-slot EMA output array into an O(1) flag test. + return true; + } + + let elements = array_elements_ptr(arr); + let mut saw_hole = false; + for i in 0..length { + let slot_bits = array_slot_bits(arr, i); + if slot_bits == crate::value::TAG_HOLE { + saw_hole = true; + continue; + } + let Some(number) = value_bits_to_number(slot_bits) else { + clear_array_numeric_layout(arr); + return false; + }; + if number.to_bits() != slot_bits { + // GC_STORE_AUDIT(POINTER_FREE): raw-f64 layout rewrite stores numeric payloads only. + std::ptr::write(elements.add(i) as *mut f64, number); + } + } + + if !saw_hole { + set_array_raw_f64_layout_flag(arr); + } else { + // Holes remain, but every non-hole slot is now canonical raw f64 — + // record the verified invariant so re-entering the loop skips the walk. + set_array_raw_f64_holes_flag(arr); + } + crate::gc::layout_init_pointer_free(arr as *mut u8); + true +} + #[inline] pub(crate) unsafe fn set_array_numeric_layout(arr: *mut ArrayHeader, layout: NumericArrayLayout) { if arr.is_null() { @@ -949,6 +1065,11 @@ pub(crate) fn transfer_array_numeric_layout(old_user: usize, new_user: usize) { unsafe { if array_has_raw_f64_layout_flag(old_user as *const ArrayHeader) { set_array_raw_f64_layout_flag(new_user as *const ArrayHeader); + } else if array_has_raw_f64_holes_flag(old_user as *const ArrayHeader) { + // #6011: relocation copies slot bits verbatim, so the verified + // raw-f64-or-holes invariant carries over to the new backing. + clear_array_raw_f64_layout_flag(new_user as *const ArrayHeader); + set_array_raw_f64_holes_flag(new_user as *const ArrayHeader); } else { clear_array_raw_f64_layout_flag(new_user as *const ArrayHeader); } @@ -1139,11 +1260,32 @@ pub extern "C" fn js_array_is_numeric_f64_layout(arr: *const ArrayHeader) -> i32 if array_numeric_layout(arr) == Some(NumericArrayLayout::RawF64) { return 1; } - if array_slots_are_numeric(arr) { - rebuild_array_numeric_raw_f64(arr as *mut ArrayHeader); + // #6011 follow-up: a holes-flagged array (`new Array(n)` mid-fill) + // cannot be DENSE raw-f64, and answering that per store via the + // verify walk below made a sequential fill loop O(N²) — the walk + // reads the already-filled prefix on every store before hitting the + // next hole (the fmr benchmark hang). While the LAST slot is still a + // hole the array is provably not dense: answer 0 in O(1). Once the + // last slot is written (a completed sequential fill), fall through so + // the single walk below can upgrade the array to the dense flag. + if array_has_raw_f64_holes_flag(arr) { + let length = (*arr).length as usize; + if length > 0 && array_slot_bits(arr, length - 1) == crate::value::TAG_HOLE { + return 0; + } + } + // #6011: single combined verify+rewrite pass. The old shape + // (`array_slots_are_numeric` scan, THEN `rebuild_array_numeric_raw_ + // f64` rewrite) walked every slot twice on the first numeric-layout + // probe of a large array — 2×100k slot reads per call for an + // EMA-style `new Array(100_000)` filled by a loop. The rebuild + // already fails cleanly on the first non-numeric slot (clearing the + // layout flag); slots converted before such a failure were genuinely + // numeric, and INT32-box → raw-f64 canonicalization is value- + // preserving for every reader, so stopping mid-way is unobservable. + if rebuild_array_numeric_raw_f64(arr as *mut ArrayHeader) { return 1; } - clear_array_numeric_layout(arr); } 0 } diff --git a/crates/perry-runtime/src/array/mod.rs b/crates/perry-runtime/src/array/mod.rs index 8d53dfdd7..b1e4511de 100644 --- a/crates/perry-runtime/src/array/mod.rs +++ b/crates/perry-runtime/src/array/mod.rs @@ -57,7 +57,10 @@ pub(crate) use self::generic::{ pub use self::generic_mutators::{ js_arraylike_pop, js_arraylike_push, js_arraylike_shift, js_arraylike_unshift, }; -pub(crate) use self::header::{array_has_arguments_object_flag, mark_array_as_arguments_object}; +pub(crate) use self::header::{ + array_has_arguments_object_flag, mark_array_as_arguments_object, + rebuild_array_numeric_raw_f64_allow_holes, +}; pub use self::header::{ js_array_clear_numeric_layout, js_array_is_numeric_f64_layout, js_array_mark_arguments_object, js_array_mark_numeric_f64_layout, js_array_note_numeric_write, js_tagged_template_get_or_init, @@ -143,8 +146,8 @@ pub(crate) use self::header::{ array_numeric_raw_f64_push_inbounds, array_numeric_raw_f64_set_inbounds, array_object_flags, array_ptr_as_proxy, canonicalize_array_numeric_store_value, clean_arr_ptr, clean_arr_ptr_mut, clear_array_numeric_layout, clear_array_numeric_layout_ptr, gc_element_slot_range, - mark_array_layout_unknown, normalize_array_receiver, note_array_slot, - note_array_slot_layout_only, rebuild_array_layout, rebuild_array_layout_exact, + mark_array_layout_unknown, mark_array_raw_f64_holes_fresh, normalize_array_receiver, + note_array_slot, note_array_slot_layout_only, rebuild_array_layout, rebuild_array_layout_exact, refresh_array_numeric_layout, replay_array_growth_write_barriers, set_array_numeric_layout, store_array_slot, transfer_array_numeric_layout, value_bits_to_number, NumericArrayLayout, MIN_ARRAY_CAPACITY, diff --git a/crates/perry-runtime/src/array/tests.rs b/crates/perry-runtime/src/array/tests.rs index abf59aa21..eaaa64235 100644 --- a/crates/perry-runtime/src/array/tests.rs +++ b/crates/perry-runtime/src/array/tests.rs @@ -584,6 +584,84 @@ fn test_numeric_array_layout_mark_rejects_holes_and_accepts_dense_numbers() { ); } +// #6011: the raw-f64-or-holes invariant flag set by `new Array(n)` — the +// packed-f64 range-loop guard's walk-free fast path — must be maintained by +// the canonicalizing store helpers and downgraded by any non-numeric store. +#[test] +fn test_new_array_holes_flag_walk_free_guard_and_sound_downgrade() { + unsafe { + // `new Array(4)`: every slot TAG_HOLE, invariant marked at allocation. + let arr = js_array_constructor_single(4.0); + assert_eq!(js_array_length(arr), 4); + assert!( + rebuild_array_numeric_raw_f64_allow_holes(arr), + "fresh `new Array(n)` passes the hole-tolerant verify" + ); + assert_eq!( + js_array_is_numeric_f64_layout(arr), + 0, + "holes invariant must NOT satisfy the strict dense probe" + ); + + // Numeric stores keep the invariant and are canonicalized to raw f64 + // bits even though the dense RawF64 flag is not set: a verbatim + // INT32-boxed slot would read as NaN payload through the guarded + // loop's raw-f64 loads. + let int32_value = f64::from_bits(crate::value::INT32_TAG | 7u64); + js_array_set_f64(arr, 0, int32_value); + let elements = (arr as *const u8).add(std::mem::size_of::()) as *const u64; + assert_eq!( + *elements, // slot 0 + 7.0f64.to_bits(), + "numeric store canonicalizes to raw f64 bits under the holes flag" + ); + assert!(rebuild_array_numeric_raw_f64_allow_holes(arr)); + + // A non-numeric store must drop the invariant so the guard re-walks + // (and fails) instead of treating string bits as raw f64. + let s = crate::string::js_string_from_bytes(b"x".as_ptr(), 1); + let s_value = + f64::from_bits(crate::value::STRING_TAG | (s as u64 & crate::value::POINTER_MASK)); + js_array_set_f64(arr, 1, s_value); + assert!( + !rebuild_array_numeric_raw_f64_allow_holes(arr), + "non-numeric store downgrades the holes invariant" + ); + } +} + +// #6011: an unmarked hole-carrying array (internal alloc path) is verified by +// a walk; the walk records the invariant so re-entry is walk-free, and the +// strict dense probe must not clear that recorded invariant when it declines +// on a hole. +#[test] +fn test_holes_invariant_recorded_by_verify_walk_survives_strict_probe() { + unsafe { + let arr = js_array_alloc_with_length(3); + js_array_set_f64(arr, 0, 1.5); + assert!( + rebuild_array_numeric_raw_f64_allow_holes(arr), + "hole-tolerant verify accepts numeric + hole slots" + ); + assert_eq!( + js_array_is_numeric_f64_layout(arr), + 0, + "strict probe still declines on the remaining holes" + ); + assert!( + rebuild_array_numeric_raw_f64_allow_holes(arr), + "strict-probe decline must not clear the recorded holes invariant" + ); + + let undefined = f64::from_bits(crate::value::TAG_UNDEFINED); + js_array_set_f64(arr, 2, undefined); + assert!( + !rebuild_array_numeric_raw_f64_allow_holes(arr), + "undefined is not a hole: it must fail the hole-tolerant verify" + ); + } +} + #[test] fn test_numeric_array_mark_canonicalizes_int32_and_nan_inline() { let arr = js_array_alloc_with_length(3); diff --git a/crates/perry-runtime/src/gc/barrier.rs b/crates/perry-runtime/src/gc/barrier.rs index 2c3899a31..58db8defb 100644 --- a/crates/perry-runtime/src/gc/barrier.rs +++ b/crates/perry-runtime/src/gc/barrier.rs @@ -984,15 +984,21 @@ pub(super) fn write_barrier_slot_inner( child: u64, external_slot: bool, ) { - bump_write_barrier_trace_counter(BarrierTraceCounter::Calls); - incremental_mark_barrier_value(child); - - // Decode child first: primitive stores are the most common skip. + // Decode child first: primitive stores are the overwhelmingly common + // case (every numeric array/field store) and need NEITHER the + // incremental-mark probe (nothing to mark) NOR the remembered set (no + // old→young edge) — so they must not pay the incremental barrier's + // unconditional thread-local access, which dominated tight numeric store + // loops (#6011: `ema[i] = ` spent more time in this preamble than + // in the store itself). let child_addr = decode_heap_addr(child); if child_addr == 0 { + bump_write_barrier_trace_counter(BarrierTraceCounter::Calls); bump_write_barrier_trace_counter(BarrierTraceCounter::NonPointerChildSkips); return; } + bump_write_barrier_trace_counter(BarrierTraceCounter::Calls); + incremental_mark_barrier_value(child); // Decode the parent — must be a NaN-boxed heap pointer. let parent_addr = decode_heap_addr(parent); if parent_addr == 0 { @@ -1065,10 +1071,17 @@ pub(super) fn decode_heap_addr(bits: u64) -> usize { if tag == POINTER_TAG || tag == STRING_TAG || tag == BIGINT_TAG { (bits & POINTER_MASK) as usize } else if tag < 0x7FF8_0000_0000_0000 { - // Possible raw pointer. Accept only if the arena side metadata - // recognizes it as a heap address; ordinary f64 payload bits - // miss the metadata table and remain non-pointers. + // Possible raw pointer. Cheap shape pre-filter first (#6011): a real + // heap address is 48-bit, above the handle band, and 8-aligned — an + // ordinary f64 payload (e.g. 100.5 = 0x4059_4000_…) has non-zero + // high bits and is rejected here without paying the page-map + // classification, which dominated tight numeric store loops. Only + // the (rare) subnormal doubles whose bits look address-shaped fall + // through to the authoritative arena lookup. let addr = bits as usize; + if (bits >> 48) != 0 || addr < 0x10000 || addr & 0x7 != 0 { + return 0; + } if matches!( crate::arena::classify_heap_generation(addr), crate::arena::HeapGeneration::Unknown diff --git a/crates/perry-runtime/src/gc/types.rs b/crates/perry-runtime/src/gc/types.rs index fe886cda4..987ab929e 100644 --- a/crates/perry-runtime/src/gc/types.rs +++ b/crates/perry-runtime/src/gc/types.rs @@ -865,6 +865,16 @@ pub(crate) const GC_ARRAY_RAW_F64_LAYOUT: u16 = 0x80; /// meaningful for `GC_TYPE_ARRAY`; it lets `util.types.isArgumentsObject` /// distinguish Perry's internal `arguments` arrays from user rest arrays. pub(crate) const GC_ARRAY_ARGUMENTS_OBJECT: u16 = 0x200; +/// #6011: every element slot in `[0, length)` holds either canonical raw-f64 +/// number bits or `TAG_HOLE` — the hole-tolerant sibling of +/// `GC_ARRAY_RAW_F64_LAYOUT`. Set when `new Array(n)` hole-initializes a +/// user-facing array (and by the range-loop guard after a hole-seeing verify +/// pass); cleared alongside the dense flag by `clear_array_numeric_layout`, +/// which every non-numeric store path already funnels through. Lets the +/// packed-f64 range-loop guard skip its whole-array verify walk. Bit 12 — +/// bits 12..13 were the last free `_reserved` bits (see the bit map above). +/// Only meaningful for `GC_TYPE_ARRAY`. +pub(crate) const GC_ARRAY_RAW_F64_HOLES: u16 = 0x1000; pub(super) const POINTER_TAG: u64 = 0x7FFD_0000_0000_0000; pub(super) const STRING_TAG: u64 = 0x7FFF_0000_0000_0000; diff --git a/crates/perry-runtime/src/typed_feedback.rs b/crates/perry-runtime/src/typed_feedback.rs index 988c95cec..c447bca1d 100644 --- a/crates/perry-runtime/src/typed_feedback.rs +++ b/crates/perry-runtime/src/typed_feedback.rs @@ -1190,6 +1190,47 @@ fn packed_f64_array_loop_guard(arr: *const ArrayHeader) -> bool { crate::array::js_array_is_numeric_f64_layout(raw_addr as *const ArrayHeader) != 0 } +/// #6011: entry guard for the packed-f64 *range* versioned loop. Validates the +/// same plain-array shape as [`packed_f64_array_loop_guard`], then proves the +/// whole static index range `[min_idx, max_idx_exclusive)` the loop can touch +/// is inside the array, and finally normalizes the slots hole-tolerantly: +/// numeric slots are rewritten to raw f64 while `TAG_HOLE` slots stay in place +/// (the guarded loop's inline loads hole-check and side-exit to the slow loop). +/// Any slot that is neither numeric nor a hole fails the guard. +fn packed_f64_array_loop_range_guard( + arr: *const ArrayHeader, + min_idx: i32, + max_idx_exclusive: i32, +) -> bool { + if !plain_array_index_guard(arr, 0, false) { + return false; + } + let raw_addr = normalize_raw_object_addr(arr as u64); + let Some(header) = gc_header_for_user_addr(raw_addr) else { + return false; + }; + unsafe { + let flags = (*header)._reserved; + if flags + & (crate::gc::OBJ_FLAG_FROZEN + | crate::gc::OBJ_FLAG_SEALED + | crate::gc::OBJ_FLAG_NO_EXTEND) + != 0 + { + return false; + } + let arr = raw_addr as *mut ArrayHeader; + let len = (*arr).length; + if len > i32::MAX as u32 { + return false; + } + if min_idx < 0 || i64::from(max_idx_exclusive) > i64::from(len) { + return false; + } + crate::array::rebuild_array_numeric_raw_f64_allow_holes(arr) + } +} + fn packed_i32_array_loop_guard(arr: *const ArrayHeader) -> bool { if !packed_f64_array_loop_guard(arr) { return false; @@ -1532,6 +1573,61 @@ pub extern "C" fn js_typed_feedback_packed_f64_array_loop_guard( } } +/// #6011: FFI wrapper for the packed-f64 range-loop guard. `min_idx` / +/// `max_idx_exclusive` are the smallest and one-past-largest indices the +/// candidate fast loop can touch (loop start + smallest offset, loop bound + +/// largest offset), both precomputed as i32 by codegen. +#[no_mangle] +pub extern "C" fn js_typed_feedback_packed_f64_range_loop_guard( + site_id: u64, + receiver: f64, + min_idx: i32, + max_idx_exclusive: i32, +) -> i32 { + let raw_addr = normalize_raw_object_addr(receiver.to_bits()); + if !typed_feedback_enabled() { + return packed_f64_array_loop_range_guard( + raw_addr as *const ArrayHeader, + min_idx, + max_idx_exclusive, + ) as i32; + } + let (class_id, heap_type, aux, element_kind) = classify_array(raw_addr, None); + let observation = Observation { + source: ObservationSource::Array, + object_addr: 0, + shape_addr: 0, + key_hash: 0, + class_id, + heap_type, + aux, + value_tag: element_kind, + }; + let pass = guard_observe( + site_id, + TypedFeedbackSiteKind::ArrayElement, + observation, + packed_f64_array_loop_range_guard( + raw_addr as *const ArrayHeader, + min_idx, + max_idx_exclusive, + ), + ); + if pass { + 1 + } else { + 0 + } +} + +#[used] +static KEEP_JS_TYPED_FEEDBACK_PACKED_F64_RANGE_LOOP_GUARD: extern "C" fn( + u64, + f64, + i32, + i32, +) -> i32 = js_typed_feedback_packed_f64_range_loop_guard; + #[no_mangle] pub extern "C" fn js_typed_feedback_packed_i32_array_loop_guard( site_id: u64, diff --git a/crates/perry-runtime/src/typed_feedback/tests.rs b/crates/perry-runtime/src/typed_feedback/tests.rs index 2931fada5..69439e485 100644 --- a/crates/perry-runtime/src/typed_feedback/tests.rs +++ b/crates/perry-runtime/src/typed_feedback/tests.rs @@ -1094,6 +1094,14 @@ fn typed_feedback_array_loop_helpers_have_lto_keepalive_anchors() { "static KEEP_JS_TYPED_FEEDBACK_PACKED_U32_ARRAY_LOOP_GUARD: extern \"C\" fn(u64, f64) -> i32", "js_typed_feedback_packed_u32_array_loop_guard", ); + // #6011: hole-tolerant range guard (rustfmt wraps the fn-pointer type, so + // anchor on the static declaration prefix only). + assert_lto_keepalive_anchor( + typed_feedback, + "KEEP_JS_TYPED_FEEDBACK_PACKED_F64_RANGE_LOOP_GUARD", + "static KEEP_JS_TYPED_FEEDBACK_PACKED_F64_RANGE_LOOP_GUARD: extern \"C\" fn(", + "js_typed_feedback_packed_f64_range_loop_guard", + ); } #[test]