From 881d14a53843df10cdcdb1e2193dca27d25cfd78 Mon Sep 17 00:00:00 2001 From: karolay Date: Thu, 22 Oct 2026 19:58:04 -0500 Subject: [PATCH 1/6] [FIX][LLVM] Handle non-unit-stride masked loads --- src/codegen/llvm/codegen_llvm.cc | 31 ++++++++++++++++++++++--- tests/python/codegen/llvm/test_basic.py | 21 +++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/codegen/llvm/codegen_llvm.cc b/src/codegen/llvm/codegen_llvm.cc index 1b579bbcf..547bfec10 100644 --- a/src/codegen/llvm/codegen_llvm.cc +++ b/src/codegen/llvm/codegen_llvm.cc @@ -1873,6 +1873,12 @@ void CodeGenLLVM::BufferAccessHelper( llvm::Value* predicate_value = nullptr; if (predicate.has_value()) { predicate_value = MakeValue(predicate.value()); + + PrimType predicate_dtype(predicate.value().ty()->dtype); + int predicate_lanes = predicate_dtype.lanes(); + if (last_index_lanes > 1 && predicate_lanes == last_index_lanes) { + predicate_value = builder_->CreateExtractElement(predicate_value, i); + } } TypedPointer buffer_ptr = @@ -1948,9 +1954,28 @@ llvm::Value* CodeGenLLVM::CreateMaskedLoad(const CallObj* op) { int alignment, bool is_volatile) { TVM_FFI_ICHECK(!is_volatile) << "The masked load intrinsic does not support declaring load as volatile."; - llvm::Instruction* value = builder_->CreateMaskedLoad(buffer_ptr.type, buffer_ptr.addr, - llvm::Align(alignment), predicate); - loads.push_back(value); + + if (predicate->getType()->isVectorTy()) { + llvm::Instruction* value = builder_->CreateMaskedLoad(buffer_ptr.type, buffer_ptr.addr, + llvm::Align(alignment), predicate); + loads.push_back(value); + return value; + } + + llvm::Type* vector_type = llvm::FixedVectorType::get(buffer_ptr.type, 1); + llvm::Type* mask_vector_type = llvm::FixedVectorType::get(predicate->getType(), 1); + + llvm::Value* mask = llvm::UndefValue::get(mask_vector_type); + mask = builder_->CreateInsertElement(mask, predicate, ConstInt32(0)); + + llvm::Instruction* value = + builder_->CreateMaskedLoad(vector_type, buffer_ptr.addr, llvm::Align(alignment), mask, + llvm::UndefValue::get(vector_type)); + + llvm::Value* scalar = builder_->CreateExtractElement(value, ConstInt32(0)); + loads.push_back(scalar); + + // Return the memory-access instruction so AddAliasInfo attaches TBAA to it. return value; }; BufferAccessHelper(buffer, indices, predicate, access_dtype, make_load); diff --git a/tests/python/codegen/llvm/test_basic.py b/tests/python/codegen/llvm/test_basic.py index 357bc63e6..6163ca6bd 100644 --- a/tests/python/codegen/llvm/test_basic.py +++ b/tests/python/codegen/llvm/test_basic.py @@ -1170,6 +1170,27 @@ def main(): tvm_tirx.compile(Module, target="llvm") +@pytest.mark.skipif(not env.has_llvm(), reason="need llvm") +def test_masked_load_non_unit_stride(): + @I.ir_module + class Module: + @T.function + def main( + A: T.Tensor((3,), "float32"), + B: T.Tensor((2,), "float32"), + ): + B[T.Ramp(0, 1, 2)] = T.call_intrin( + "tirx.masked_load", + A, + T.Ramp(0, 2, 2), + T.Broadcast(T.bool(True), 2), + ty="float32x2", + ) + + with tvm_tirx.target.Target("llvm"): + tvm_tirx.tirx.compile(Module) + + def test_invalid_volatile_masked_buffer_load(): @I.ir_module class Module: From df0eef8bdf6c65a5321f0ef5a430a0706c83ddf8 Mon Sep 17 00:00:00 2001 From: Nayeli Date: Wed, 28 Oct 2026 12:04:56 -0500 Subject: [PATCH 2/6] [FIX][LLVM] Avoid lanes() on scalable predicates in masked access --- src/codegen/llvm/codegen_llvm.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/codegen/llvm/codegen_llvm.cc b/src/codegen/llvm/codegen_llvm.cc index 547bfec10..a2cd0fe0f 100644 --- a/src/codegen/llvm/codegen_llvm.cc +++ b/src/codegen/llvm/codegen_llvm.cc @@ -1874,9 +1874,10 @@ void CodeGenLLVM::BufferAccessHelper( if (predicate.has_value()) { predicate_value = MakeValue(predicate.value()); + // lanes() throws for scalable vectors, so check IsScalableVector() first. PrimType predicate_dtype(predicate.value().ty()->dtype); - int predicate_lanes = predicate_dtype.lanes(); - if (last_index_lanes > 1 && predicate_lanes == last_index_lanes) { + if (last_index_lanes > 1 && !predicate_dtype.IsScalableVector() && + predicate_dtype.lanes() == last_index_lanes) { predicate_value = builder_->CreateExtractElement(predicate_value, i); } } From 7334db15cab2b9dd1ede271f969a3a616e7b629a Mon Sep 17 00:00:00 2001 From: Nayeli Date: Wed, 28 Oct 2026 12:05:04 -0500 Subject: [PATCH 3/6] [FIX][LLVM] Handle non-unit-stride masked stores --- src/codegen/llvm/codegen_llvm.cc | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/codegen/llvm/codegen_llvm.cc b/src/codegen/llvm/codegen_llvm.cc index a2cd0fe0f..2b19b380c 100644 --- a/src/codegen/llvm/codegen_llvm.cc +++ b/src/codegen/llvm/codegen_llvm.cc @@ -2012,8 +2012,24 @@ llvm::Value* CodeGenLLVM::CreateMaskedStore(const CallObj* op) { << "The masked store intrinsic does not support declaring store as volatile."; llvm::Value* to_store = subelement_i == -1 ? value : builder_->CreateExtractElement(value, subelement_i); - last_store = builder_->CreateMaskedStore(to_store, buffer_ptr.addr, llvm::Align(alignment), - predicate); + + if (predicate->getType()->isVectorTy()) { + last_store = builder_->CreateMaskedStore(to_store, buffer_ptr.addr, + llvm::Align(alignment), predicate); + return last_store; + } + + // llvm.masked.store only accepts vectors: store a scalar lane as a 1-element vector. + llvm::Type* vector_type = llvm::FixedVectorType::get(to_store->getType(), 1); + llvm::Type* mask_vector_type = llvm::FixedVectorType::get(predicate->getType(), 1); + + llvm::Value* vector_value = builder_->CreateInsertElement( + llvm::UndefValue::get(vector_type), to_store, ConstInt32(0)); + llvm::Value* mask = builder_->CreateInsertElement(llvm::UndefValue::get(mask_vector_type), + predicate, ConstInt32(0)); + + last_store = builder_->CreateMaskedStore(vector_value, buffer_ptr.addr, + llvm::Align(alignment), mask); return last_store; }; BufferAccessHelper(buffer, indices, predicate, value_dtype, make_store); From 75c9d7d130f98e5b9631c5f6947312329bd385a2 Mon Sep 17 00:00:00 2001 From: Nayeli Date: Thu, 29 Oct 2026 09:26:55 -0500 Subject: [PATCH 4/6] [TEST][LLVM] Cover non-unit-stride masked load and store --- tests/python/codegen/llvm/test_basic.py | 53 ++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/tests/python/codegen/llvm/test_basic.py b/tests/python/codegen/llvm/test_basic.py index 6163ca6bd..12a835ae2 100644 --- a/tests/python/codegen/llvm/test_basic.py +++ b/tests/python/codegen/llvm/test_basic.py @@ -1187,8 +1187,57 @@ def main( ty="float32x2", ) - with tvm_tirx.target.Target("llvm"): - tvm_tirx.tirx.compile(Module) + tvm_tirx.compile(Module, target="llvm") + + +@pytest.mark.skipif(not env.has_llvm(), reason="need llvm") +@pytest.mark.parametrize("n", [0, 1, 2]) +def test_masked_load_non_unit_stride_values(n): + @I.ir_module + class Module: + @T.function + def main(A: T.Tensor((3,), "float32"), B: T.Tensor((2,), "float32"), n: T.int32): + B[T.Ramp(0, 1, 2)] = T.call_intrin( + "tirx.masked_load", + A, + T.Ramp(0, 2, 2), + T.Ramp(0, 1, 2) < T.Broadcast(n, 2), + ty="float32x2", + ) + + f = tvm_tirx.compile(Module, target="llvm") + a = tvm_tirx.runtime.tensor(np.array([1.0, 2.0, 3.0], dtype="float32")) + b = tvm_tirx.runtime.tensor(np.full(2, -1.0, dtype="float32")) + f(a, b, n) + # Inactive lanes are poison, so only the active ones are checked. + np.testing.assert_array_equal(b.numpy()[:n], [1.0, 3.0][:n]) + + +@pytest.mark.skipif(not env.has_llvm(), reason="need llvm") +@pytest.mark.parametrize( + "n, expected", [(0, [-1.0, -1.0, -1.0]), (1, [1.0, -1.0, -1.0]), (2, [1.0, -1.0, 2.0])] +) +def test_masked_store_non_unit_stride(n, expected): + @I.ir_module + class Module: + @T.function + def main(A: T.Tensor((2,), "float32"), B: T.Tensor((3,), "float32"), n: T.int32): + T.evaluate( + T.call_intrin( + "tirx.masked_store", + B, + A[T.Ramp(0, 1, 2)], + T.Ramp(0, 2, 2), + T.Ramp(0, 1, 2) < T.Broadcast(n, 2), + ty="void", + ) + ) + + f = tvm_tirx.compile(Module, target="llvm") + a = tvm_tirx.runtime.tensor(np.array([1.0, 2.0], dtype="float32")) + b = tvm_tirx.runtime.tensor(np.full(3, -1.0, dtype="float32")) + f(a, b, n) + np.testing.assert_array_equal(b.numpy(), expected) def test_invalid_volatile_masked_buffer_load(): From 81f400a0ea534fa7363a9be914a785e8034a646d Mon Sep 17 00:00:00 2001 From: karolay Date: Sat, 7 Nov 2026 16:26:40 -0500 Subject: [PATCH 5/6] [TEST][LLVM] Cover second-lane-only non-unit-stride masked access --- tests/python/codegen/llvm/test_basic.py | 40 ++++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/tests/python/codegen/llvm/test_basic.py b/tests/python/codegen/llvm/test_basic.py index 12a835ae2..c61137b9f 100644 --- a/tests/python/codegen/llvm/test_basic.py +++ b/tests/python/codegen/llvm/test_basic.py @@ -1191,44 +1191,62 @@ def main( @pytest.mark.skipif(not env.has_llvm(), reason="need llvm") -@pytest.mark.parametrize("n", [0, 1, 2]) -def test_masked_load_non_unit_stride_values(n): +@pytest.mark.parametrize("start, end", [(0, 0), (0, 1), (1, 2), (0, 2)]) +def test_masked_load_non_unit_stride_values(start, end): @I.ir_module class Module: @T.function - def main(A: T.Tensor((3,), "float32"), B: T.Tensor((2,), "float32"), n: T.int32): + def main( + A: T.Tensor((3,), "float32"), + B: T.Tensor((2,), "float32"), + start: T.int32, + end: T.int32, + ): B[T.Ramp(0, 1, 2)] = T.call_intrin( "tirx.masked_load", A, T.Ramp(0, 2, 2), - T.Ramp(0, 1, 2) < T.Broadcast(n, 2), + (T.Ramp(0, 1, 2) >= T.Broadcast(start, 2)) + & (T.Ramp(0, 1, 2) < T.Broadcast(end, 2)), ty="float32x2", ) f = tvm_tirx.compile(Module, target="llvm") a = tvm_tirx.runtime.tensor(np.array([1.0, 2.0, 3.0], dtype="float32")) b = tvm_tirx.runtime.tensor(np.full(2, -1.0, dtype="float32")) - f(a, b, n) + f(a, b, start, end) # Inactive lanes are poison, so only the active ones are checked. - np.testing.assert_array_equal(b.numpy()[:n], [1.0, 3.0][:n]) + np.testing.assert_array_equal(b.numpy()[start:end], [1.0, 3.0][start:end]) @pytest.mark.skipif(not env.has_llvm(), reason="need llvm") @pytest.mark.parametrize( - "n, expected", [(0, [-1.0, -1.0, -1.0]), (1, [1.0, -1.0, -1.0]), (2, [1.0, -1.0, 2.0])] + "start, end, expected", + [ + (0, 0, [-1.0, -1.0, -1.0]), + (0, 1, [1.0, -1.0, -1.0]), + (1, 2, [-1.0, -1.0, 2.0]), + (0, 2, [1.0, -1.0, 2.0]), + ], ) -def test_masked_store_non_unit_stride(n, expected): +def test_masked_store_non_unit_stride(start, end, expected): @I.ir_module class Module: @T.function - def main(A: T.Tensor((2,), "float32"), B: T.Tensor((3,), "float32"), n: T.int32): + def main( + A: T.Tensor((2,), "float32"), + B: T.Tensor((3,), "float32"), + start: T.int32, + end: T.int32, + ): T.evaluate( T.call_intrin( "tirx.masked_store", B, A[T.Ramp(0, 1, 2)], T.Ramp(0, 2, 2), - T.Ramp(0, 1, 2) < T.Broadcast(n, 2), + (T.Ramp(0, 1, 2) >= T.Broadcast(start, 2)) + & (T.Ramp(0, 1, 2) < T.Broadcast(end, 2)), ty="void", ) ) @@ -1236,7 +1254,7 @@ def main(A: T.Tensor((2,), "float32"), B: T.Tensor((3,), "float32"), n: T.int32) f = tvm_tirx.compile(Module, target="llvm") a = tvm_tirx.runtime.tensor(np.array([1.0, 2.0], dtype="float32")) b = tvm_tirx.runtime.tensor(np.full(3, -1.0, dtype="float32")) - f(a, b, n) + f(a, b, start, end) np.testing.assert_array_equal(b.numpy(), expected) From 399a313afc3f03ceb8befe75b6ce5869964cc92e Mon Sep 17 00:00:00 2001 From: Deyvydt Date: Mon, 9 Nov 2026 20:42:10 -0500 Subject: [PATCH 6/6] [TEST][LLVM] Cover 4-lane non-unit-stride masked access for more dtypes --- tests/python/codegen/llvm/test_basic.py | 57 +++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/python/codegen/llvm/test_basic.py b/tests/python/codegen/llvm/test_basic.py index c61137b9f..26de7e63d 100644 --- a/tests/python/codegen/llvm/test_basic.py +++ b/tests/python/codegen/llvm/test_basic.py @@ -1258,6 +1258,63 @@ def main( np.testing.assert_array_equal(b.numpy(), expected) +@pytest.mark.skipif(not env.has_llvm(), reason="need llvm") +@pytest.mark.parametrize("dtype", ["float32", "int8", "float64"]) +@pytest.mark.parametrize("op", ["load", "store"]) +def test_masked_access_non_unit_stride_last_lane(op, dtype): + # Only lane 3 of 4 is active, so a wrong predicate lane (e.g. i % 2) is caught. + @I.ir_module + class LoadModule: + @T.function + def main( + A: T.Tensor((10,), dtype), + B: T.Tensor((4,), dtype), + lane: T.int32, + ): + B[T.Ramp(0, 1, 4)] = T.call_intrin( + "tirx.masked_load", + A, + T.Ramp(0, 3, 4), + T.Ramp(0, 1, 4) == T.Broadcast(lane, 4), + ty=f"{dtype}x4", + ) + + @I.ir_module + class StoreModule: + @T.function + def main( + A: T.Tensor((4,), dtype), + B: T.Tensor((10,), dtype), + lane: T.int32, + ): + T.evaluate( + T.call_intrin( + "tirx.masked_store", + B, + A[T.Ramp(0, 1, 4)], + T.Ramp(0, 3, 4), + T.Ramp(0, 1, 4) == T.Broadcast(lane, 4), + ty="void", + ) + ) + + if op == "load": + f = tvm_tirx.compile(LoadModule, target="llvm") + a = tvm_tirx.runtime.tensor(np.arange(1, 11).astype(dtype)) + b = tvm_tirx.runtime.tensor(np.full(4, -1, dtype=dtype)) + f(a, b, 3) + # Inactive lanes are poison, so only the active one is checked. + assert b.numpy()[3] == 10 + else: + f = tvm_tirx.compile(StoreModule, target="llvm") + a = tvm_tirx.runtime.tensor(np.arange(1, 5).astype(dtype)) + b = tvm_tirx.runtime.tensor(np.full(10, -1, dtype=dtype)) + f(a, b, 3) + expected = np.full(10, -1, dtype=dtype) + expected[9] = 4 + np.testing.assert_array_equal(b.numpy(), expected) + + def test_invalid_volatile_masked_buffer_load(): @I.ir_module class Module: