diff --git a/src/codegen/llvm/codegen_llvm.cc b/src/codegen/llvm/codegen_llvm.cc index 1b579bbcf..2b19b380c 100644 --- a/src/codegen/llvm/codegen_llvm.cc +++ b/src/codegen/llvm/codegen_llvm.cc @@ -1873,6 +1873,13 @@ void CodeGenLLVM::BufferAccessHelper( llvm::Value* predicate_value = nullptr; if (predicate.has_value()) { predicate_value = MakeValue(predicate.value()); + + // lanes() throws for scalable vectors, so check IsScalableVector() first. + PrimType predicate_dtype(predicate.value().ty()->dtype); + if (last_index_lanes > 1 && !predicate_dtype.IsScalableVector() && + predicate_dtype.lanes() == last_index_lanes) { + predicate_value = builder_->CreateExtractElement(predicate_value, i); + } } TypedPointer buffer_ptr = @@ -1948,9 +1955,28 @@ llvm::Value* CodeGenLLVM::CreateMaskedLoad(const CallObj* op) { int alignment, bool is_volatile) { TVM_FFI_ICHECK(!is_volatile) << "The masked load intrinsic does not support declaring load as volatile."; - llvm::Instruction* value = builder_->CreateMaskedLoad(buffer_ptr.type, buffer_ptr.addr, - llvm::Align(alignment), predicate); - loads.push_back(value); + + if (predicate->getType()->isVectorTy()) { + llvm::Instruction* value = builder_->CreateMaskedLoad(buffer_ptr.type, buffer_ptr.addr, + llvm::Align(alignment), predicate); + loads.push_back(value); + return value; + } + + llvm::Type* vector_type = llvm::FixedVectorType::get(buffer_ptr.type, 1); + llvm::Type* mask_vector_type = llvm::FixedVectorType::get(predicate->getType(), 1); + + llvm::Value* mask = llvm::UndefValue::get(mask_vector_type); + mask = builder_->CreateInsertElement(mask, predicate, ConstInt32(0)); + + llvm::Instruction* value = + builder_->CreateMaskedLoad(vector_type, buffer_ptr.addr, llvm::Align(alignment), mask, + llvm::UndefValue::get(vector_type)); + + llvm::Value* scalar = builder_->CreateExtractElement(value, ConstInt32(0)); + loads.push_back(scalar); + + // Return the memory-access instruction so AddAliasInfo attaches TBAA to it. return value; }; BufferAccessHelper(buffer, indices, predicate, access_dtype, make_load); @@ -1986,8 +2012,24 @@ llvm::Value* CodeGenLLVM::CreateMaskedStore(const CallObj* op) { << "The masked store intrinsic does not support declaring store as volatile."; llvm::Value* to_store = subelement_i == -1 ? value : builder_->CreateExtractElement(value, subelement_i); - last_store = builder_->CreateMaskedStore(to_store, buffer_ptr.addr, llvm::Align(alignment), - predicate); + + if (predicate->getType()->isVectorTy()) { + last_store = builder_->CreateMaskedStore(to_store, buffer_ptr.addr, + llvm::Align(alignment), predicate); + return last_store; + } + + // llvm.masked.store only accepts vectors: store a scalar lane as a 1-element vector. + llvm::Type* vector_type = llvm::FixedVectorType::get(to_store->getType(), 1); + llvm::Type* mask_vector_type = llvm::FixedVectorType::get(predicate->getType(), 1); + + llvm::Value* vector_value = builder_->CreateInsertElement( + llvm::UndefValue::get(vector_type), to_store, ConstInt32(0)); + llvm::Value* mask = builder_->CreateInsertElement(llvm::UndefValue::get(mask_vector_type), + predicate, ConstInt32(0)); + + last_store = builder_->CreateMaskedStore(vector_value, buffer_ptr.addr, + llvm::Align(alignment), mask); return last_store; }; BufferAccessHelper(buffer, indices, predicate, value_dtype, make_store); diff --git a/tests/python/codegen/llvm/test_basic.py b/tests/python/codegen/llvm/test_basic.py index 357bc63e6..26de7e63d 100644 --- a/tests/python/codegen/llvm/test_basic.py +++ b/tests/python/codegen/llvm/test_basic.py @@ -1170,6 +1170,151 @@ def main(): tvm_tirx.compile(Module, target="llvm") +@pytest.mark.skipif(not env.has_llvm(), reason="need llvm") +def test_masked_load_non_unit_stride(): + @I.ir_module + class Module: + @T.function + def main( + A: T.Tensor((3,), "float32"), + B: T.Tensor((2,), "float32"), + ): + B[T.Ramp(0, 1, 2)] = T.call_intrin( + "tirx.masked_load", + A, + T.Ramp(0, 2, 2), + T.Broadcast(T.bool(True), 2), + ty="float32x2", + ) + + tvm_tirx.compile(Module, target="llvm") + + +@pytest.mark.skipif(not env.has_llvm(), reason="need llvm") +@pytest.mark.parametrize("start, end", [(0, 0), (0, 1), (1, 2), (0, 2)]) +def test_masked_load_non_unit_stride_values(start, end): + @I.ir_module + class Module: + @T.function + def main( + A: T.Tensor((3,), "float32"), + B: T.Tensor((2,), "float32"), + start: T.int32, + end: T.int32, + ): + B[T.Ramp(0, 1, 2)] = T.call_intrin( + "tirx.masked_load", + A, + T.Ramp(0, 2, 2), + (T.Ramp(0, 1, 2) >= T.Broadcast(start, 2)) + & (T.Ramp(0, 1, 2) < T.Broadcast(end, 2)), + ty="float32x2", + ) + + f = tvm_tirx.compile(Module, target="llvm") + a = tvm_tirx.runtime.tensor(np.array([1.0, 2.0, 3.0], dtype="float32")) + b = tvm_tirx.runtime.tensor(np.full(2, -1.0, dtype="float32")) + f(a, b, start, end) + # Inactive lanes are poison, so only the active ones are checked. + np.testing.assert_array_equal(b.numpy()[start:end], [1.0, 3.0][start:end]) + + +@pytest.mark.skipif(not env.has_llvm(), reason="need llvm") +@pytest.mark.parametrize( + "start, end, expected", + [ + (0, 0, [-1.0, -1.0, -1.0]), + (0, 1, [1.0, -1.0, -1.0]), + (1, 2, [-1.0, -1.0, 2.0]), + (0, 2, [1.0, -1.0, 2.0]), + ], +) +def test_masked_store_non_unit_stride(start, end, expected): + @I.ir_module + class Module: + @T.function + def main( + A: T.Tensor((2,), "float32"), + B: T.Tensor((3,), "float32"), + start: T.int32, + end: T.int32, + ): + T.evaluate( + T.call_intrin( + "tirx.masked_store", + B, + A[T.Ramp(0, 1, 2)], + T.Ramp(0, 2, 2), + (T.Ramp(0, 1, 2) >= T.Broadcast(start, 2)) + & (T.Ramp(0, 1, 2) < T.Broadcast(end, 2)), + ty="void", + ) + ) + + f = tvm_tirx.compile(Module, target="llvm") + a = tvm_tirx.runtime.tensor(np.array([1.0, 2.0], dtype="float32")) + b = tvm_tirx.runtime.tensor(np.full(3, -1.0, dtype="float32")) + f(a, b, start, end) + np.testing.assert_array_equal(b.numpy(), expected) + + +@pytest.mark.skipif(not env.has_llvm(), reason="need llvm") +@pytest.mark.parametrize("dtype", ["float32", "int8", "float64"]) +@pytest.mark.parametrize("op", ["load", "store"]) +def test_masked_access_non_unit_stride_last_lane(op, dtype): + # Only lane 3 of 4 is active, so a wrong predicate lane (e.g. i % 2) is caught. + @I.ir_module + class LoadModule: + @T.function + def main( + A: T.Tensor((10,), dtype), + B: T.Tensor((4,), dtype), + lane: T.int32, + ): + B[T.Ramp(0, 1, 4)] = T.call_intrin( + "tirx.masked_load", + A, + T.Ramp(0, 3, 4), + T.Ramp(0, 1, 4) == T.Broadcast(lane, 4), + ty=f"{dtype}x4", + ) + + @I.ir_module + class StoreModule: + @T.function + def main( + A: T.Tensor((4,), dtype), + B: T.Tensor((10,), dtype), + lane: T.int32, + ): + T.evaluate( + T.call_intrin( + "tirx.masked_store", + B, + A[T.Ramp(0, 1, 4)], + T.Ramp(0, 3, 4), + T.Ramp(0, 1, 4) == T.Broadcast(lane, 4), + ty="void", + ) + ) + + if op == "load": + f = tvm_tirx.compile(LoadModule, target="llvm") + a = tvm_tirx.runtime.tensor(np.arange(1, 11).astype(dtype)) + b = tvm_tirx.runtime.tensor(np.full(4, -1, dtype=dtype)) + f(a, b, 3) + # Inactive lanes are poison, so only the active one is checked. + assert b.numpy()[3] == 10 + else: + f = tvm_tirx.compile(StoreModule, target="llvm") + a = tvm_tirx.runtime.tensor(np.arange(1, 5).astype(dtype)) + b = tvm_tirx.runtime.tensor(np.full(10, -1, dtype=dtype)) + f(a, b, 3) + expected = np.full(10, -1, dtype=dtype) + expected[9] = 4 + np.testing.assert_array_equal(b.numpy(), expected) + + def test_invalid_volatile_masked_buffer_load(): @I.ir_module class Module: