Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 47 additions & 5 deletions src/codegen/llvm/codegen_llvm.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1873,6 +1873,13 @@ void CodeGenLLVM::BufferAccessHelper(
llvm::Value* predicate_value = nullptr;
if (predicate.has_value()) {
predicate_value = MakeValue(predicate.value());

// lanes() throws for scalable vectors, so check IsScalableVector() first.
PrimType predicate_dtype(predicate.value().ty()->dtype);
if (last_index_lanes > 1 && !predicate_dtype.IsScalableVector() &&
predicate_dtype.lanes() == last_index_lanes) {
predicate_value = builder_->CreateExtractElement(predicate_value, i);
}
}

TypedPointer buffer_ptr =
Expand Down Expand Up @@ -1948,9 +1955,28 @@ llvm::Value* CodeGenLLVM::CreateMaskedLoad(const CallObj* op) {
int alignment, bool is_volatile) {
TVM_FFI_ICHECK(!is_volatile)
<< "The masked load intrinsic does not support declaring load as volatile.";
llvm::Instruction* value = builder_->CreateMaskedLoad(buffer_ptr.type, buffer_ptr.addr,
llvm::Align(alignment), predicate);
loads.push_back(value);

if (predicate->getType()->isVectorTy()) {
llvm::Instruction* value = builder_->CreateMaskedLoad(buffer_ptr.type, buffer_ptr.addr,
llvm::Align(alignment), predicate);
loads.push_back(value);
return value;
}

llvm::Type* vector_type = llvm::FixedVectorType::get(buffer_ptr.type, 1);
llvm::Type* mask_vector_type = llvm::FixedVectorType::get(predicate->getType(), 1);

llvm::Value* mask = llvm::UndefValue::get(mask_vector_type);
mask = builder_->CreateInsertElement(mask, predicate, ConstInt32(0));

llvm::Instruction* value =
builder_->CreateMaskedLoad(vector_type, buffer_ptr.addr, llvm::Align(alignment), mask,
llvm::UndefValue::get(vector_type));

llvm::Value* scalar = builder_->CreateExtractElement(value, ConstInt32(0));
loads.push_back(scalar);

// Return the memory-access instruction so AddAliasInfo attaches TBAA to it.
return value;
};
BufferAccessHelper(buffer, indices, predicate, access_dtype, make_load);
Expand Down Expand Up @@ -1986,8 +2012,24 @@ llvm::Value* CodeGenLLVM::CreateMaskedStore(const CallObj* op) {
<< "The masked store intrinsic does not support declaring store as volatile.";
llvm::Value* to_store =
subelement_i == -1 ? value : builder_->CreateExtractElement(value, subelement_i);
last_store = builder_->CreateMaskedStore(to_store, buffer_ptr.addr, llvm::Align(alignment),
predicate);

if (predicate->getType()->isVectorTy()) {
last_store = builder_->CreateMaskedStore(to_store, buffer_ptr.addr,
llvm::Align(alignment), predicate);
return last_store;
}

// llvm.masked.store only accepts vectors: store a scalar lane as a 1-element vector.
llvm::Type* vector_type = llvm::FixedVectorType::get(to_store->getType(), 1);
llvm::Type* mask_vector_type = llvm::FixedVectorType::get(predicate->getType(), 1);

llvm::Value* vector_value = builder_->CreateInsertElement(
llvm::UndefValue::get(vector_type), to_store, ConstInt32(0));
llvm::Value* mask = builder_->CreateInsertElement(llvm::UndefValue::get(mask_vector_type),
predicate, ConstInt32(0));

last_store = builder_->CreateMaskedStore(vector_value, buffer_ptr.addr,
llvm::Align(alignment), mask);
return last_store;
};
BufferAccessHelper(buffer, indices, predicate, value_dtype, make_store);
Expand Down
145 changes: 145 additions & 0 deletions tests/python/codegen/llvm/test_basic.py
Original file line number Diff line number Diff line change
Expand Up @@ -1170,6 +1170,151 @@ def main():
tvm_tirx.compile(Module, target="llvm")


@pytest.mark.skipif(not env.has_llvm(), reason="need llvm")
def test_masked_load_non_unit_stride():
@I.ir_module
class Module:
@T.function
def main(
A: T.Tensor((3,), "float32"),
B: T.Tensor((2,), "float32"),
):
B[T.Ramp(0, 1, 2)] = T.call_intrin(
"tirx.masked_load",
A,
T.Ramp(0, 2, 2),
T.Broadcast(T.bool(True), 2),
ty="float32x2",
)

tvm_tirx.compile(Module, target="llvm")


@pytest.mark.skipif(not env.has_llvm(), reason="need llvm")
@pytest.mark.parametrize("start, end", [(0, 0), (0, 1), (1, 2), (0, 2)])
def test_masked_load_non_unit_stride_values(start, end):
@I.ir_module
class Module:
@T.function
def main(
A: T.Tensor((3,), "float32"),
B: T.Tensor((2,), "float32"),
start: T.int32,
end: T.int32,
):
B[T.Ramp(0, 1, 2)] = T.call_intrin(
"tirx.masked_load",
A,
T.Ramp(0, 2, 2),
(T.Ramp(0, 1, 2) >= T.Broadcast(start, 2))
& (T.Ramp(0, 1, 2) < T.Broadcast(end, 2)),
ty="float32x2",
)

f = tvm_tirx.compile(Module, target="llvm")
a = tvm_tirx.runtime.tensor(np.array([1.0, 2.0, 3.0], dtype="float32"))
b = tvm_tirx.runtime.tensor(np.full(2, -1.0, dtype="float32"))
f(a, b, start, end)
# Inactive lanes are poison, so only the active ones are checked.
np.testing.assert_array_equal(b.numpy()[start:end], [1.0, 3.0][start:end])


@pytest.mark.skipif(not env.has_llvm(), reason="need llvm")
@pytest.mark.parametrize(
"start, end, expected",
[
(0, 0, [-1.0, -1.0, -1.0]),
(0, 1, [1.0, -1.0, -1.0]),
(1, 2, [-1.0, -1.0, 2.0]),
(0, 2, [1.0, -1.0, 2.0]),
],
)
def test_masked_store_non_unit_stride(start, end, expected):
@I.ir_module
class Module:
@T.function
def main(
A: T.Tensor((2,), "float32"),
B: T.Tensor((3,), "float32"),
start: T.int32,
end: T.int32,
):
T.evaluate(
T.call_intrin(
"tirx.masked_store",
B,
A[T.Ramp(0, 1, 2)],
T.Ramp(0, 2, 2),
(T.Ramp(0, 1, 2) >= T.Broadcast(start, 2))
& (T.Ramp(0, 1, 2) < T.Broadcast(end, 2)),
ty="void",
)
)

f = tvm_tirx.compile(Module, target="llvm")
a = tvm_tirx.runtime.tensor(np.array([1.0, 2.0], dtype="float32"))
b = tvm_tirx.runtime.tensor(np.full(3, -1.0, dtype="float32"))
f(a, b, start, end)
np.testing.assert_array_equal(b.numpy(), expected)


@pytest.mark.skipif(not env.has_llvm(), reason="need llvm")
@pytest.mark.parametrize("dtype", ["float32", "int8", "float64"])
@pytest.mark.parametrize("op", ["load", "store"])
def test_masked_access_non_unit_stride_last_lane(op, dtype):
# Only lane 3 of 4 is active, so a wrong predicate lane (e.g. i % 2) is caught.
@I.ir_module
class LoadModule:
@T.function
def main(
A: T.Tensor((10,), dtype),
B: T.Tensor((4,), dtype),
lane: T.int32,
):
B[T.Ramp(0, 1, 4)] = T.call_intrin(
"tirx.masked_load",
A,
T.Ramp(0, 3, 4),
T.Ramp(0, 1, 4) == T.Broadcast(lane, 4),
ty=f"{dtype}x4",
)

@I.ir_module
class StoreModule:
@T.function
def main(
A: T.Tensor((4,), dtype),
B: T.Tensor((10,), dtype),
lane: T.int32,
):
T.evaluate(
T.call_intrin(
"tirx.masked_store",
B,
A[T.Ramp(0, 1, 4)],
T.Ramp(0, 3, 4),
T.Ramp(0, 1, 4) == T.Broadcast(lane, 4),
ty="void",
)
)

if op == "load":
f = tvm_tirx.compile(LoadModule, target="llvm")
a = tvm_tirx.runtime.tensor(np.arange(1, 11).astype(dtype))
b = tvm_tirx.runtime.tensor(np.full(4, -1, dtype=dtype))
f(a, b, 3)
# Inactive lanes are poison, so only the active one is checked.
assert b.numpy()[3] == 10
else:
f = tvm_tirx.compile(StoreModule, target="llvm")
a = tvm_tirx.runtime.tensor(np.arange(1, 5).astype(dtype))
b = tvm_tirx.runtime.tensor(np.full(10, -1, dtype=dtype))
f(a, b, 3)
expected = np.full(10, -1, dtype=dtype)
expected[9] = 4
np.testing.assert_array_equal(b.numpy(), expected)


def test_invalid_volatile_masked_buffer_load():
@I.ir_module
class Module:
Expand Down