From 02d5259668f46cbcb9081d98307fb5979d4be67f Mon Sep 17 00:00:00 2001 From: Hongyi Jin Date: Thu, 12 Nov 2026 01:24:29 +0000 Subject: [PATCH 01/11] [FEAT] Allocate CUDA device memory in a locality domain CUDA 13.5 exposes locality domains: on GPUs such as Rubin the SMs and device memory are split into domains, and an SM reads memory of its own domain faster. Let TIRx place tensors in a domain: - Memory scopes `global.locality_domain.` (k < 8). `Empty` serves them from a per-(device, domain) memory pool created with CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN; the plain `global` path is unchanged, and cudaFree releases both kinds. So `runtime.empty`, `tensor` and `copyto` take `mem_scope="global.locality_domain."` as they are. - `runtime.GetCudaLocalityDomainCount`, `...MultiprocessorCount` and `runtime.GetCudaTensorLocalityDomain`, wrapped by the new `tvm_tirx.backend.cuda.locality` module (`locality_domain_count`, `locality_domain_sm_count`, `locality_domain_of`, `locality_domain_scope`). The locality-domain calls use the driver API: the CUDA runtime loaded at run time can predate 13.5 (e.g. a PyTorch wheel's), and then neither knows the attributes nor fills `cudaPointerAttributes::localityDomainOrdinal`. They are guarded by CUDA_VERSION at build time and by the driver version at run time; without support a device reports one domain and the scopes raise a clear error. Tests (VR200, 2 domains): placement reported back per domain, data round trips including copyto, out-of-range domains, freed memory returning to the device, use as the first CUDA call of a process, kernels and DLPack/torch on localized tensors, and a kernel that writes the half living in the domain each CTA runs on. On VR200, reading 2 x 4 GiB with each domain's CTAs reading their own domain's buffer gives 8.98 TB/s, against 7.57 TB/s from cudaMalloc buffers and 5.98 TB/s when every domain reads the other domain's buffer. Co-Authored-By: Claude Opus 5.5 --- .../python/backend.cuda.locality.rst | 35 +++ docs/reference/python/backend.cuda.rst | 2 + python/tvm_tirx/backend/cuda/__init__.py | 1 + python/tvm_tirx/backend/cuda/locality.py | 106 +++++++++ src/backend/cuda/runtime/cuda_device_api.cc | 201 ++++++++++++++++- .../backend/cuda/test_locality_domain.py | 212 ++++++++++++++++++ 6 files changed, 550 insertions(+), 7 deletions(-) create mode 100644 docs/reference/python/backend.cuda.locality.rst create mode 100644 python/tvm_tirx/backend/cuda/locality.py create mode 100644 tests/python/backend/cuda/test_locality_domain.py diff --git a/docs/reference/python/backend.cuda.locality.rst b/docs/reference/python/backend.cuda.locality.rst new file mode 100644 index 000000000..ccd6deb38 --- /dev/null +++ b/docs/reference/python/backend.cuda.locality.rst @@ -0,0 +1,35 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +tvm_tirx.backend.cuda.locality +============================== + +.. automodule:: tvm_tirx.backend.cuda.locality + :no-members: + +.. currentmodule:: tvm_tirx.backend.cuda.locality + +Functions +--------- + +.. autosummary:: + :toctree: generated/ + + locality_domain_count + locality_domain_of + locality_domain_scope + locality_domain_sm_count diff --git a/docs/reference/python/backend.cuda.rst b/docs/reference/python/backend.cuda.rst index c66adc94d..c93fb1bb4 100644 --- a/docs/reference/python/backend.cuda.rst +++ b/docs/reference/python/backend.cuda.rst @@ -37,6 +37,7 @@ Namespaces * :doc:`tvm_tirx.backend.cuda.compiler ` * :doc:`tvm_tirx.backend.cuda.iket ` * :doc:`tvm_tirx.backend.cuda.launch ` +* :doc:`tvm_tirx.backend.cuda.locality ` * :doc:`tvm_tirx.backend.cuda.ptx ` * :doc:`tvm_tirx.backend.cuda.thread_scope ` * :doc:`tvm_tirx.backend.cuda.tile ` @@ -52,6 +53,7 @@ Namespaces backend.cuda.compiler backend.cuda.iket backend.cuda.launch + backend.cuda.locality backend.cuda.ptx backend.cuda.thread_scope backend.cuda.tile diff --git a/python/tvm_tirx/backend/cuda/__init__.py b/python/tvm_tirx/backend/cuda/__init__.py index 497426972..976772819 100644 --- a/python/tvm_tirx/backend/cuda/__init__.py +++ b/python/tvm_tirx/backend/cuda/__init__.py @@ -35,6 +35,7 @@ "axis", "thread_scope", "launch", + "locality", "op", "ptx", "script", diff --git a/python/tvm_tirx/backend/cuda/locality.py b/python/tvm_tirx/backend/cuda/locality.py new file mode 100644 index 000000000..6627a1996 --- /dev/null +++ b/python/tvm_tirx/backend/cuda/locality.py @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""CUDA locality domains: per-domain device memory and topology queries. + +Some GPUs (e.g. Rubin) split their SMs and device memory into locality domains; +an SM reads memory of its own domain faster than memory of another one. A +tensor is placed in domain ``k`` by allocating it with the memory scope +``global.locality_domain.``:: + + from tvm_tirx.backend.cuda import locality + + dev = tvm_tirx.cuda(0) + for k in range(locality.locality_domain_count(dev)): + scope = locality.locality_domain_scope(k) # "global.locality_domain." + x = tvm_tirx.runtime.empty((n,), "float32", dev, mem_scope=scope) + assert locality.locality_domain_of(x) == k + +A kernel reads the domain of the SM it runs on with +``T.cuda.mov_sreg(32, "locality_domain_id")`` (PTX ISA 9.5) and can then work on +the data of its own domain. Memory of any domain stays accessible from every +SM; only the access speed differs. A device without locality domains reports +one domain, and ``global.locality_domain.0`` then behaves like ``global``. +""" + +import tvm_ffi + +__all__ = [ + "MAX_LOCALITY_DOMAINS", + "locality_domain_count", + "locality_domain_of", + "locality_domain_scope", + "locality_domain_sm_count", +] + +#: Number of ``global.locality_domain.`` memory scopes registered (k < this). +MAX_LOCALITY_DOMAINS = 8 + + +def locality_domain_count(device) -> int: + """Return the number of locality domains of a CUDA device (1 if it has none). + + Parameters + ---------- + device : tvm_tirx.runtime.Device + A CUDA device, e.g. ``tvm_tirx.cuda(0)``. + """ + return int(tvm_ffi.get_global_func("runtime.GetCudaLocalityDomainCount")(device.index)) + + +def locality_domain_sm_count(device) -> int: + """Return the number of SMs guaranteed to each locality domain of a CUDA device. + + A device without locality domains reports all of its SMs. On a device with + domains, SMs beyond ``count * per_domain`` are supplemental SMs; a kernel + tells them apart with ``T.cuda.mov_sreg(32, "locality_domain_supplemental_sm")``. + + Parameters + ---------- + device : tvm_tirx.runtime.Device + A CUDA device, e.g. ``tvm_tirx.cuda(0)``. + """ + return int( + tvm_ffi.get_global_func("runtime.GetCudaLocalityDomainMultiprocessorCount")(device.index) + ) + + +def locality_domain_scope(domain: int) -> str: + """Return the memory scope that places an allocation in locality domain ``domain``. + + Parameters + ---------- + domain : int + The locality domain, in ``[0, locality_domain_count(device))``. + """ + if not 0 <= domain < MAX_LOCALITY_DOMAINS: + raise ValueError( + f"locality domain {domain} is outside the registered scopes [0, {MAX_LOCALITY_DOMAINS})" + ) + return f"global.locality_domain.{domain}" + + +def locality_domain_of(tensor) -> int: + """Return the locality domain a CUDA tensor's memory belongs to, or -1 if it has none. + + Memory allocated with the plain ``global`` scope is not localized and reports -1. + + Parameters + ---------- + tensor : tvm_tirx.runtime.Tensor or any DLPack-compatible tensor + A tensor in CUDA device memory. + """ + return int(tvm_ffi.get_global_func("runtime.GetCudaTensorLocalityDomain")(tensor)) diff --git a/src/backend/cuda/runtime/cuda_device_api.cc b/src/backend/cuda/runtime/cuda_device_api.cc index 745d10be5..b7a6dabe4 100644 --- a/src/backend/cuda/runtime/cuda_device_api.cc +++ b/src/backend/cuda/runtime/cuda_device_api.cc @@ -34,6 +34,12 @@ #include #include +#include +#include +#include +#include +#include +#include #include "../../../runtime/workspace_pool.h" @@ -52,6 +58,136 @@ namespace runtime { } #endif +namespace { + +/*! + * \brief Device memory localized to one CUDA locality domain. + * + * A device with several locality domains (CU_DEVICE_ATTRIBUTE_LOCALITY_DOMAIN_COUNT) + * splits its memory between them; SMs read their own domain's memory faster. + * `Empty` places a tensor in domain k for the memory scope + * `global.locality_domain.`. The scopes are registered for k in + * [0, kMaxLocalityDomains); a device validates k against its own domain count. + */ +constexpr int kMaxLocalityDomains = 8; +constexpr const char* kLocalityDomainScopePrefix = "global.locality_domain."; + +void RegisterLocalityDomainScopes() { + static const bool registered = [] { + for (int k = 0; k < kMaxLocalityDomains; ++k) { + std::string name = kLocalityDomainScopePrefix + std::to_string(k); + if (!MemoryScope::IsDefined(name)) { + ffi::reflection::EnumDef(name.c_str()) + .set_attr(attr::kMemoryAllocProperty, static_cast(MemoryAllocProperty::kNormal)); + } + } + return true; + }(); + (void)registered; +} + +/*! \brief The domain k of `global.locality_domain.`, or nullopt for any other scope. */ +std::optional LocalityDomainOf(const MemoryScope& scope) { + const ffi::String& name = scope.name(); + const std::string_view view(name.data(), name.size()); + const std::string_view prefix(kLocalityDomainScopePrefix); + if (view.substr(0, prefix.size()) != prefix) return std::nullopt; + return std::stoi(std::string(view.substr(prefix.size()))); +} + +// Locality domains go through the driver API, not the CUDA runtime: the +// runtime library loaded at run time may predate CUDA 13.5 (e.g. the one a +// PyTorch wheel ships) and then knows neither the attributes nor the new +// cudaPointerAttributes field, while the driver always matches the device. +#if CUDA_VERSION >= 13050 +/*! \brief Whether the installed driver implements locality domains (CUDA 13.5+). */ +bool DriverHasLocalityDomains() { + static const bool supported = [] { + int version = 0; + return cuDriverGetVersion(&version) == CUDA_SUCCESS && version >= 13050; + }(); + return supported; +} + +int GetDriverAttribute(CUdevice_attribute attribute, int device_id) { + CUDA_DRIVER_CALL(cuInit(0)); + int value = 0; + CUDA_DRIVER_CALL(cuDeviceGetAttribute(&value, attribute, device_id)); + return value; +} +#endif + +/*! \brief Number of locality domains of a device; 1 when it has none or CUDA predates them. */ +int GetCudaLocalityDomainCount(int device_id) { + int count = 0; +#if CUDA_VERSION >= 13050 + if (DriverHasLocalityDomains()) { + count = GetDriverAttribute(CU_DEVICE_ATTRIBUTE_LOCALITY_DOMAIN_COUNT, device_id); + } +#endif + return count > 0 ? count : 1; +} + +/*! \brief SMs guaranteed to each locality domain; the device's SM count without domains. */ +int GetCudaLocalityDomainMultiprocessorCount(int device_id) { + int count = 0; +#if CUDA_VERSION >= 13050 + if (DriverHasLocalityDomains()) { + count = GetDriverAttribute(CU_DEVICE_ATTRIBUTE_LOCALITY_DOMAIN_MULTIPROCESSOR_COUNT, device_id); + } +#endif + if (count <= 0) { + TVM_FFI_CHECK_CUDA_ERROR( + cudaDeviceGetAttribute(&count, cudaDevAttrMultiProcessorCount, device_id)); + } + return count; +} + +/*! \brief The locality domain a tensor's memory belongs to, or -1 if it is not localized. */ +int GetCudaTensorLocalityDomain(ffi::TensorView tensor) { + TVM_FFI_CHECK(tensor.device().device_type == kDLCUDA, ValueError) + << "Expected a tensor in CUDA device memory, got device type " << tensor.device().device_type; +#if CUDA_VERSION >= 13050 + if (DriverHasLocalityDomains()) { + int ordinal = -1; + CUDA_DRIVER_CALL(cuPointerGetAttribute(&ordinal, CU_POINTER_ATTRIBUTE_LOCALITY_DOMAIN_ORDINAL, + reinterpret_cast(tensor.data_ptr()))); + return ordinal; + } +#endif + return -1; +} + +#if CUDA_VERSION >= 13050 +/*! + * \brief The memory pool of one (device, locality domain), created on first use. + * + * Pools live until process exit (they are never destroyed) so that tensors freed + * during static destruction still have a valid pool. The release threshold stays + * at its default of 0: freed memory goes back to the device at the next + * synchronization instead of staying reserved by the pool. + */ +CUmemoryPool GetLocalityDomainPool(int device_id, int domain) { + static std::mutex mutex; + static auto* pools = new std::map, CUmemoryPool>(); + std::lock_guard lock(mutex); + auto it = pools->find({device_id, domain}); + if (it != pools->end()) return it->second; + CUmemPoolProps props{}; + props.allocType = CU_MEM_ALLOCATION_TYPE_PINNED; + props.handleTypes = CU_MEM_HANDLE_TYPE_NONE; + props.location.type = CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN; + props.location.localized.deviceId = static_cast(device_id); + props.location.localized.localityDomainId = static_cast(domain); + CUmemoryPool pool; + CUDA_DRIVER_CALL(cuMemPoolCreate(&pool, &props)); + pools->emplace(std::make_pair(device_id, domain), pool); + return pool; +} +#endif + +} // namespace + class CUDADeviceAPI final : public DeviceAPI { public: void SetDevice(Device dev) final { TVM_FFI_CHECK_CUDA_ERROR(cudaSetDevice(dev.device_id)); } @@ -154,22 +290,32 @@ class CUDADeviceAPI final : public DeviceAPI { } ffi::Tensor Empty(ffi::ShapeView shape, DLDataType dtype, DLDevice device, ffi::Optional mem_scope = std::nullopt) final { - TVM_FFI_CHECK(!mem_scope.has_value() || mem_scope.value() == runtime::MemoryScope::Global(), - ValueError) - << "Unsupported memory scope: " - << mem_scope.value_or(runtime::MemoryScope::Global()).name(); + // `global` (the default) is a plain device allocation; + // `global.locality_domain.` places the tensor in locality domain k. + std::optional locality_domain; + if (mem_scope.has_value() && !(mem_scope.value() == runtime::MemoryScope::Global())) { + locality_domain = LocalityDomainOf(mem_scope.value()); + TVM_FFI_CHECK(locality_domain.has_value(), ValueError) + << "Unsupported memory scope: " << mem_scope.value().name(); + } struct Allocator { CUDADeviceAPI* api; + std::optional locality_domain; void AllocData(DLTensor* tensor) { + size_t nbytes = ffi::GetDataSize(*tensor); + if (locality_domain.has_value()) { + tensor->data = api->AllocLocalityDomainSpace(tensor->device, nbytes, *locality_domain); + return; + } size_t alignment = std::max(static_cast(kAllocAlignment), static_cast((tensor->dtype.bits / 8) * tensor->dtype.lanes)); - tensor->data = api->AllocDataSpace(tensor->device, ffi::GetDataSize(*tensor), alignment, - tensor->dtype); + tensor->data = api->AllocDataSpace(tensor->device, nbytes, alignment, tensor->dtype); } + // cudaFree also releases memory-pool allocations, so both kinds share FreeDataSpace. void FreeData(DLTensor* tensor) { api->FreeDataSpace(tensor->device, tensor->data); } }; - return ffi::Tensor::FromNDAlloc(Allocator{this}, shape, dtype, device); + return ffi::Tensor::FromNDAlloc(Allocator{this, locality_domain}, shape, dtype, device); } private: @@ -190,6 +336,37 @@ class CUDADeviceAPI final : public DeviceAPI { return ret; } + void* AllocLocalityDomainSpace(Device dev, size_t nbytes, int domain) { + TVM_FFI_CHECK(dev.device_type == kDLCUDA, ValueError) + << "Memory scope " << kLocalityDomainScopePrefix << domain << " requires a CUDA device"; +#if CUDA_VERSION >= 13050 + TVM_FFI_CHECK(DriverHasLocalityDomains(), RuntimeError) + << "Memory scope " << kLocalityDomainScopePrefix << domain + << " requires a CUDA driver for CUDA 13.5 or newer"; + ffi::CUDADeviceGuard device_guard(dev.device_id); + // The pool calls below use the driver API, which needs a current context; make + // the runtime create the device's primary context and bind it to this thread, + // as it would for its own first call (the allocation may be the first CUDA call). + TVM_FFI_CHECK_CUDA_ERROR(cudaFree(nullptr)); + int count = GetCudaLocalityDomainCount(dev.device_id); + TVM_FFI_CHECK(domain < count, ValueError) + << "Memory scope " << kLocalityDomainScopePrefix << domain + << " is out of range: CUDA device " << dev.device_id << " has " << count + << " locality domain(s)"; + CUdeviceptr ret; + CUDA_DRIVER_CALL(cuMemAllocFromPoolAsync(&ret, nbytes, + GetLocalityDomainPool(dev.device_id, domain), + /*hStream=*/nullptr)); + // Like cudaMalloc, the allocation is usable from any stream once this returns. + CUDA_DRIVER_CALL(cuStreamSynchronize(nullptr)); + TVM_FFI_ICHECK_EQ(ret % kAllocAlignment, 0U); + return reinterpret_cast(ret); +#else + TVM_FFI_THROW(RuntimeError) << "Memory scope " << kLocalityDomainScopePrefix << domain + << " requires TVM-TIRx built with CUDA 13.5 or newer"; +#endif + } + void FreeDataSpace(Device dev, void* ptr) { if (std::uncaught_exceptions() && cudaPeekAtLastError() == cudaErrorIllegalAddress) { // For most CUDA calls, an error from an API call will be @@ -420,6 +597,16 @@ TVM_FFI_STATIC_INIT_BLOCK() { refl::GlobalDef().def("runtime.GetCudaDeviceCount", GetCudaDeviceCount); } +TVM_FFI_STATIC_INIT_BLOCK() { + namespace refl = tvm::ffi::reflection; + RegisterLocalityDomainScopes(); + refl::GlobalDef() + .def("runtime.GetCudaLocalityDomainCount", GetCudaLocalityDomainCount) + .def("runtime.GetCudaLocalityDomainMultiprocessorCount", + GetCudaLocalityDomainMultiprocessorCount) + .def("runtime.GetCudaTensorLocalityDomain", GetCudaTensorLocalityDomain); +} + #if (CUDA_VERSION >= 12000) /** * \brief FFI wrapper for cuTensorMapEncodeTiled. diff --git a/tests/python/backend/cuda/test_locality_domain.py b/tests/python/backend/cuda/test_locality_domain.py new file mode 100644 index 000000000..b5d78e7a1 --- /dev/null +++ b/tests/python/backend/cuda/test_locality_domain.py @@ -0,0 +1,212 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Locality-domain memory scopes of the CUDA device API.""" + +import numpy as np +import pytest + +import tvm_tirx +import tvm_tirx.testing +from tvm_tirx.backend.cuda import locality +from tvm_tirx.script import tirx as T +from tvm_tirx.testing import env + +requires_cuda = pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") + + +def test_locality_domain_scopes_registered(): + for k in range(locality.MAX_LOCALITY_DOMAINS): + assert tvm_tirx.runtime.MemoryScope(locality.locality_domain_scope(k)).name == ( + f"global.locality_domain.{k}" + ) + with pytest.raises(ValueError): + locality.locality_domain_scope(locality.MAX_LOCALITY_DOMAINS) + with pytest.raises(ValueError): + tvm_tirx.runtime.MemoryScope(f"global.locality_domain.{locality.MAX_LOCALITY_DOMAINS}") + + +@pytest.mark.gpu +@requires_cuda +def test_locality_domain_topology(): + dev = tvm_tirx.cuda(0) + count = locality.locality_domain_count(dev) + per_domain = locality.locality_domain_sm_count(dev) + assert count >= 1 + assert 0 < per_domain * count <= dev.multi_processor_count + + +@pytest.mark.gpu +@requires_cuda +def test_locality_domain_allocation_roundtrip(): + """Each scope places memory in its domain; plain `global` memory is not localized.""" + dev = tvm_tirx.cuda(0) + count = locality.locality_domain_count(dev) + plain = tvm_tirx.runtime.empty((1024,), "float32", dev) + assert locality.locality_domain_of(plain) == -1 + for k in range(count): + data = np.arange(4096, dtype=np.float32) + k + x = tvm_tirx.runtime.tensor(data, dev, mem_scope=locality.locality_domain_scope(k)) + np.testing.assert_array_equal(x.numpy(), data) + y = tvm_tirx.runtime.empty((4096,), "float32", dev, mem_scope=f"global.locality_domain.{k}") + y.copyfrom(x) + np.testing.assert_array_equal(y.numpy(), data) + z = plain.copyto(dev, mem_scope=locality.locality_domain_scope(k)) + if count > 1: + # On a device without domains, domain 0 is the whole device and may report -1. + assert locality.locality_domain_of(x) == k + assert locality.locality_domain_of(y) == k + assert locality.locality_domain_of(z) == k + + +@pytest.mark.gpu +@requires_cuda +def test_locality_domain_allocation_as_first_cuda_call(): + """A localized allocation works as the very first CUDA operation of a process.""" + import subprocess + import sys + + code = ( + "import tvm_tirx\n" + "from tvm_tirx.backend.cuda import locality\n" + "dev = tvm_tirx.cuda(0)\n" + "x = tvm_tirx.runtime.empty((256,), 'float32', dev, mem_scope='global.locality_domain.0')\n" + "print('ok', x.shape)\n" + ) + out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=False) + assert out.returncode == 0, out.stderr + assert out.stdout.startswith("ok") + + +@pytest.mark.gpu +@requires_cuda +def test_locality_domain_out_of_range(): + dev = tvm_tirx.cuda(0) + count = locality.locality_domain_count(dev) + if count >= locality.MAX_LOCALITY_DOMAINS: + pytest.skip("every registered scope is a valid domain on this device") + with pytest.raises(ValueError, match=f"has {count} locality domain"): + tvm_tirx.runtime.empty( + (16,), "float32", dev, mem_scope=locality.locality_domain_scope(count) + ) + + +@pytest.mark.gpu +@requires_cuda +def test_locality_domain_free_returns_memory(): + """Freed localized tensors go back to the device rather than staying in the pool.""" + dev = tvm_tirx.cuda(0) + nbytes = 1 << 30 + scope = locality.locality_domain_scope(locality.locality_domain_count(dev) - 1) + for _ in range(3): # warm up the pool and the allocator + tvm_tirx.runtime.empty((nbytes // 4,), "float32", dev, mem_scope=scope) + before = dev.available_global_memory + for _ in range(8): + x = tvm_tirx.runtime.empty((nbytes // 4,), "float32", dev, mem_scope=scope) + del x + dev.sync() + assert dev.available_global_memory >= before - nbytes + + +@pytest.mark.gpu +@requires_cuda +def test_locality_domain_tensor_in_kernel_and_dlpack(): + """Localized tensors are ordinary CUDA tensors for kernels and DLPack consumers.""" + torch = pytest.importorskip("torch") + + @T.function + def double(A: T.Tensor((1024,), "float32"), B: T.Tensor((1024,), "float32")): + T.device_entry(launch=T.cuda.LaunchConfig(grid=4, block=256)) + i = T.cuda.block_idx("x") * 256 + T.cuda.thread_idx("x") + B[i] = A[i] * T.float32(2) + + mod = tvm_tirx.compile( + tvm_tirx.IRModule({"main": double}), + target=tvm_tirx.target.Target({"kind": "cuda", "arch": env.cuda_arch()}), + pipeline="tirx", + ) + dev = tvm_tirx.cuda(0) + last = locality.locality_domain_count(dev) - 1 + data = np.random.default_rng(0).standard_normal(1024).astype(np.float32) + + def run_and_check(): + a = tvm_tirx.runtime.tensor(data, dev, mem_scope=locality.locality_domain_scope(0)) + b = tvm_tirx.runtime.empty( + (1024,), "float32", dev, mem_scope=locality.locality_domain_scope(last) + ) + mod(a, b) + np.testing.assert_array_equal(b.numpy(), data * 2) + t = torch.from_dlpack(b) + assert t.is_cuda + np.testing.assert_array_equal(t.cpu().numpy(), data * 2) + # The query accepts any DLPack tensor, e.g. torch's own (plain) allocations. + assert locality.locality_domain_of(torch.zeros(4, device="cuda")) == -1 + + tvm_tirx.testing.run_with_gpu_lock(run_and_check) + + +@pytest.mark.gpu +@requires_cuda +def test_locality_domain_aware_kernel(): + """Each CTA writes the half of the output that lives in the domain it runs on.""" + dev = tvm_tirx.cuda(0) + count = locality.locality_domain_count(dev) + if count < 2 or env.cuda_arch() is None or int(env.cuda_arch()[3:6]) < 100: + pytest.skip("needs an sm_100+ GPU with several locality domains") + + @T.function + def fill( + O0: T.Tensor((4096,), "int32"), + O1: T.Tensor((4096,), "int32"), + C: T.Tensor((2,), "int32"), + ): + T.device_entry(launch=T.cuda.LaunchConfig(grid=64, block=64)) + dom = T.cuda.mov_sreg(32, "locality_domain_id") + tx = T.cuda.thread_idx("x") + if tx == 0: + if dom == 0: + T.cuda.atomic_add(C.ptr_to([0]), T.int32(1)) + else: + T.cuda.atomic_add(C.ptr_to([1]), T.int32(1)) + for i in T.serial(64): + if dom == 0: + O0[i * 64 + tx] = dom + else: + O1[i * 64 + tx] = dom + + mod = tvm_tirx.compile( + tvm_tirx.IRModule({"main": fill}), + target=tvm_tirx.target.Target({"kind": "cuda", "arch": env.cuda_arch()}), + pipeline="tirx", + ) + + def run_and_check(): + o = [ + tvm_tirx.runtime.tensor( + np.full(4096, -1, np.int32), dev, mem_scope=locality.locality_domain_scope(k) + ) + for k in range(2) + ] + c = tvm_tirx.runtime.tensor(np.zeros(2, np.int32), dev) + mod(o[0], o[1], c) + ctas = c.numpy() + assert ctas.sum() == 64 + for k in range(2): + got = o[k].numpy() + # A domain that received no CTA leaves its output untouched. + assert (got == (k if ctas[k] else -1)).all() + + tvm_tirx.testing.run_with_gpu_lock(run_and_check) From a0cb79e6d8dc02db80349089ada4163a004532ad Mon Sep 17 00:00:00 2001 From: Hongyi Jin Date: Thu, 12 Nov 2026 07:03:11 +0000 Subject: [PATCH 02/11] [TEST] Domain-aware persistent work-queue kernel over localized buffers CTAs read the buffer of the locality domain they run in through a per-domain chunk queue, then steal from the other domain's queue, since the hardware decides where CTAs land. An XOR checksum and the queue counters verify that every chunk is read exactly once, with a full grid and with a single CTA. --- .../backend/cuda/test_locality_domain.py | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/tests/python/backend/cuda/test_locality_domain.py b/tests/python/backend/cuda/test_locality_domain.py index b5d78e7a1..96ebd84bf 100644 --- a/tests/python/backend/cuda/test_locality_domain.py +++ b/tests/python/backend/cuda/test_locality_domain.py @@ -210,3 +210,94 @@ def run_and_check(): assert (got == (k if ctas[k] else -1)).all() tvm_tirx.testing.run_with_gpu_lock(run_and_check) + + +def _domain_queue_reader(n_words, chunk_words, grid, threads=256): + """A persistent kernel: CTAs read the buffer of their own domain, then help the other. + + B lives in locality domain k and Ctr[k] is its chunk queue. Every word read + is XORed into Sink[0], so the result proves each chunk was read exactly once. + """ + steps = chunk_words // (threads * 4) + n_chunks = n_words // chunk_words + + @T.macro + def drain(B, Ctr, k, tx, chunk, c, r, acc): + if tx == 0: + chunk[0] = T.cuda.atomic_add(Ctr.ptr_to([k]), T.uint32(1)) + T.cuda.cta_sync() + c[0] = chunk[0] + T.cuda.cta_sync() + while c[0] < T.uint32(n_chunks): + for i in T.unroll(steps): + idx = T.Cast("int32", c[0]) * chunk_words + (i * threads + tx) * 4 + T.ptx.ld.global_.cs.v4.u32(r[0], r[1], r[2], r[3], B.ptr_to([idx])) + acc[0] = acc[0] ^ r[0] ^ r[1] ^ r[2] ^ r[3] + if tx == 0: + chunk[0] = T.cuda.atomic_add(Ctr.ptr_to([k]), T.uint32(1)) + T.cuda.cta_sync() + c[0] = chunk[0] + T.cuda.cta_sync() + + @T.function + def read( + B0: T.Tensor((n_words,), "uint32"), + B1: T.Tensor((n_words,), "uint32"), + Ctr: T.Tensor((2,), "uint32"), + Sink: T.Tensor((1,), "uint32"), + ): + T.device_entry(launch=T.cuda.LaunchConfig(grid=grid, block=threads)) + dom = T.cuda.mov_sreg(32, "locality_domain_id") + tx = T.cuda.thread_idx("x") + chunk = T.alloc_shared([1], "uint32") + c = T.alloc_local([1], "uint32") + r = T.alloc_local([4], "uint32") + acc = T.alloc_local([1], "uint32") + acc[0] = T.uint32(0) + # The hardware decides which domain a CTA runs in, so a domain may get + # few CTAs or none: drain the own queue first, then the other one. + if dom == 0: + drain(B0, Ctr, 0, tx, chunk, c, r, acc) + drain(B1, Ctr, 1, tx, chunk, c, r, acc) + else: + drain(B1, Ctr, 1, tx, chunk, c, r, acc) + drain(B0, Ctr, 0, tx, chunk, c, r, acc) + T.ptx.red.global_.xor.b32(Sink.ptr_to([0]), acc[0]) + + return read + + +@pytest.mark.gpu +@requires_cuda +@pytest.mark.parametrize("grid", ["full", 1]) +def test_locality_domain_work_queue_kernel(grid): + """A domain-aware persistent kernel reads every chunk exactly once, however CTAs land.""" + dev = tvm_tirx.cuda(0) + count = locality.locality_domain_count(dev) + if count != 2 or env.cuda_arch() is None or int(env.cuda_arch()[3:6]) < 100: + pytest.skip("needs an sm_100+ GPU with two locality domains") + n_words, chunk_words = 1 << 22, 1 << 14 + if grid == "full": + grid = dev.multi_processor_count * 2 + mod = tvm_tirx.compile( + tvm_tirx.IRModule({"main": _domain_queue_reader(n_words, chunk_words, grid)}), + target=tvm_tirx.target.Target({"kind": "cuda", "arch": env.cuda_arch()}), + pipeline="tirx", + ) + rng = np.random.default_rng(0) + data = [rng.integers(0, 1 << 32, n_words, dtype=np.uint32) for _ in range(2)] + + def run_and_check(): + b = [ + tvm_tirx.runtime.tensor(data[k], dev, mem_scope=locality.locality_domain_scope(k)) + for k in range(2) + ] + ctr = tvm_tirx.runtime.tensor(np.zeros(2, np.uint32), dev) + sink = tvm_tirx.runtime.tensor(np.zeros(1, np.uint32), dev) + mod(b[0], b[1], ctr, sink) + want = np.bitwise_xor.reduce(data[0]) ^ np.bitwise_xor.reduce(data[1]) + assert sink.numpy()[0] == want + # Each queue hands out all its chunks, then one end marker to every CTA. + assert (ctr.numpy() == n_words // chunk_words + grid).all() + + tvm_tirx.testing.run_with_gpu_lock(run_and_check) From 10a0cbdda4703ae9998bb3182c6dab4531fc7f91 Mon Sep 17 00:00:00 2001 From: Hongyi Jin Date: Fri, 13 Nov 2026 00:02:04 +0000 Subject: [PATCH 03/11] [REFACTOR] Move locality domains to tvm_tirx.runtime.locality_domain Address review feedback and align the API with torch.cuda.green_contexts: - Python module tvm_tirx.backend.cuda.locality -> tvm_tirx.runtime.locality_domain, with get_num_locality_domains, is_localization_supported, get_num_sms_per_locality_domain, get_locality_domain_id and locality_domain_scope. Device arguments accept a Device or a device id. - tvm-ffi functions are registered as runtime.cuda., named after the Python function they back (e.g. runtime.cuda.GetNumLocalityDomains). - get_locality_domain_id returns None, not -1, for memory that is not localized. - Host-side support is gated on CUDA 13.4, where the driver APIs first appear. - Docs page and tests move to runtime. --- docs/reference/python/backend.cuda.rst | 2 - ...cality.rst => runtime.locality_domain.rst} | 15 +- docs/reference/python/runtime.rst | 12 ++ python/tvm_tirx/backend/cuda/__init__.py | 1 - python/tvm_tirx/backend/cuda/locality.py | 106 ------------- python/tvm_tirx/runtime/__init__.py | 1 + python/tvm_tirx/runtime/locality_domain.py | 144 ++++++++++++++++++ src/backend/cuda/runtime/cuda_device_api.cc | 51 ++++--- .../cuda => runtime}/test_locality_domain.py | 66 ++++---- 9 files changed, 232 insertions(+), 166 deletions(-) rename docs/reference/python/{backend.cuda.locality.rst => runtime.locality_domain.rst} (76%) delete mode 100644 python/tvm_tirx/backend/cuda/locality.py create mode 100644 python/tvm_tirx/runtime/locality_domain.py rename tests/python/{backend/cuda => runtime}/test_locality_domain.py (81%) diff --git a/docs/reference/python/backend.cuda.rst b/docs/reference/python/backend.cuda.rst index c93fb1bb4..c66adc94d 100644 --- a/docs/reference/python/backend.cuda.rst +++ b/docs/reference/python/backend.cuda.rst @@ -37,7 +37,6 @@ Namespaces * :doc:`tvm_tirx.backend.cuda.compiler ` * :doc:`tvm_tirx.backend.cuda.iket ` * :doc:`tvm_tirx.backend.cuda.launch ` -* :doc:`tvm_tirx.backend.cuda.locality ` * :doc:`tvm_tirx.backend.cuda.ptx ` * :doc:`tvm_tirx.backend.cuda.thread_scope ` * :doc:`tvm_tirx.backend.cuda.tile ` @@ -53,7 +52,6 @@ Namespaces backend.cuda.compiler backend.cuda.iket backend.cuda.launch - backend.cuda.locality backend.cuda.ptx backend.cuda.thread_scope backend.cuda.tile diff --git a/docs/reference/python/backend.cuda.locality.rst b/docs/reference/python/runtime.locality_domain.rst similarity index 76% rename from docs/reference/python/backend.cuda.locality.rst rename to docs/reference/python/runtime.locality_domain.rst index ccd6deb38..092104da7 100644 --- a/docs/reference/python/backend.cuda.locality.rst +++ b/docs/reference/python/runtime.locality_domain.rst @@ -15,13 +15,13 @@ specific language governing permissions and limitations under the License. -tvm_tirx.backend.cuda.locality -============================== +tvm_tirx.runtime.locality_domain +================================ -.. automodule:: tvm_tirx.backend.cuda.locality +.. automodule:: tvm_tirx.runtime.locality_domain :no-members: -.. currentmodule:: tvm_tirx.backend.cuda.locality +.. currentmodule:: tvm_tirx.runtime.locality_domain Functions --------- @@ -29,7 +29,8 @@ Functions .. autosummary:: :toctree: generated/ - locality_domain_count - locality_domain_of + get_num_locality_domains + is_localization_supported + get_num_sms_per_locality_domain + get_locality_domain_id locality_domain_scope - locality_domain_sm_count diff --git a/docs/reference/python/runtime.rst b/docs/reference/python/runtime.rst index 62a439acc..355ab100c 100644 --- a/docs/reference/python/runtime.rst +++ b/docs/reference/python/runtime.rst @@ -23,6 +23,18 @@ tvm_tirx.runtime .. currentmodule:: tvm_tirx.runtime +Namespaces +---------- + +* :doc:`tvm_tirx.runtime.locality_domain ` + +.. toctree:: + :hidden: + :maxdepth: 1 + :titlesonly: + + runtime.locality_domain + Classes ------- diff --git a/python/tvm_tirx/backend/cuda/__init__.py b/python/tvm_tirx/backend/cuda/__init__.py index 976772819..497426972 100644 --- a/python/tvm_tirx/backend/cuda/__init__.py +++ b/python/tvm_tirx/backend/cuda/__init__.py @@ -35,7 +35,6 @@ "axis", "thread_scope", "launch", - "locality", "op", "ptx", "script", diff --git a/python/tvm_tirx/backend/cuda/locality.py b/python/tvm_tirx/backend/cuda/locality.py deleted file mode 100644 index 6627a1996..000000000 --- a/python/tvm_tirx/backend/cuda/locality.py +++ /dev/null @@ -1,106 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -"""CUDA locality domains: per-domain device memory and topology queries. - -Some GPUs (e.g. Rubin) split their SMs and device memory into locality domains; -an SM reads memory of its own domain faster than memory of another one. A -tensor is placed in domain ``k`` by allocating it with the memory scope -``global.locality_domain.``:: - - from tvm_tirx.backend.cuda import locality - - dev = tvm_tirx.cuda(0) - for k in range(locality.locality_domain_count(dev)): - scope = locality.locality_domain_scope(k) # "global.locality_domain." - x = tvm_tirx.runtime.empty((n,), "float32", dev, mem_scope=scope) - assert locality.locality_domain_of(x) == k - -A kernel reads the domain of the SM it runs on with -``T.cuda.mov_sreg(32, "locality_domain_id")`` (PTX ISA 9.5) and can then work on -the data of its own domain. Memory of any domain stays accessible from every -SM; only the access speed differs. A device without locality domains reports -one domain, and ``global.locality_domain.0`` then behaves like ``global``. -""" - -import tvm_ffi - -__all__ = [ - "MAX_LOCALITY_DOMAINS", - "locality_domain_count", - "locality_domain_of", - "locality_domain_scope", - "locality_domain_sm_count", -] - -#: Number of ``global.locality_domain.`` memory scopes registered (k < this). -MAX_LOCALITY_DOMAINS = 8 - - -def locality_domain_count(device) -> int: - """Return the number of locality domains of a CUDA device (1 if it has none). - - Parameters - ---------- - device : tvm_tirx.runtime.Device - A CUDA device, e.g. ``tvm_tirx.cuda(0)``. - """ - return int(tvm_ffi.get_global_func("runtime.GetCudaLocalityDomainCount")(device.index)) - - -def locality_domain_sm_count(device) -> int: - """Return the number of SMs guaranteed to each locality domain of a CUDA device. - - A device without locality domains reports all of its SMs. On a device with - domains, SMs beyond ``count * per_domain`` are supplemental SMs; a kernel - tells them apart with ``T.cuda.mov_sreg(32, "locality_domain_supplemental_sm")``. - - Parameters - ---------- - device : tvm_tirx.runtime.Device - A CUDA device, e.g. ``tvm_tirx.cuda(0)``. - """ - return int( - tvm_ffi.get_global_func("runtime.GetCudaLocalityDomainMultiprocessorCount")(device.index) - ) - - -def locality_domain_scope(domain: int) -> str: - """Return the memory scope that places an allocation in locality domain ``domain``. - - Parameters - ---------- - domain : int - The locality domain, in ``[0, locality_domain_count(device))``. - """ - if not 0 <= domain < MAX_LOCALITY_DOMAINS: - raise ValueError( - f"locality domain {domain} is outside the registered scopes [0, {MAX_LOCALITY_DOMAINS})" - ) - return f"global.locality_domain.{domain}" - - -def locality_domain_of(tensor) -> int: - """Return the locality domain a CUDA tensor's memory belongs to, or -1 if it has none. - - Memory allocated with the plain ``global`` scope is not localized and reports -1. - - Parameters - ---------- - tensor : tvm_tirx.runtime.Tensor or any DLPack-compatible tensor - A tensor in CUDA device memory. - """ - return int(tvm_ffi.get_global_func("runtime.GetCudaTensorLocalityDomain")(tensor)) diff --git a/python/tvm_tirx/runtime/__init__.py b/python/tvm_tirx/runtime/__init__.py index f68ff66ac..1d79175f2 100644 --- a/python/tvm_tirx/runtime/__init__.py +++ b/python/tvm_tirx/runtime/__init__.py @@ -28,6 +28,7 @@ # function exposures from ._tensor import cpu, cuda, opencl, vulkan, metal from ._tensor import vpi, rocm, ext_dev, hexagon +from . import locality_domain from . import _ffi_api diff --git a/python/tvm_tirx/runtime/locality_domain.py b/python/tvm_tirx/runtime/locality_domain.py new file mode 100644 index 000000000..81f7fa6a9 --- /dev/null +++ b/python/tvm_tirx/runtime/locality_domain.py @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""CUDA locality domains: per-domain device memory and topology queries. + +Some GPUs (e.g. Rubin) split their SMs and device memory into locality domains; +an SM reads memory of its own domain faster than memory of another one. A +tensor is placed in domain ``k`` by allocating it with the memory scope +``global.locality_domain.``:: + + from tvm_tirx.runtime import locality_domain + + dev = tvm_tirx.cuda(0) + for k in range(locality_domain.get_num_locality_domains(dev)): + scope = locality_domain.locality_domain_scope(k) # "global.locality_domain." + x = tvm_tirx.runtime.empty((n,), "float32", dev, mem_scope=scope) + assert locality_domain.get_locality_domain_id(x) == k + +The query names and their semantics follow ``torch.cuda.green_contexts`` +(``get_num_locality_domains``, ``is_localization_supported``) and torch's +``locality_domain_id``. Each query wraps the tvm-ffi function +``runtime.cuda.`` of the same name, e.g. ``get_num_locality_domains`` is +``runtime.cuda.GetNumLocalityDomains``. + +A kernel reads the domain of the SM it runs on with +``T.cuda.mov_sreg(32, "locality_domain_id")`` (PTX ISA 9.5) and can then work on +the data of its own domain. Memory of any domain stays accessible from every +SM; only the access speed differs. A device without locality domains reports +one domain, and ``global.locality_domain.0`` then behaves like ``global``. +""" + +import tvm_ffi + +from .device import Device + +__all__ = [ + "MAX_LOCALITY_DOMAINS", + "get_locality_domain_id", + "get_num_locality_domains", + "get_num_sms_per_locality_domain", + "is_localization_supported", + "locality_domain_scope", +] + +#: Number of ``global.locality_domain.`` memory scopes registered (k < this). +MAX_LOCALITY_DOMAINS = 8 + + +def _device_id(device: Device | int) -> int: + if isinstance(device, int): + return device + if device.dlpack_device_type() != tvm_ffi.DLDeviceType.kDLCUDA: + raise ValueError(f"Expected a CUDA device, got {device}") + return device.index + + +def get_num_locality_domains(device: Device | int = 0) -> int: + """Return the number of locality domains of a CUDA device. + + A device without locality domains, or a CUDA driver older than 13.4, + reports 1. + + Parameters + ---------- + device : tvm_tirx.runtime.Device or int + A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id. + """ + return int(tvm_ffi.get_global_func("runtime.cuda.GetNumLocalityDomains")(_device_id(device))) + + +def is_localization_supported(device: Device | int = 0) -> bool: + """Return whether memory can be localized on a CUDA device. + + True when the CUDA driver supports locality domains and the device has two + or more of them. + + Parameters + ---------- + device : tvm_tirx.runtime.Device or int + A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id. + """ + return bool(tvm_ffi.get_global_func("runtime.cuda.IsLocalizationSupported")(_device_id(device))) + + +def get_num_sms_per_locality_domain(device: Device | int = 0) -> int: + """Return the number of SMs guaranteed to each locality domain of a CUDA device. + + A device without locality domains reports all of its SMs. On a device with + domains, SMs beyond ``num_domains * sms_per_domain`` are supplemental SMs; a + kernel tells them apart with + ``T.cuda.mov_sreg(32, "locality_domain_supplemental_sm")``. + + Parameters + ---------- + device : tvm_tirx.runtime.Device or int + A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id. + """ + return int( + tvm_ffi.get_global_func("runtime.cuda.GetNumSMsPerLocalityDomain")(_device_id(device)) + ) + + +def get_locality_domain_id(tensor) -> int | None: + """Return the locality domain a CUDA tensor's memory belongs to. + + Returns None for memory that is not localized, e.g. memory allocated with + the plain ``global`` scope. + + Parameters + ---------- + tensor : tvm_tirx.runtime.Tensor or any DLPack-compatible tensor + A tensor in CUDA device memory. + """ + domain = tvm_ffi.get_global_func("runtime.cuda.GetLocalityDomainId")(tensor) + return None if domain is None else int(domain) + + +def locality_domain_scope(locality_domain_id: int) -> str: + """Return the memory scope that places an allocation in a locality domain. + + Parameters + ---------- + locality_domain_id : int + The locality domain, in ``[0, get_num_locality_domains(device))``. + """ + if not 0 <= locality_domain_id < MAX_LOCALITY_DOMAINS: + raise ValueError( + f"locality domain {locality_domain_id} is outside the registered scopes " + f"[0, {MAX_LOCALITY_DOMAINS})" + ) + return f"global.locality_domain.{locality_domain_id}" diff --git a/src/backend/cuda/runtime/cuda_device_api.cc b/src/backend/cuda/runtime/cuda_device_api.cc index b7a6dabe4..e2b772d2d 100644 --- a/src/backend/cuda/runtime/cuda_device_api.cc +++ b/src/backend/cuda/runtime/cuda_device_api.cc @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -96,15 +97,15 @@ std::optional LocalityDomainOf(const MemoryScope& scope) { } // Locality domains go through the driver API, not the CUDA runtime: the -// runtime library loaded at run time may predate CUDA 13.5 (e.g. the one a +// runtime library loaded at run time may predate CUDA 13.4 (e.g. the one a // PyTorch wheel ships) and then knows neither the attributes nor the new // cudaPointerAttributes field, while the driver always matches the device. -#if CUDA_VERSION >= 13050 -/*! \brief Whether the installed driver implements locality domains (CUDA 13.5+). */ +#if CUDA_VERSION >= 13040 +/*! \brief Whether the installed driver implements locality domains (CUDA 13.4+). */ bool DriverHasLocalityDomains() { static const bool supported = [] { int version = 0; - return cuDriverGetVersion(&version) == CUDA_SUCCESS && version >= 13050; + return cuDriverGetVersion(&version) == CUDA_SUCCESS && version >= 13040; }(); return supported; } @@ -117,10 +118,13 @@ int GetDriverAttribute(CUdevice_attribute attribute, int device_id) { } #endif +// The functions below back `tvm_tirx.runtime.locality_domain`; each one is +// registered as `runtime.cuda.` for the Python function of the same name. + /*! \brief Number of locality domains of a device; 1 when it has none or CUDA predates them. */ -int GetCudaLocalityDomainCount(int device_id) { +int GetNumLocalityDomains(int device_id) { int count = 0; -#if CUDA_VERSION >= 13050 +#if CUDA_VERSION >= 13040 if (DriverHasLocalityDomains()) { count = GetDriverAttribute(CU_DEVICE_ATTRIBUTE_LOCALITY_DOMAIN_COUNT, device_id); } @@ -128,10 +132,13 @@ int GetCudaLocalityDomainCount(int device_id) { return count > 0 ? count : 1; } +/*! \brief Whether memory can be localized on a device: it has two or more locality domains. */ +bool IsLocalizationSupported(int device_id) { return GetNumLocalityDomains(device_id) > 1; } + /*! \brief SMs guaranteed to each locality domain; the device's SM count without domains. */ -int GetCudaLocalityDomainMultiprocessorCount(int device_id) { +int GetNumSMsPerLocalityDomain(int device_id) { int count = 0; -#if CUDA_VERSION >= 13050 +#if CUDA_VERSION >= 13040 if (DriverHasLocalityDomains()) { count = GetDriverAttribute(CU_DEVICE_ATTRIBUTE_LOCALITY_DOMAIN_MULTIPROCESSOR_COUNT, device_id); } @@ -143,22 +150,22 @@ int GetCudaLocalityDomainMultiprocessorCount(int device_id) { return count; } -/*! \brief The locality domain a tensor's memory belongs to, or -1 if it is not localized. */ -int GetCudaTensorLocalityDomain(ffi::TensorView tensor) { +/*! \brief The locality domain a tensor's memory belongs to, or None if it is not localized. */ +ffi::Optional GetLocalityDomainId(ffi::TensorView tensor) { TVM_FFI_CHECK(tensor.device().device_type == kDLCUDA, ValueError) << "Expected a tensor in CUDA device memory, got device type " << tensor.device().device_type; -#if CUDA_VERSION >= 13050 +#if CUDA_VERSION >= 13040 if (DriverHasLocalityDomains()) { int ordinal = -1; CUDA_DRIVER_CALL(cuPointerGetAttribute(&ordinal, CU_POINTER_ATTRIBUTE_LOCALITY_DOMAIN_ORDINAL, reinterpret_cast(tensor.data_ptr()))); - return ordinal; + if (ordinal >= 0) return static_cast(ordinal); } #endif - return -1; + return std::nullopt; } -#if CUDA_VERSION >= 13050 +#if CUDA_VERSION >= 13040 /*! * \brief The memory pool of one (device, locality domain), created on first use. * @@ -339,16 +346,16 @@ class CUDADeviceAPI final : public DeviceAPI { void* AllocLocalityDomainSpace(Device dev, size_t nbytes, int domain) { TVM_FFI_CHECK(dev.device_type == kDLCUDA, ValueError) << "Memory scope " << kLocalityDomainScopePrefix << domain << " requires a CUDA device"; -#if CUDA_VERSION >= 13050 +#if CUDA_VERSION >= 13040 TVM_FFI_CHECK(DriverHasLocalityDomains(), RuntimeError) << "Memory scope " << kLocalityDomainScopePrefix << domain - << " requires a CUDA driver for CUDA 13.5 or newer"; + << " requires a CUDA driver for CUDA 13.4 or newer"; ffi::CUDADeviceGuard device_guard(dev.device_id); // The pool calls below use the driver API, which needs a current context; make // the runtime create the device's primary context and bind it to this thread, // as it would for its own first call (the allocation may be the first CUDA call). TVM_FFI_CHECK_CUDA_ERROR(cudaFree(nullptr)); - int count = GetCudaLocalityDomainCount(dev.device_id); + int count = GetNumLocalityDomains(dev.device_id); TVM_FFI_CHECK(domain < count, ValueError) << "Memory scope " << kLocalityDomainScopePrefix << domain << " is out of range: CUDA device " << dev.device_id << " has " << count @@ -363,7 +370,7 @@ class CUDADeviceAPI final : public DeviceAPI { return reinterpret_cast(ret); #else TVM_FFI_THROW(RuntimeError) << "Memory scope " << kLocalityDomainScopePrefix << domain - << " requires TVM-TIRx built with CUDA 13.5 or newer"; + << " requires TVM-TIRx built with CUDA 13.4 or newer"; #endif } @@ -601,10 +608,10 @@ TVM_FFI_STATIC_INIT_BLOCK() { namespace refl = tvm::ffi::reflection; RegisterLocalityDomainScopes(); refl::GlobalDef() - .def("runtime.GetCudaLocalityDomainCount", GetCudaLocalityDomainCount) - .def("runtime.GetCudaLocalityDomainMultiprocessorCount", - GetCudaLocalityDomainMultiprocessorCount) - .def("runtime.GetCudaTensorLocalityDomain", GetCudaTensorLocalityDomain); + .def("runtime.cuda.GetNumLocalityDomains", GetNumLocalityDomains) + .def("runtime.cuda.IsLocalizationSupported", IsLocalizationSupported) + .def("runtime.cuda.GetNumSMsPerLocalityDomain", GetNumSMsPerLocalityDomain) + .def("runtime.cuda.GetLocalityDomainId", GetLocalityDomainId); } #if (CUDA_VERSION >= 12000) diff --git a/tests/python/backend/cuda/test_locality_domain.py b/tests/python/runtime/test_locality_domain.py similarity index 81% rename from tests/python/backend/cuda/test_locality_domain.py rename to tests/python/runtime/test_locality_domain.py index 96ebd84bf..36c73e0e5 100644 --- a/tests/python/backend/cuda/test_locality_domain.py +++ b/tests/python/runtime/test_locality_domain.py @@ -21,7 +21,7 @@ import tvm_tirx import tvm_tirx.testing -from tvm_tirx.backend.cuda import locality +from tvm_tirx.runtime import locality_domain from tvm_tirx.script import tirx as T from tvm_tirx.testing import env @@ -29,24 +29,33 @@ def test_locality_domain_scopes_registered(): - for k in range(locality.MAX_LOCALITY_DOMAINS): - assert tvm_tirx.runtime.MemoryScope(locality.locality_domain_scope(k)).name == ( + for k in range(locality_domain.MAX_LOCALITY_DOMAINS): + assert tvm_tirx.runtime.MemoryScope(locality_domain.locality_domain_scope(k)).name == ( f"global.locality_domain.{k}" ) with pytest.raises(ValueError): - locality.locality_domain_scope(locality.MAX_LOCALITY_DOMAINS) + locality_domain.locality_domain_scope(locality_domain.MAX_LOCALITY_DOMAINS) with pytest.raises(ValueError): - tvm_tirx.runtime.MemoryScope(f"global.locality_domain.{locality.MAX_LOCALITY_DOMAINS}") + tvm_tirx.runtime.MemoryScope( + f"global.locality_domain.{locality_domain.MAX_LOCALITY_DOMAINS}" + ) @pytest.mark.gpu @requires_cuda def test_locality_domain_topology(): dev = tvm_tirx.cuda(0) - count = locality.locality_domain_count(dev) - per_domain = locality.locality_domain_sm_count(dev) + count = locality_domain.get_num_locality_domains(dev) + per_domain = locality_domain.get_num_sms_per_locality_domain(dev) assert count >= 1 assert 0 < per_domain * count <= dev.multi_processor_count + assert locality_domain.is_localization_supported(dev) == (count > 1) + # A device id works in place of a Device, as in torch.cuda.green_contexts. + assert locality_domain.get_num_locality_domains(0) == count + assert locality_domain.get_num_sms_per_locality_domain(0) == per_domain + assert locality_domain.is_localization_supported(0) == (count > 1) + with pytest.raises(ValueError, match="CUDA device"): + locality_domain.get_num_locality_domains(tvm_tirx.cpu(0)) @pytest.mark.gpu @@ -54,22 +63,22 @@ def test_locality_domain_topology(): def test_locality_domain_allocation_roundtrip(): """Each scope places memory in its domain; plain `global` memory is not localized.""" dev = tvm_tirx.cuda(0) - count = locality.locality_domain_count(dev) + count = locality_domain.get_num_locality_domains(dev) plain = tvm_tirx.runtime.empty((1024,), "float32", dev) - assert locality.locality_domain_of(plain) == -1 + assert locality_domain.get_locality_domain_id(plain) is None for k in range(count): data = np.arange(4096, dtype=np.float32) + k - x = tvm_tirx.runtime.tensor(data, dev, mem_scope=locality.locality_domain_scope(k)) + x = tvm_tirx.runtime.tensor(data, dev, mem_scope=locality_domain.locality_domain_scope(k)) np.testing.assert_array_equal(x.numpy(), data) y = tvm_tirx.runtime.empty((4096,), "float32", dev, mem_scope=f"global.locality_domain.{k}") y.copyfrom(x) np.testing.assert_array_equal(y.numpy(), data) - z = plain.copyto(dev, mem_scope=locality.locality_domain_scope(k)) + z = plain.copyto(dev, mem_scope=locality_domain.locality_domain_scope(k)) if count > 1: - # On a device without domains, domain 0 is the whole device and may report -1. - assert locality.locality_domain_of(x) == k - assert locality.locality_domain_of(y) == k - assert locality.locality_domain_of(z) == k + # On a device without domains, domain 0 is the whole device and reports None. + assert locality_domain.get_locality_domain_id(x) == k + assert locality_domain.get_locality_domain_id(y) == k + assert locality_domain.get_locality_domain_id(z) == k @pytest.mark.gpu @@ -81,7 +90,6 @@ def test_locality_domain_allocation_as_first_cuda_call(): code = ( "import tvm_tirx\n" - "from tvm_tirx.backend.cuda import locality\n" "dev = tvm_tirx.cuda(0)\n" "x = tvm_tirx.runtime.empty((256,), 'float32', dev, mem_scope='global.locality_domain.0')\n" "print('ok', x.shape)\n" @@ -95,12 +103,12 @@ def test_locality_domain_allocation_as_first_cuda_call(): @requires_cuda def test_locality_domain_out_of_range(): dev = tvm_tirx.cuda(0) - count = locality.locality_domain_count(dev) - if count >= locality.MAX_LOCALITY_DOMAINS: + count = locality_domain.get_num_locality_domains(dev) + if count >= locality_domain.MAX_LOCALITY_DOMAINS: pytest.skip("every registered scope is a valid domain on this device") with pytest.raises(ValueError, match=f"has {count} locality domain"): tvm_tirx.runtime.empty( - (16,), "float32", dev, mem_scope=locality.locality_domain_scope(count) + (16,), "float32", dev, mem_scope=locality_domain.locality_domain_scope(count) ) @@ -110,7 +118,7 @@ def test_locality_domain_free_returns_memory(): """Freed localized tensors go back to the device rather than staying in the pool.""" dev = tvm_tirx.cuda(0) nbytes = 1 << 30 - scope = locality.locality_domain_scope(locality.locality_domain_count(dev) - 1) + scope = locality_domain.locality_domain_scope(locality_domain.get_num_locality_domains(dev) - 1) for _ in range(3): # warm up the pool and the allocator tvm_tirx.runtime.empty((nbytes // 4,), "float32", dev, mem_scope=scope) before = dev.available_global_memory @@ -139,13 +147,13 @@ def double(A: T.Tensor((1024,), "float32"), B: T.Tensor((1024,), "float32")): pipeline="tirx", ) dev = tvm_tirx.cuda(0) - last = locality.locality_domain_count(dev) - 1 + last = locality_domain.get_num_locality_domains(dev) - 1 data = np.random.default_rng(0).standard_normal(1024).astype(np.float32) def run_and_check(): - a = tvm_tirx.runtime.tensor(data, dev, mem_scope=locality.locality_domain_scope(0)) + a = tvm_tirx.runtime.tensor(data, dev, mem_scope=locality_domain.locality_domain_scope(0)) b = tvm_tirx.runtime.empty( - (1024,), "float32", dev, mem_scope=locality.locality_domain_scope(last) + (1024,), "float32", dev, mem_scope=locality_domain.locality_domain_scope(last) ) mod(a, b) np.testing.assert_array_equal(b.numpy(), data * 2) @@ -153,7 +161,7 @@ def run_and_check(): assert t.is_cuda np.testing.assert_array_equal(t.cpu().numpy(), data * 2) # The query accepts any DLPack tensor, e.g. torch's own (plain) allocations. - assert locality.locality_domain_of(torch.zeros(4, device="cuda")) == -1 + assert locality_domain.get_locality_domain_id(torch.zeros(4, device="cuda")) is None tvm_tirx.testing.run_with_gpu_lock(run_and_check) @@ -163,7 +171,7 @@ def run_and_check(): def test_locality_domain_aware_kernel(): """Each CTA writes the half of the output that lives in the domain it runs on.""" dev = tvm_tirx.cuda(0) - count = locality.locality_domain_count(dev) + count = locality_domain.get_num_locality_domains(dev) if count < 2 or env.cuda_arch() is None or int(env.cuda_arch()[3:6]) < 100: pytest.skip("needs an sm_100+ GPU with several locality domains") @@ -196,7 +204,7 @@ def fill( def run_and_check(): o = [ tvm_tirx.runtime.tensor( - np.full(4096, -1, np.int32), dev, mem_scope=locality.locality_domain_scope(k) + np.full(4096, -1, np.int32), dev, mem_scope=locality_domain.locality_domain_scope(k) ) for k in range(2) ] @@ -273,7 +281,7 @@ def read( def test_locality_domain_work_queue_kernel(grid): """A domain-aware persistent kernel reads every chunk exactly once, however CTAs land.""" dev = tvm_tirx.cuda(0) - count = locality.locality_domain_count(dev) + count = locality_domain.get_num_locality_domains(dev) if count != 2 or env.cuda_arch() is None or int(env.cuda_arch()[3:6]) < 100: pytest.skip("needs an sm_100+ GPU with two locality domains") n_words, chunk_words = 1 << 22, 1 << 14 @@ -289,7 +297,9 @@ def test_locality_domain_work_queue_kernel(grid): def run_and_check(): b = [ - tvm_tirx.runtime.tensor(data[k], dev, mem_scope=locality.locality_domain_scope(k)) + tvm_tirx.runtime.tensor( + data[k], dev, mem_scope=locality_domain.locality_domain_scope(k) + ) for k in range(2) ] ctr = tvm_tirx.runtime.tensor(np.zeros(2, np.uint32), dev) From 9c29a49dce025dcc5fa1eec6cc7dc77612cf7887 Mon Sep 17 00:00:00 2001 From: Hongyi Jin Date: Fri, 13 Nov 2026 01:26:44 +0000 Subject: [PATCH 04/11] [FIX] Fall back to plain device memory without locality domains CI builds with CUDA 13.1, where global.locality_domain.0 raised instead of behaving like global as documented. Domain 0 on a device that reports a single locality domain, including any device without CUDA 13.4 support, is now a plain cudaMalloc allocation. The scope registration test is skipped when TVM-TIRx is built without the CUDA runtime. --- src/backend/cuda/runtime/cuda_device_api.cc | 22 +++++++++++--------- tests/python/runtime/test_locality_domain.py | 18 ++++++++++++++-- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/backend/cuda/runtime/cuda_device_api.cc b/src/backend/cuda/runtime/cuda_device_api.cc index e2b772d2d..81912efbb 100644 --- a/src/backend/cuda/runtime/cuda_device_api.cc +++ b/src/backend/cuda/runtime/cuda_device_api.cc @@ -346,20 +346,22 @@ class CUDADeviceAPI final : public DeviceAPI { void* AllocLocalityDomainSpace(Device dev, size_t nbytes, int domain) { TVM_FFI_CHECK(dev.device_type == kDLCUDA, ValueError) << "Memory scope " << kLocalityDomainScopePrefix << domain << " requires a CUDA device"; -#if CUDA_VERSION >= 13040 - TVM_FFI_CHECK(DriverHasLocalityDomains(), RuntimeError) + int count = GetNumLocalityDomains(dev.device_id); + TVM_FFI_CHECK(domain < count, ValueError) << "Memory scope " << kLocalityDomainScopePrefix << domain - << " requires a CUDA driver for CUDA 13.4 or newer"; + << " is out of range: CUDA device " << dev.device_id << " has " << count + << " locality domain(s)"; + if (count == 1) { + // Without locality domains, or without CUDA 13.4 support for them, domain 0 + // is the whole device. + return AllocDataSpace(dev, nbytes, kAllocAlignment, DLDataType{kDLUInt, 8, 1}); + } +#if CUDA_VERSION >= 13040 ffi::CUDADeviceGuard device_guard(dev.device_id); // The pool calls below use the driver API, which needs a current context; make // the runtime create the device's primary context and bind it to this thread, // as it would for its own first call (the allocation may be the first CUDA call). TVM_FFI_CHECK_CUDA_ERROR(cudaFree(nullptr)); - int count = GetNumLocalityDomains(dev.device_id); - TVM_FFI_CHECK(domain < count, ValueError) - << "Memory scope " << kLocalityDomainScopePrefix << domain - << " is out of range: CUDA device " << dev.device_id << " has " << count - << " locality domain(s)"; CUdeviceptr ret; CUDA_DRIVER_CALL(cuMemAllocFromPoolAsync(&ret, nbytes, GetLocalityDomainPool(dev.device_id, domain), @@ -369,8 +371,8 @@ class CUDADeviceAPI final : public DeviceAPI { TVM_FFI_ICHECK_EQ(ret % kAllocAlignment, 0U); return reinterpret_cast(ret); #else - TVM_FFI_THROW(RuntimeError) << "Memory scope " << kLocalityDomainScopePrefix << domain - << " requires TVM-TIRx built with CUDA 13.4 or newer"; + TVM_FFI_THROW(InternalError) + << "A device reports several locality domains only with CUDA 13.4+"; #endif } diff --git a/tests/python/runtime/test_locality_domain.py b/tests/python/runtime/test_locality_domain.py index 36c73e0e5..4eab702a9 100644 --- a/tests/python/runtime/test_locality_domain.py +++ b/tests/python/runtime/test_locality_domain.py @@ -18,6 +18,7 @@ import numpy as np import pytest +import tvm_ffi import tvm_tirx import tvm_tirx.testing @@ -26,15 +27,28 @@ from tvm_tirx.testing import env requires_cuda = pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") +requires_cuda_runtime = pytest.mark.skipif( + tvm_ffi.get_global_func("runtime.cuda.GetNumLocalityDomains", allow_missing=True) is None, + reason="TVM-TIRx is built without the CUDA runtime", +) +def test_locality_domain_scope_names(): + for k in range(locality_domain.MAX_LOCALITY_DOMAINS): + assert locality_domain.locality_domain_scope(k) == f"global.locality_domain.{k}" + with pytest.raises(ValueError): + locality_domain.locality_domain_scope(locality_domain.MAX_LOCALITY_DOMAINS) + with pytest.raises(ValueError): + locality_domain.locality_domain_scope(-1) + + +@requires_cuda_runtime def test_locality_domain_scopes_registered(): + """The CUDA runtime registers the scopes, even on a machine without a GPU.""" for k in range(locality_domain.MAX_LOCALITY_DOMAINS): assert tvm_tirx.runtime.MemoryScope(locality_domain.locality_domain_scope(k)).name == ( f"global.locality_domain.{k}" ) - with pytest.raises(ValueError): - locality_domain.locality_domain_scope(locality_domain.MAX_LOCALITY_DOMAINS) with pytest.raises(ValueError): tvm_tirx.runtime.MemoryScope( f"global.locality_domain.{locality_domain.MAX_LOCALITY_DOMAINS}" From ff687850fc7f03f113df14d4663c235fca375922 Mon Sep 17 00:00:00 2001 From: Hongyi Jin Date: Fri, 13 Nov 2026 22:39:17 +0000 Subject: [PATCH 05/11] [REFACTOR] Address review: move locality domains to backend.cuda.runtime - Python module tvm_tirx.runtime.locality_domain -> tvm_tirx.backend.cuda.runtime.locality_domain, calling the runtime.cuda.* functions through a private _ffi_api module; docs and tests move with it. - Locality-domain scopes carry their domain as the memory-scope enum attribute cuda.locality_domain_id instead of being parsed from the name. - Pools live in a LocalityDomainPoolManager, a vector indexed by device and domain behind a member mutex, still never destroyed. - Drop the redundant static registration guard and alignment check; make the context current with cudaSetDevice instead of cudaFree(nullptr). - Free pool memory with cuMemFreeAsync on the legacy default stream: unlike for cudaMalloc memory, cuMemFree does not wait for kernels still using it. Test dropping a tensor right after launching a kernel on it. - Skip the kernels that read %locality_domain_id unless nvcc is 13.5+, and document that domain 0 falls back to plain memory without domain support. --- docs/reference/python/backend.cuda.rst | 2 + ... backend.cuda.runtime.locality_domain.rst} | 8 +- docs/reference/python/runtime.rst | 12 -- python/tvm_tirx/backend/cuda/__init__.py | 2 + .../tvm_tirx/backend/cuda/runtime/__init__.py | 19 +++ .../tvm_tirx/backend/cuda/runtime/_ffi_api.py | 22 +++ .../cuda}/runtime/locality_domain.py | 30 ++-- python/tvm_tirx/runtime/__init__.py | 1 - src/backend/cuda/runtime/cuda_device_api.cc | 134 ++++++++++++------ .../cuda}/test_locality_domain.py | 61 +++++++- 10 files changed, 209 insertions(+), 82 deletions(-) rename docs/reference/python/{runtime.locality_domain.rst => backend.cuda.runtime.locality_domain.rst} (82%) create mode 100644 python/tvm_tirx/backend/cuda/runtime/__init__.py create mode 100644 python/tvm_tirx/backend/cuda/runtime/_ffi_api.py rename python/tvm_tirx/{ => backend/cuda}/runtime/locality_domain.py (83%) rename tests/python/{runtime => backend/cuda}/test_locality_domain.py (85%) diff --git a/docs/reference/python/backend.cuda.rst b/docs/reference/python/backend.cuda.rst index c66adc94d..d8cc773b5 100644 --- a/docs/reference/python/backend.cuda.rst +++ b/docs/reference/python/backend.cuda.rst @@ -38,6 +38,7 @@ Namespaces * :doc:`tvm_tirx.backend.cuda.iket ` * :doc:`tvm_tirx.backend.cuda.launch ` * :doc:`tvm_tirx.backend.cuda.ptx ` +* :doc:`tvm_tirx.backend.cuda.runtime.locality_domain ` * :doc:`tvm_tirx.backend.cuda.thread_scope ` * :doc:`tvm_tirx.backend.cuda.tile ` @@ -53,6 +54,7 @@ Namespaces backend.cuda.iket backend.cuda.launch backend.cuda.ptx + backend.cuda.runtime.locality_domain backend.cuda.thread_scope backend.cuda.tile diff --git a/docs/reference/python/runtime.locality_domain.rst b/docs/reference/python/backend.cuda.runtime.locality_domain.rst similarity index 82% rename from docs/reference/python/runtime.locality_domain.rst rename to docs/reference/python/backend.cuda.runtime.locality_domain.rst index 092104da7..c479fe9ef 100644 --- a/docs/reference/python/runtime.locality_domain.rst +++ b/docs/reference/python/backend.cuda.runtime.locality_domain.rst @@ -15,13 +15,13 @@ specific language governing permissions and limitations under the License. -tvm_tirx.runtime.locality_domain -================================ +tvm_tirx.backend.cuda.runtime.locality_domain +============================================= -.. automodule:: tvm_tirx.runtime.locality_domain +.. automodule:: tvm_tirx.backend.cuda.runtime.locality_domain :no-members: -.. currentmodule:: tvm_tirx.runtime.locality_domain +.. currentmodule:: tvm_tirx.backend.cuda.runtime.locality_domain Functions --------- diff --git a/docs/reference/python/runtime.rst b/docs/reference/python/runtime.rst index 355ab100c..62a439acc 100644 --- a/docs/reference/python/runtime.rst +++ b/docs/reference/python/runtime.rst @@ -23,18 +23,6 @@ tvm_tirx.runtime .. currentmodule:: tvm_tirx.runtime -Namespaces ----------- - -* :doc:`tvm_tirx.runtime.locality_domain ` - -.. toctree:: - :hidden: - :maxdepth: 1 - :titlesonly: - - runtime.locality_domain - Classes ------- diff --git a/python/tvm_tirx/backend/cuda/__init__.py b/python/tvm_tirx/backend/cuda/__init__.py index 497426972..3a8fe5f9b 100644 --- a/python/tvm_tirx/backend/cuda/__init__.py +++ b/python/tvm_tirx/backend/cuda/__init__.py @@ -37,6 +37,7 @@ "launch", "op", "ptx", + "runtime", "script", "target_tags", "tile", @@ -140,6 +141,7 @@ def __getattr__(name: str): "op", "ptx", "register_backend", + "runtime", "script", "script_namespace", "script_namespaces", diff --git a/python/tvm_tirx/backend/cuda/runtime/__init__.py b/python/tvm_tirx/backend/cuda/runtime/__init__.py new file mode 100644 index 000000000..4a26c1c79 --- /dev/null +++ b/python/tvm_tirx/backend/cuda/runtime/__init__.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""CUDA runtime extensions of TIRx.""" + +from . import locality_domain diff --git a/python/tvm_tirx/backend/cuda/runtime/_ffi_api.py b/python/tvm_tirx/backend/cuda/runtime/_ffi_api.py new file mode 100644 index 000000000..218e9fb6c --- /dev/null +++ b/python/tvm_tirx/backend/cuda/runtime/_ffi_api.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""FFI APIs for tvm_tirx.backend.cuda.runtime""" + +import tvm_ffi + +# Exports functions registered in the runtime.cuda namespace. +tvm_ffi.init_ffi_api("runtime.cuda", __name__) diff --git a/python/tvm_tirx/runtime/locality_domain.py b/python/tvm_tirx/backend/cuda/runtime/locality_domain.py similarity index 83% rename from python/tvm_tirx/runtime/locality_domain.py rename to python/tvm_tirx/backend/cuda/runtime/locality_domain.py index 81f7fa6a9..506dfdaf1 100644 --- a/python/tvm_tirx/runtime/locality_domain.py +++ b/python/tvm_tirx/backend/cuda/runtime/locality_domain.py @@ -21,7 +21,7 @@ tensor is placed in domain ``k`` by allocating it with the memory scope ``global.locality_domain.``:: - from tvm_tirx.runtime import locality_domain + from tvm_tirx.backend.cuda.runtime import locality_domain dev = tvm_tirx.cuda(0) for k in range(locality_domain.get_num_locality_domains(dev)): @@ -35,16 +35,25 @@ ``runtime.cuda.`` of the same name, e.g. ``get_num_locality_domains`` is ``runtime.cuda.GetNumLocalityDomains``. +Like ``cudaMalloc`` memory, a localized tensor is ready on every stream once it +is allocated. It is freed in the order of the legacy default stream, after the +work queued on it and on every blocking stream. Work on a nonblocking stream +must be synchronized before the last reference to the tensor goes away. + A kernel reads the domain of the SM it runs on with ``T.cuda.mov_sreg(32, "locality_domain_id")`` (PTX ISA 9.5) and can then work on the data of its own domain. Memory of any domain stays accessible from every -SM; only the access speed differs. A device without locality domains reports -one domain, and ``global.locality_domain.0`` then behaves like ``global``. +SM; only the access speed differs. A device without locality domains, or with +a CUDA build or driver older than 13.4, reports one domain: +``global.locality_domain.0`` then behaves like ``global``, and +``get_locality_domain_id`` returns None for its memory. """ import tvm_ffi -from .device import Device +from tvm_tirx.runtime import Device + +from . import _ffi_api __all__ = [ "MAX_LOCALITY_DOMAINS", @@ -78,7 +87,7 @@ def get_num_locality_domains(device: Device | int = 0) -> int: device : tvm_tirx.runtime.Device or int A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id. """ - return int(tvm_ffi.get_global_func("runtime.cuda.GetNumLocalityDomains")(_device_id(device))) + return int(_ffi_api.GetNumLocalityDomains(_device_id(device))) def is_localization_supported(device: Device | int = 0) -> bool: @@ -92,7 +101,7 @@ def is_localization_supported(device: Device | int = 0) -> bool: device : tvm_tirx.runtime.Device or int A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id. """ - return bool(tvm_ffi.get_global_func("runtime.cuda.IsLocalizationSupported")(_device_id(device))) + return bool(_ffi_api.IsLocalizationSupported(_device_id(device))) def get_num_sms_per_locality_domain(device: Device | int = 0) -> int: @@ -108,23 +117,22 @@ def get_num_sms_per_locality_domain(device: Device | int = 0) -> int: device : tvm_tirx.runtime.Device or int A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id. """ - return int( - tvm_ffi.get_global_func("runtime.cuda.GetNumSMsPerLocalityDomain")(_device_id(device)) - ) + return int(_ffi_api.GetNumSMsPerLocalityDomain(_device_id(device))) def get_locality_domain_id(tensor) -> int | None: """Return the locality domain a CUDA tensor's memory belongs to. Returns None for memory that is not localized, e.g. memory allocated with - the plain ``global`` scope. + the plain ``global`` scope, or with ``global.locality_domain.0`` on a device + that reports a single locality domain. Parameters ---------- tensor : tvm_tirx.runtime.Tensor or any DLPack-compatible tensor A tensor in CUDA device memory. """ - domain = tvm_ffi.get_global_func("runtime.cuda.GetLocalityDomainId")(tensor) + domain = _ffi_api.GetLocalityDomainId(tensor) return None if domain is None else int(domain) diff --git a/python/tvm_tirx/runtime/__init__.py b/python/tvm_tirx/runtime/__init__.py index 1d79175f2..f68ff66ac 100644 --- a/python/tvm_tirx/runtime/__init__.py +++ b/python/tvm_tirx/runtime/__init__.py @@ -28,7 +28,6 @@ # function exposures from ._tensor import cpu, cuda, opencl, vulkan, metal from ._tensor import vpi, rocm, ext_dev, hexagon -from . import locality_domain from . import _ffi_api diff --git a/src/backend/cuda/runtime/cuda_device_api.cc b/src/backend/cuda/runtime/cuda_device_api.cc index 81912efbb..b9f08599d 100644 --- a/src/backend/cuda/runtime/cuda_device_api.cc +++ b/src/backend/cuda/runtime/cuda_device_api.cc @@ -35,12 +35,11 @@ #include #include -#include #include #include #include -#include #include +#include #include "../../../runtime/workspace_pool.h" @@ -73,27 +72,28 @@ namespace { constexpr int kMaxLocalityDomains = 8; constexpr const char* kLocalityDomainScopePrefix = "global.locality_domain."; +/*! \brief Memory-scope attribute: the locality domain k of `global.locality_domain.`. */ +constexpr ffi::reflection::AttrKey kLocalityDomainIdAttr{"cuda.locality_domain_id"}; + void RegisterLocalityDomainScopes() { - static const bool registered = [] { - for (int k = 0; k < kMaxLocalityDomains; ++k) { - std::string name = kLocalityDomainScopePrefix + std::to_string(k); - if (!MemoryScope::IsDefined(name)) { - ffi::reflection::EnumDef(name.c_str()) - .set_attr(attr::kMemoryAllocProperty, static_cast(MemoryAllocProperty::kNormal)); - } + for (int k = 0; k < kMaxLocalityDomains; ++k) { + std::string name = kLocalityDomainScopePrefix + std::to_string(k); + if (!MemoryScope::IsDefined(name)) { + ffi::reflection::EnumDef(name.c_str()) + .set_attr(attr::kMemoryAllocProperty, static_cast(MemoryAllocProperty::kNormal)) + .set_attr(kLocalityDomainIdAttr, k); } - return true; - }(); - (void)registered; + } } -/*! \brief The domain k of `global.locality_domain.`, or nullopt for any other scope. */ +/*! \brief The locality domain of a memory scope; nullopt for other scopes. */ std::optional LocalityDomainOf(const MemoryScope& scope) { - const ffi::String& name = scope.name(); - const std::string_view view(name.data(), name.size()); - const std::string_view prefix(kLocalityDomainScopePrefix); - if (view.substr(0, prefix.size()) != prefix) return std::nullopt; - return std::stoi(std::string(view.substr(prefix.size()))); + static ffi::reflection::TypeAttrColumn state_column(ffi::type_attr::kEnumState); + auto state = state_column[MemoryScopeObj::RuntimeTypeIndex()].cast(); + if (auto column = state->attrs.Get(kLocalityDomainIdAttr.name)) { + if (auto value = column->Get(scope)) return static_cast(value->cast()); + } + return std::nullopt; } // Locality domains go through the driver API, not the CUDA runtime: the @@ -167,30 +167,50 @@ ffi::Optional GetLocalityDomainId(ffi::TensorView tensor) { #if CUDA_VERSION >= 13040 /*! - * \brief The memory pool of one (device, locality domain), created on first use. + * \brief The memory pools of CUDA locality domains, one per (device, domain), + * each created on first use. * * Pools live until process exit (they are never destroyed) so that tensors freed * during static destruction still have a valid pool. The release threshold stays * at its default of 0: freed memory goes back to the device at the next * synchronization instead of staying reserved by the pool. */ -CUmemoryPool GetLocalityDomainPool(int device_id, int domain) { - static std::mutex mutex; - static auto* pools = new std::map, CUmemoryPool>(); - std::lock_guard lock(mutex); - auto it = pools->find({device_id, domain}); - if (it != pools->end()) return it->second; - CUmemPoolProps props{}; - props.allocType = CU_MEM_ALLOCATION_TYPE_PINNED; - props.handleTypes = CU_MEM_HANDLE_TYPE_NONE; - props.location.type = CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN; - props.location.localized.deviceId = static_cast(device_id); - props.location.localized.localityDomainId = static_cast(domain); - CUmemoryPool pool; - CUDA_DRIVER_CALL(cuMemPoolCreate(&pool, &props)); - pools->emplace(std::make_pair(device_id, domain), pool); - return pool; -} +class LocalityDomainPoolManager { + public: + static LocalityDomainPoolManager* Global() { + // Never destroyed, for the same reason as the pools. + static auto* manager = new LocalityDomainPoolManager(); + return manager; + } + + CUmemoryPool GetLocalityDomainPool(int device_id, int locality_domain_id) { + TVM_FFI_CHECK(device_id >= 0, ValueError) << "Invalid CUDA device id " << device_id; + TVM_FFI_CHECK(0 <= locality_domain_id && locality_domain_id < kMaxLocalityDomains, ValueError) + << "Invalid locality domain " << locality_domain_id; + std::lock_guard lock(mutex_); + if (static_cast(device_id) >= pools_.size()) { + pools_.resize(device_id + 1, std::vector(kMaxLocalityDomains, nullptr)); + } + CUmemoryPool& pool = pools_[device_id][locality_domain_id]; + if (pool == nullptr) { + CUmemPoolProps props{}; + props.allocType = CU_MEM_ALLOCATION_TYPE_PINNED; + props.handleTypes = CU_MEM_HANDLE_TYPE_NONE; + props.location.type = CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN; + props.location.localized.deviceId = static_cast(device_id); + props.location.localized.localityDomainId = static_cast(locality_domain_id); + CUmemoryPool created = nullptr; + CUDA_DRIVER_CALL(cuMemPoolCreate(&created, &props)); + pool = created; + } + return pool; + } + + private: + std::mutex mutex_; + /*! \brief pools_[device][domain]; nullptr until created. */ + std::vector> pools_; +}; #endif } // namespace @@ -319,8 +339,13 @@ class CUDADeviceAPI final : public DeviceAPI { static_cast((tensor->dtype.bits / 8) * tensor->dtype.lanes)); tensor->data = api->AllocDataSpace(tensor->device, nbytes, alignment, tensor->dtype); } - // cudaFree also releases memory-pool allocations, so both kinds share FreeDataSpace. - void FreeData(DLTensor* tensor) { api->FreeDataSpace(tensor->device, tensor->data); } + void FreeData(DLTensor* tensor) { + if (locality_domain.has_value()) { + api->FreeLocalityDomainSpace(tensor->device, tensor->data); + return; + } + api->FreeDataSpace(tensor->device, tensor->data); + } }; return ffi::Tensor::FromNDAlloc(Allocator{this, locality_domain}, shape, dtype, device); } @@ -347,7 +372,7 @@ class CUDADeviceAPI final : public DeviceAPI { TVM_FFI_CHECK(dev.device_type == kDLCUDA, ValueError) << "Memory scope " << kLocalityDomainScopePrefix << domain << " requires a CUDA device"; int count = GetNumLocalityDomains(dev.device_id); - TVM_FFI_CHECK(domain < count, ValueError) + TVM_FFI_CHECK(0 <= domain && domain < count, ValueError) << "Memory scope " << kLocalityDomainScopePrefix << domain << " is out of range: CUDA device " << dev.device_id << " has " << count << " locality domain(s)"; @@ -358,17 +383,17 @@ class CUDADeviceAPI final : public DeviceAPI { } #if CUDA_VERSION >= 13040 ffi::CUDADeviceGuard device_guard(dev.device_id); - // The pool calls below use the driver API, which needs a current context; make - // the runtime create the device's primary context and bind it to this thread, - // as it would for its own first call (the allocation may be the first CUDA call). - TVM_FFI_CHECK_CUDA_ERROR(cudaFree(nullptr)); + // Ensure a current context even when the guard did not switch devices: the + // driver calls below need one, and this may be the first CUDA call. Since + // CUDA 12.0, cudaSetDevice initializes the device's primary context. + TVM_FFI_CHECK_CUDA_ERROR(cudaSetDevice(dev.device_id)); CUdeviceptr ret; - CUDA_DRIVER_CALL(cuMemAllocFromPoolAsync(&ret, nbytes, - GetLocalityDomainPool(dev.device_id, domain), - /*hStream=*/nullptr)); + CUDA_DRIVER_CALL(cuMemAllocFromPoolAsync( + &ret, nbytes, + LocalityDomainPoolManager::Global()->GetLocalityDomainPool(dev.device_id, domain), + /*hStream=*/nullptr)); // Like cudaMalloc, the allocation is usable from any stream once this returns. CUDA_DRIVER_CALL(cuStreamSynchronize(nullptr)); - TVM_FFI_ICHECK_EQ(ret % kAllocAlignment, 0U); return reinterpret_cast(ret); #else TVM_FFI_THROW(InternalError) @@ -376,6 +401,21 @@ class CUDADeviceAPI final : public DeviceAPI { #endif } + void FreeLocalityDomainSpace(Device dev, void* ptr) { + // Without locality domains, AllocLocalityDomainSpace made a plain allocation. + if (GetNumLocalityDomains(dev.device_id) == 1) return FreeDataSpace(dev, ptr); +#if CUDA_VERSION >= 13040 + // As in FreeDataSpace, skip the free while unwinding from a sticky CUDA error. + if (std::uncaught_exceptions() && cudaPeekAtLastError() == cudaErrorIllegalAddress) return; + ffi::CUDADeviceGuard device_guard(dev.device_id); + // Unlike for cudaMalloc memory, cuMemFree of pool memory does not wait for + // kernels still using it. Free in the order of the legacy default stream + // instead, which runs after the work already queued on it and on every + // blocking stream; nonblocking streams must synchronize before the tensor dies. + CUDA_DRIVER_CALL(cuMemFreeAsync(reinterpret_cast(ptr), /*hStream=*/nullptr)); +#endif + } + void FreeDataSpace(Device dev, void* ptr) { if (std::uncaught_exceptions() && cudaPeekAtLastError() == cudaErrorIllegalAddress) { // For most CUDA calls, an error from an API call will be diff --git a/tests/python/runtime/test_locality_domain.py b/tests/python/backend/cuda/test_locality_domain.py similarity index 85% rename from tests/python/runtime/test_locality_domain.py rename to tests/python/backend/cuda/test_locality_domain.py index 4eab702a9..65488f2fa 100644 --- a/tests/python/runtime/test_locality_domain.py +++ b/tests/python/backend/cuda/test_locality_domain.py @@ -22,7 +22,7 @@ import tvm_tirx import tvm_tirx.testing -from tvm_tirx.runtime import locality_domain +from tvm_tirx.backend.cuda.runtime import locality_domain from tvm_tirx.script import tirx as T from tvm_tirx.testing import env @@ -33,6 +33,16 @@ ) +def _skip_unless_domain_aware_kernels(dev): + """Skip unless kernels can read %locality_domain_id and see several domains.""" + if locality_domain.get_num_locality_domains(dev) < 2: + pytest.skip("needs a GPU with several locality domains") + if env.cuda_arch() is None or int(env.cuda_arch()[3:6]) < 100: + pytest.skip("needs an sm_100+ GPU") + if not env.has_nvcc_version(13, 5): + pytest.skip("%locality_domain_id needs PTX ISA 9.5 (CUDA 13.5)") + + def test_locality_domain_scope_names(): for k in range(locality_domain.MAX_LOCALITY_DOMAINS): assert locality_domain.locality_domain_scope(k) == f"global.locality_domain.{k}" @@ -143,6 +153,45 @@ def test_locality_domain_free_returns_memory(): assert dev.available_global_memory >= before - nbytes +@pytest.mark.gpu +@requires_cuda +def test_locality_domain_tensor_freed_while_kernel_runs(): + """Dropping a tensor right after launching a kernel on it is safe. + + The free is ordered after the kernel, so the memory is not handed out again, + or returned to the device, while the kernel still writes it. + """ + n, steps = 1 << 24, 256 + + @T.function + def bump(A: T.Tensor((n,), "int32")): + T.device_entry(launch=T.cuda.LaunchConfig(grid=n // 256, block=256)) + i = T.cuda.block_idx("x") * 256 + T.cuda.thread_idx("x") + for _ in T.serial(steps): + A[i] = A[i] + 1 + + mod = tvm_tirx.compile( + tvm_tirx.IRModule({"main": bump}), + target=tvm_tirx.target.Target({"kind": "cuda", "arch": env.cuda_arch()}), + pipeline="tirx", + ) + dev = tvm_tirx.cuda(0) + + def run_and_check(): + for k in range(locality_domain.get_num_locality_domains(dev)): + scope = locality_domain.locality_domain_scope(k) + for _ in range(4): + x = tvm_tirx.runtime.empty((n,), "int32", dev, mem_scope=scope) + mod(x) + del x # no synchronization: the kernel may still be running + y = tvm_tirx.runtime.tensor(np.zeros(n, np.int32), dev, mem_scope=scope) + mod(y) + np.testing.assert_array_equal(y.numpy(), np.full(n, steps, np.int32)) + dev.sync() + + tvm_tirx.testing.run_with_gpu_lock(run_and_check) + + @pytest.mark.gpu @requires_cuda def test_locality_domain_tensor_in_kernel_and_dlpack(): @@ -185,9 +234,7 @@ def run_and_check(): def test_locality_domain_aware_kernel(): """Each CTA writes the half of the output that lives in the domain it runs on.""" dev = tvm_tirx.cuda(0) - count = locality_domain.get_num_locality_domains(dev) - if count < 2 or env.cuda_arch() is None or int(env.cuda_arch()[3:6]) < 100: - pytest.skip("needs an sm_100+ GPU with several locality domains") + _skip_unless_domain_aware_kernels(dev) @T.function def fill( @@ -295,9 +342,9 @@ def read( def test_locality_domain_work_queue_kernel(grid): """A domain-aware persistent kernel reads every chunk exactly once, however CTAs land.""" dev = tvm_tirx.cuda(0) - count = locality_domain.get_num_locality_domains(dev) - if count != 2 or env.cuda_arch() is None or int(env.cuda_arch()[3:6]) < 100: - pytest.skip("needs an sm_100+ GPU with two locality domains") + _skip_unless_domain_aware_kernels(dev) + if locality_domain.get_num_locality_domains(dev) != 2: + pytest.skip("needs a GPU with exactly two locality domains") n_words, chunk_words = 1 << 22, 1 << 14 if grid == "full": grid = dev.multi_processor_count * 2 From be2727abfe652624c763d5657b4bcd1fee036d27 Mon Sep 17 00:00:00 2001 From: Hongyi Jin Date: Sat, 14 Nov 2026 00:54:58 +0000 Subject: [PATCH 06/11] [FIX] Address second review of locality-domain memory - Free localized memory like global memory: synchronize the device, then cudaFree. cudaFree of pool memory does not wait for kernels still using it, and cuMemFreeAsync on the legacy default stream did not cover nonblocking streams. - Make a context current before cuPointerGetAttribute in GetLocalityDomainId, which may run on a thread that never used CUDA. - Query the locality domain count once per device, through cuDeviceGet. - Tests: run free_returns_memory under the GPU lock; parse two-digit archs. --- .../backend/cuda/runtime/locality_domain.py | 4 +- src/backend/cuda/runtime/cuda_device_api.cc | 52 +++++++++++++------ .../backend/cuda/test_locality_domain.py | 27 ++++++---- 3 files changed, 53 insertions(+), 30 deletions(-) diff --git a/python/tvm_tirx/backend/cuda/runtime/locality_domain.py b/python/tvm_tirx/backend/cuda/runtime/locality_domain.py index 506dfdaf1..d8da9263b 100644 --- a/python/tvm_tirx/backend/cuda/runtime/locality_domain.py +++ b/python/tvm_tirx/backend/cuda/runtime/locality_domain.py @@ -36,9 +36,7 @@ ``runtime.cuda.GetNumLocalityDomains``. Like ``cudaMalloc`` memory, a localized tensor is ready on every stream once it -is allocated. It is freed in the order of the legacy default stream, after the -work queued on it and on every blocking stream. Work on a nonblocking stream -must be synchronized before the last reference to the tensor goes away. +is allocated, and freeing it waits for the work on every stream of the device. A kernel reads the domain of the SM it runs on with ``T.cuda.mov_sreg(32, "locality_domain_id")`` (PTX ISA 9.5) and can then work on diff --git a/src/backend/cuda/runtime/cuda_device_api.cc b/src/backend/cuda/runtime/cuda_device_api.cc index b9f08599d..df639b8f1 100644 --- a/src/backend/cuda/runtime/cuda_device_api.cc +++ b/src/backend/cuda/runtime/cuda_device_api.cc @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -86,7 +87,8 @@ void RegisterLocalityDomainScopes() { } } -/*! \brief The locality domain of a memory scope; nullopt for other scopes. */ +/*! \brief The locality domain of a memory scope, or nullopt if it is not a locality-domain scope. + */ std::optional LocalityDomainOf(const MemoryScope& scope) { static ffi::reflection::TypeAttrColumn state_column(ffi::type_attr::kEnumState); auto state = state_column[MemoryScopeObj::RuntimeTypeIndex()].cast(); @@ -112,24 +114,37 @@ bool DriverHasLocalityDomains() { int GetDriverAttribute(CUdevice_attribute attribute, int device_id) { CUDA_DRIVER_CALL(cuInit(0)); + CUdevice device; + CUDA_DRIVER_CALL(cuDeviceGet(&device, device_id)); int value = 0; - CUDA_DRIVER_CALL(cuDeviceGetAttribute(&value, attribute, device_id)); + CUDA_DRIVER_CALL(cuDeviceGetAttribute(&value, attribute, device)); return value; } #endif -// The functions below back `tvm_tirx.runtime.locality_domain`; each one is -// registered as `runtime.cuda.` for the Python function of the same name. +// The functions below back `tvm_tirx.backend.cuda.runtime.locality_domain`; each +// one is registered as `runtime.cuda.` for the Python function of the same name. /*! \brief Number of locality domains of a device; 1 when it has none or CUDA predates them. */ int GetNumLocalityDomains(int device_id) { + // Queried once per device: every localized allocation and free needs it. + static std::mutex mutex; + static std::vector cache; // 0 = not queried yet + std::lock_guard lock(mutex); + TVM_FFI_CHECK(device_id >= 0, ValueError) << "Invalid CUDA device id " << device_id; + if (static_cast(device_id) < cache.size() && cache[device_id] > 0) { + return cache[device_id]; + } int count = 0; #if CUDA_VERSION >= 13040 if (DriverHasLocalityDomains()) { count = GetDriverAttribute(CU_DEVICE_ATTRIBUTE_LOCALITY_DOMAIN_COUNT, device_id); } #endif - return count > 0 ? count : 1; + if (count <= 0) count = 1; + if (static_cast(device_id) >= cache.size()) cache.resize(device_id + 1, 0); + cache[device_id] = count; + return count; } /*! \brief Whether memory can be localized on a device: it has two or more locality domains. */ @@ -156,6 +171,9 @@ ffi::Optional GetLocalityDomainId(ffi::TensorView tensor) { << "Expected a tensor in CUDA device memory, got device type " << tensor.device().device_type; #if CUDA_VERSION >= 13040 if (DriverHasLocalityDomains()) { + // The driver call needs a current context, which a fresh thread may lack. + ffi::CUDADeviceGuard device_guard(tensor.device().device_id); + TVM_FFI_CHECK_CUDA_ERROR(cudaSetDevice(tensor.device().device_id)); int ordinal = -1; CUDA_DRIVER_CALL(cuPointerGetAttribute(&ordinal, CU_POINTER_ATTRIBUTE_LOCALITY_DOMAIN_ORDINAL, reinterpret_cast(tensor.data_ptr()))); @@ -402,18 +420,18 @@ class CUDADeviceAPI final : public DeviceAPI { } void FreeLocalityDomainSpace(Device dev, void* ptr) { - // Without locality domains, AllocLocalityDomainSpace made a plain allocation. - if (GetNumLocalityDomains(dev.device_id) == 1) return FreeDataSpace(dev, ptr); -#if CUDA_VERSION >= 13040 - // As in FreeDataSpace, skip the free while unwinding from a sticky CUDA error. - if (std::uncaught_exceptions() && cudaPeekAtLastError() == cudaErrorIllegalAddress) return; - ffi::CUDADeviceGuard device_guard(dev.device_id); - // Unlike for cudaMalloc memory, cuMemFree of pool memory does not wait for - // kernels still using it. Free in the order of the legacy default stream - // instead, which runs after the work already queued on it and on every - // blocking stream; nonblocking streams must synchronize before the tensor dies. - CUDA_DRIVER_CALL(cuMemFreeAsync(reinterpret_cast(ptr), /*hStream=*/nullptr)); -#endif + // With several locality domains the memory comes from a pool (otherwise it + // is a plain cudaMalloc allocation). cudaFree of cudaMalloc memory waits for + // all work on the device, but cudaFree of pool memory does not, so + // synchronize the device first: as for global memory, no kernel on any + // stream may still use the memory once it is freed. + if (GetNumLocalityDomains(dev.device_id) > 1) { + // As in FreeDataSpace, skip while unwinding from a sticky CUDA error. + if (std::uncaught_exceptions() && cudaPeekAtLastError() == cudaErrorIllegalAddress) return; + ffi::CUDADeviceGuard device_guard(dev.device_id); + TVM_FFI_CHECK_CUDA_ERROR(cudaDeviceSynchronize()); + } + FreeDataSpace(dev, ptr); } void FreeDataSpace(Device dev, void* ptr) { diff --git a/tests/python/backend/cuda/test_locality_domain.py b/tests/python/backend/cuda/test_locality_domain.py index 65488f2fa..53c79ce71 100644 --- a/tests/python/backend/cuda/test_locality_domain.py +++ b/tests/python/backend/cuda/test_locality_domain.py @@ -16,6 +16,8 @@ # under the License. """Locality-domain memory scopes of the CUDA device API.""" +import re + import numpy as np import pytest import tvm_ffi @@ -37,7 +39,8 @@ def _skip_unless_domain_aware_kernels(dev): """Skip unless kernels can read %locality_domain_id and see several domains.""" if locality_domain.get_num_locality_domains(dev) < 2: pytest.skip("needs a GPU with several locality domains") - if env.cuda_arch() is None or int(env.cuda_arch()[3:6]) < 100: + arch = env.cuda_arch() + if arch is None or int(re.match(r"sm_(\d+)", arch).group(1)) < 100: pytest.skip("needs an sm_100+ GPU") if not env.has_nvcc_version(13, 5): pytest.skip("%locality_domain_id needs PTX ISA 9.5 (CUDA 13.5)") @@ -143,14 +146,18 @@ def test_locality_domain_free_returns_memory(): dev = tvm_tirx.cuda(0) nbytes = 1 << 30 scope = locality_domain.locality_domain_scope(locality_domain.get_num_locality_domains(dev) - 1) - for _ in range(3): # warm up the pool and the allocator - tvm_tirx.runtime.empty((nbytes // 4,), "float32", dev, mem_scope=scope) - before = dev.available_global_memory - for _ in range(8): - x = tvm_tirx.runtime.empty((nbytes // 4,), "float32", dev, mem_scope=scope) - del x - dev.sync() - assert dev.available_global_memory >= before - nbytes + + def run_and_check(): + for _ in range(3): # warm up the pool and the allocator + tvm_tirx.runtime.empty((nbytes // 4,), "float32", dev, mem_scope=scope) + before = dev.available_global_memory + for _ in range(8): + x = tvm_tirx.runtime.empty((nbytes // 4,), "float32", dev, mem_scope=scope) + del x + dev.sync() + assert dev.available_global_memory >= before - nbytes + + tvm_tirx.testing.run_with_gpu_lock(run_and_check) @pytest.mark.gpu @@ -158,7 +165,7 @@ def test_locality_domain_free_returns_memory(): def test_locality_domain_tensor_freed_while_kernel_runs(): """Dropping a tensor right after launching a kernel on it is safe. - The free is ordered after the kernel, so the memory is not handed out again, + The free waits for the kernel, so the memory is not handed out again, or returned to the device, while the kernel still writes it. """ n, steps = 1 << 24, 256 From a47f11d8d7b20c9a9f4fe9524e755ea201301a3e Mon Sep 17 00:00:00 2001 From: Hongyi Jin Date: Sat, 14 Nov 2026 01:04:08 +0000 Subject: [PATCH 07/11] [REFACTOR] Choose the single-domain fallback when creating the tensor Empty validates a locality-domain scope and, on a device with a single domain, allocates it like global. AllocLocalityDomainSpace and FreeLocalityDomainSpace now only handle devices with several domains. --- src/backend/cuda/runtime/cuda_device_api.cc | 37 ++++++++++----------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/src/backend/cuda/runtime/cuda_device_api.cc b/src/backend/cuda/runtime/cuda_device_api.cc index df639b8f1..34e771bb9 100644 --- a/src/backend/cuda/runtime/cuda_device_api.cc +++ b/src/backend/cuda/runtime/cuda_device_api.cc @@ -342,6 +342,15 @@ class CUDADeviceAPI final : public DeviceAPI { locality_domain = LocalityDomainOf(mem_scope.value()); TVM_FFI_CHECK(locality_domain.has_value(), ValueError) << "Unsupported memory scope: " << mem_scope.value().name(); + TVM_FFI_CHECK(device.device_type == kDLCUDA, ValueError) + << "Memory scope " << mem_scope.value().name() << " requires a CUDA device"; + int count = GetNumLocalityDomains(device.device_id); + TVM_FFI_CHECK(*locality_domain < count, ValueError) + << "Memory scope " << mem_scope.value().name() << " is out of range: CUDA device " + << device.device_id << " has " << count << " locality domain(s)"; + // Without locality domains, or without CUDA 13.4 support for them, domain 0 + // is the whole device: allocate it like `global`. + if (count == 1) locality_domain = std::nullopt; } struct Allocator { CUDADeviceAPI* api; @@ -386,19 +395,8 @@ class CUDADeviceAPI final : public DeviceAPI { return ret; } + /*! \brief Allocate in locality domain `domain` of a device with several domains. */ void* AllocLocalityDomainSpace(Device dev, size_t nbytes, int domain) { - TVM_FFI_CHECK(dev.device_type == kDLCUDA, ValueError) - << "Memory scope " << kLocalityDomainScopePrefix << domain << " requires a CUDA device"; - int count = GetNumLocalityDomains(dev.device_id); - TVM_FFI_CHECK(0 <= domain && domain < count, ValueError) - << "Memory scope " << kLocalityDomainScopePrefix << domain - << " is out of range: CUDA device " << dev.device_id << " has " << count - << " locality domain(s)"; - if (count == 1) { - // Without locality domains, or without CUDA 13.4 support for them, domain 0 - // is the whole device. - return AllocDataSpace(dev, nbytes, kAllocAlignment, DLDataType{kDLUInt, 8, 1}); - } #if CUDA_VERSION >= 13040 ffi::CUDADeviceGuard device_guard(dev.device_id); // Ensure a current context even when the guard did not switch devices: the @@ -419,15 +417,14 @@ class CUDADeviceAPI final : public DeviceAPI { #endif } + /*! \brief Free memory from AllocLocalityDomainSpace. */ void FreeLocalityDomainSpace(Device dev, void* ptr) { - // With several locality domains the memory comes from a pool (otherwise it - // is a plain cudaMalloc allocation). cudaFree of cudaMalloc memory waits for - // all work on the device, but cudaFree of pool memory does not, so - // synchronize the device first: as for global memory, no kernel on any - // stream may still use the memory once it is freed. - if (GetNumLocalityDomains(dev.device_id) > 1) { - // As in FreeDataSpace, skip while unwinding from a sticky CUDA error. - if (std::uncaught_exceptions() && cudaPeekAtLastError() == cudaErrorIllegalAddress) return; + // As in FreeDataSpace, skip the free while unwinding from a sticky CUDA error. + if (std::uncaught_exceptions() && cudaPeekAtLastError() == cudaErrorIllegalAddress) return; + // cudaFree of cudaMalloc memory waits for all work on the device, but + // cudaFree of pool memory does not. Synchronize the device first so that, + // as for global memory, no kernel on any stream still uses the memory. + { ffi::CUDADeviceGuard device_guard(dev.device_id); TVM_FFI_CHECK_CUDA_ERROR(cudaDeviceSynchronize()); } From 7e674f7300c6bafbd8f8ee92fc522da3199a4ff0 Mon Sep 17 00:00:00 2001 From: tqchen Date: Sat, 14 Nov 2026 01:34:25 +0000 Subject: [PATCH 08/11] [REFACTOR] Cache locality-domain enum attribute column --- src/backend/cuda/runtime/cuda_device_api.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/backend/cuda/runtime/cuda_device_api.cc b/src/backend/cuda/runtime/cuda_device_api.cc index 34e771bb9..876aae831 100644 --- a/src/backend/cuda/runtime/cuda_device_api.cc +++ b/src/backend/cuda/runtime/cuda_device_api.cc @@ -90,11 +90,12 @@ void RegisterLocalityDomainScopes() { /*! \brief The locality domain of a memory scope, or nullopt if it is not a locality-domain scope. */ std::optional LocalityDomainOf(const MemoryScope& scope) { - static ffi::reflection::TypeAttrColumn state_column(ffi::type_attr::kEnumState); - auto state = state_column[MemoryScopeObj::RuntimeTypeIndex()].cast(); - if (auto column = state->attrs.Get(kLocalityDomainIdAttr.name)) { - if (auto value = column->Get(scope)) return static_cast(value->cast()); - } + static auto column = [] { + ffi::reflection::TypeAttrColumn state_column(ffi::type_attr::kEnumState); + auto state = state_column[MemoryScopeObj::RuntimeTypeIndex()].cast(); + return state->attrs.at(kLocalityDomainIdAttr.name); + }(); + if (auto value = column.Get(scope)) return static_cast(value->cast()); return std::nullopt; } From 8a798abf4d8dbb520cfdf363a8187e5bf402bf23 Mon Sep 17 00:00:00 2001 From: tqchen Date: Sat, 14 Nov 2026 01:38:28 +0000 Subject: [PATCH 09/11] [FIX] Probe loaded NVRTC for locality-domain kernel tests --- python/tvm_tirx/testing/env.py | 26 ++++++++++ .../backend/cuda/test_locality_domain.py | 50 +++++++------------ 2 files changed, 45 insertions(+), 31 deletions(-) diff --git a/python/tvm_tirx/testing/env.py b/python/tvm_tirx/testing/env.py index fbba72148..5f6a8dd5f 100644 --- a/python/tvm_tirx/testing/env.py +++ b/python/tvm_tirx/testing/env.py @@ -31,6 +31,7 @@ "has_cuda_arch", # version / capability "has_cuda_compute", + "has_cuda_locality_domain_nvrtc_support", "has_cudagraph", "has_gpu", "has_llvm", @@ -221,6 +222,31 @@ def has_nvcc_version(major: int, minor: int = 0, release: int = 0) -> bool: return has_cuda() and _nvcc_version() >= (major, minor, release) +@functools.cache +def has_cuda_locality_domain_nvrtc_support() -> bool: + """True if locality-domain kernels can run with the default NVRTC compiler. + + Requires a CUDA runtime and an sm_100+ device with multiple locality domains, + plus loaded NVRTC 13.5+ for the PTX 9.5 locality-domain special registers. + This probes kernel-test capability, not just localized allocation support. + Missing bindings, libraries, or failed capability queries return False. + """ + try: + if not has_cuda_compute(10): + return False + get_num_domains = tvm_ffi.get_global_func( + "runtime.cuda.GetNumLocalityDomains", allow_missing=True + ) + if get_num_domains is None or get_num_domains(0) < 2: + return False + from cuda.bindings import nvrtc # pylint: disable=import-outside-toplevel + + status, major, minor = nvrtc.nvrtcVersion() + return status == nvrtc.nvrtcResult.NVRTC_SUCCESS and (major, minor) >= (13, 5) + except Exception: # pylint: disable=broad-except + return False + + @functools.cache def _llvm_version_major() -> int: """Return the major LLVM version, or 0 if LLVM is unavailable.""" diff --git a/tests/python/backend/cuda/test_locality_domain.py b/tests/python/backend/cuda/test_locality_domain.py index 53c79ce71..09845b7ab 100644 --- a/tests/python/backend/cuda/test_locality_domain.py +++ b/tests/python/backend/cuda/test_locality_domain.py @@ -16,8 +16,6 @@ # under the License. """Locality-domain memory scopes of the CUDA device API.""" -import re - import numpy as np import pytest import tvm_ffi @@ -28,23 +26,6 @@ from tvm_tirx.script import tirx as T from tvm_tirx.testing import env -requires_cuda = pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") -requires_cuda_runtime = pytest.mark.skipif( - tvm_ffi.get_global_func("runtime.cuda.GetNumLocalityDomains", allow_missing=True) is None, - reason="TVM-TIRx is built without the CUDA runtime", -) - - -def _skip_unless_domain_aware_kernels(dev): - """Skip unless kernels can read %locality_domain_id and see several domains.""" - if locality_domain.get_num_locality_domains(dev) < 2: - pytest.skip("needs a GPU with several locality domains") - arch = env.cuda_arch() - if arch is None or int(re.match(r"sm_(\d+)", arch).group(1)) < 100: - pytest.skip("needs an sm_100+ GPU") - if not env.has_nvcc_version(13, 5): - pytest.skip("%locality_domain_id needs PTX ISA 9.5 (CUDA 13.5)") - def test_locality_domain_scope_names(): for k in range(locality_domain.MAX_LOCALITY_DOMAINS): @@ -55,7 +36,10 @@ def test_locality_domain_scope_names(): locality_domain.locality_domain_scope(-1) -@requires_cuda_runtime +@pytest.mark.skipif( + tvm_ffi.get_global_func("runtime.cuda.GetNumLocalityDomains", allow_missing=True) is None, + reason="TVM-TIRx is built without the CUDA runtime", +) def test_locality_domain_scopes_registered(): """The CUDA runtime registers the scopes, even on a machine without a GPU.""" for k in range(locality_domain.MAX_LOCALITY_DOMAINS): @@ -69,7 +53,7 @@ def test_locality_domain_scopes_registered(): @pytest.mark.gpu -@requires_cuda +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") def test_locality_domain_topology(): dev = tvm_tirx.cuda(0) count = locality_domain.get_num_locality_domains(dev) @@ -86,7 +70,7 @@ def test_locality_domain_topology(): @pytest.mark.gpu -@requires_cuda +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") def test_locality_domain_allocation_roundtrip(): """Each scope places memory in its domain; plain `global` memory is not localized.""" dev = tvm_tirx.cuda(0) @@ -109,7 +93,7 @@ def test_locality_domain_allocation_roundtrip(): @pytest.mark.gpu -@requires_cuda +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") def test_locality_domain_allocation_as_first_cuda_call(): """A localized allocation works as the very first CUDA operation of a process.""" import subprocess @@ -127,7 +111,7 @@ def test_locality_domain_allocation_as_first_cuda_call(): @pytest.mark.gpu -@requires_cuda +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") def test_locality_domain_out_of_range(): dev = tvm_tirx.cuda(0) count = locality_domain.get_num_locality_domains(dev) @@ -140,7 +124,7 @@ def test_locality_domain_out_of_range(): @pytest.mark.gpu -@requires_cuda +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") def test_locality_domain_free_returns_memory(): """Freed localized tensors go back to the device rather than staying in the pool.""" dev = tvm_tirx.cuda(0) @@ -161,7 +145,7 @@ def run_and_check(): @pytest.mark.gpu -@requires_cuda +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") def test_locality_domain_tensor_freed_while_kernel_runs(): """Dropping a tensor right after launching a kernel on it is safe. @@ -200,7 +184,7 @@ def run_and_check(): @pytest.mark.gpu -@requires_cuda +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") def test_locality_domain_tensor_in_kernel_and_dlpack(): """Localized tensors are ordinary CUDA tensors for kernels and DLPack consumers.""" torch = pytest.importorskip("torch") @@ -237,11 +221,13 @@ def run_and_check(): @pytest.mark.gpu -@requires_cuda +@pytest.mark.skipif( + not env.has_cuda_locality_domain_nvrtc_support(), + reason="needs multiple CUDA locality domains and NVRTC 13.5+", +) def test_locality_domain_aware_kernel(): """Each CTA writes the half of the output that lives in the domain it runs on.""" dev = tvm_tirx.cuda(0) - _skip_unless_domain_aware_kernels(dev) @T.function def fill( @@ -344,12 +330,14 @@ def read( @pytest.mark.gpu -@requires_cuda +@pytest.mark.skipif( + not env.has_cuda_locality_domain_nvrtc_support(), + reason="needs multiple CUDA locality domains and NVRTC 13.5+", +) @pytest.mark.parametrize("grid", ["full", 1]) def test_locality_domain_work_queue_kernel(grid): """A domain-aware persistent kernel reads every chunk exactly once, however CTAs land.""" dev = tvm_tirx.cuda(0) - _skip_unless_domain_aware_kernels(dev) if locality_domain.get_num_locality_domains(dev) != 2: pytest.skip("needs a GPU with exactly two locality domains") n_words, chunk_words = 1 << 22, 1 << 14 From c3c8f724497a61de006c2f174cf53ee4c026702b Mon Sep 17 00:00:00 2001 From: tqchen Date: Sat, 14 Nov 2026 01:38:28 +0000 Subject: [PATCH 10/11] [DOCS] Clarify locality-domain example hardware requirement --- python/tvm_tirx/backend/cuda/runtime/locality_domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tvm_tirx/backend/cuda/runtime/locality_domain.py b/python/tvm_tirx/backend/cuda/runtime/locality_domain.py index d8da9263b..b8d5b43ea 100644 --- a/python/tvm_tirx/backend/cuda/runtime/locality_domain.py +++ b/python/tvm_tirx/backend/cuda/runtime/locality_domain.py @@ -19,7 +19,7 @@ Some GPUs (e.g. Rubin) split their SMs and device memory into locality domains; an SM reads memory of its own domain faster than memory of another one. A tensor is placed in domain ``k`` by allocating it with the memory scope -``global.locality_domain.``:: +``global.locality_domain.``. On a GPU with multiple locality domains:: from tvm_tirx.backend.cuda.runtime import locality_domain From c8412c7488e8ba6145dd01a31c4a6808455cc55e Mon Sep 17 00:00:00 2001 From: tqchen Date: Sat, 14 Nov 2026 01:41:12 +0000 Subject: [PATCH 11/11] [FIX] Return no locality domain for empty CUDA tensors --- src/backend/cuda/runtime/cuda_device_api.cc | 1 + tests/python/backend/cuda/test_locality_domain.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/src/backend/cuda/runtime/cuda_device_api.cc b/src/backend/cuda/runtime/cuda_device_api.cc index 876aae831..4828973e8 100644 --- a/src/backend/cuda/runtime/cuda_device_api.cc +++ b/src/backend/cuda/runtime/cuda_device_api.cc @@ -170,6 +170,7 @@ int GetNumSMsPerLocalityDomain(int device_id) { ffi::Optional GetLocalityDomainId(ffi::TensorView tensor) { TVM_FFI_CHECK(tensor.device().device_type == kDLCUDA, ValueError) << "Expected a tensor in CUDA device memory, got device type " << tensor.device().device_type; + if (tensor.data_ptr() == nullptr) return std::nullopt; #if CUDA_VERSION >= 13040 if (DriverHasLocalityDomains()) { // The driver call needs a current context, which a fresh thread may lack. diff --git a/tests/python/backend/cuda/test_locality_domain.py b/tests/python/backend/cuda/test_locality_domain.py index 09845b7ab..f7639e96b 100644 --- a/tests/python/backend/cuda/test_locality_domain.py +++ b/tests/python/backend/cuda/test_locality_domain.py @@ -92,6 +92,15 @@ def test_locality_domain_allocation_roundtrip(): assert locality_domain.get_locality_domain_id(z) == k +@pytest.mark.gpu +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") +@pytest.mark.parametrize("shape", [(0,), (2, 0, 3)]) +def test_locality_domain_id_empty_tensor(shape): + """Empty CUDA tensors have no localized allocation to query.""" + tensor = tvm_tirx.runtime.empty(shape, "float32", tvm_tirx.cuda(0)) + assert locality_domain.get_locality_domain_id(tensor) is None + + @pytest.mark.gpu @pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") def test_locality_domain_allocation_as_first_cuda_call():