diff --git a/docs/reference/python/backend.cuda.rst b/docs/reference/python/backend.cuda.rst index c66adc94d..d8cc773b5 100644 --- a/docs/reference/python/backend.cuda.rst +++ b/docs/reference/python/backend.cuda.rst @@ -38,6 +38,7 @@ Namespaces * :doc:`tvm_tirx.backend.cuda.iket ` * :doc:`tvm_tirx.backend.cuda.launch ` * :doc:`tvm_tirx.backend.cuda.ptx ` +* :doc:`tvm_tirx.backend.cuda.runtime.locality_domain ` * :doc:`tvm_tirx.backend.cuda.thread_scope ` * :doc:`tvm_tirx.backend.cuda.tile ` @@ -53,6 +54,7 @@ Namespaces backend.cuda.iket backend.cuda.launch backend.cuda.ptx + backend.cuda.runtime.locality_domain backend.cuda.thread_scope backend.cuda.tile diff --git a/docs/reference/python/backend.cuda.runtime.locality_domain.rst b/docs/reference/python/backend.cuda.runtime.locality_domain.rst new file mode 100644 index 000000000..c479fe9ef --- /dev/null +++ b/docs/reference/python/backend.cuda.runtime.locality_domain.rst @@ -0,0 +1,36 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +tvm_tirx.backend.cuda.runtime.locality_domain +============================================= + +.. automodule:: tvm_tirx.backend.cuda.runtime.locality_domain + :no-members: + +.. currentmodule:: tvm_tirx.backend.cuda.runtime.locality_domain + +Functions +--------- + +.. autosummary:: + :toctree: generated/ + + get_num_locality_domains + is_localization_supported + get_num_sms_per_locality_domain + get_locality_domain_id + locality_domain_scope diff --git a/python/tvm_tirx/backend/cuda/__init__.py b/python/tvm_tirx/backend/cuda/__init__.py index 497426972..3a8fe5f9b 100644 --- a/python/tvm_tirx/backend/cuda/__init__.py +++ b/python/tvm_tirx/backend/cuda/__init__.py @@ -37,6 +37,7 @@ "launch", "op", "ptx", + "runtime", "script", "target_tags", "tile", @@ -140,6 +141,7 @@ def __getattr__(name: str): "op", "ptx", "register_backend", + "runtime", "script", "script_namespace", "script_namespaces", diff --git a/python/tvm_tirx/backend/cuda/runtime/__init__.py b/python/tvm_tirx/backend/cuda/runtime/__init__.py new file mode 100644 index 000000000..4a26c1c79 --- /dev/null +++ b/python/tvm_tirx/backend/cuda/runtime/__init__.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""CUDA runtime extensions of TIRx.""" + +from . import locality_domain diff --git a/python/tvm_tirx/backend/cuda/runtime/_ffi_api.py b/python/tvm_tirx/backend/cuda/runtime/_ffi_api.py new file mode 100644 index 000000000..218e9fb6c --- /dev/null +++ b/python/tvm_tirx/backend/cuda/runtime/_ffi_api.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""FFI APIs for tvm_tirx.backend.cuda.runtime""" + +import tvm_ffi + +# Exports functions registered in the runtime.cuda namespace. +tvm_ffi.init_ffi_api("runtime.cuda", __name__) diff --git a/python/tvm_tirx/backend/cuda/runtime/locality_domain.py b/python/tvm_tirx/backend/cuda/runtime/locality_domain.py new file mode 100644 index 000000000..b8d5b43ea --- /dev/null +++ b/python/tvm_tirx/backend/cuda/runtime/locality_domain.py @@ -0,0 +1,150 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""CUDA locality domains: per-domain device memory and topology queries. + +Some GPUs (e.g. Rubin) split their SMs and device memory into locality domains; +an SM reads memory of its own domain faster than memory of another one. A +tensor is placed in domain ``k`` by allocating it with the memory scope +``global.locality_domain.``. On a GPU with multiple locality domains:: + + from tvm_tirx.backend.cuda.runtime import locality_domain + + dev = tvm_tirx.cuda(0) + for k in range(locality_domain.get_num_locality_domains(dev)): + scope = locality_domain.locality_domain_scope(k) # "global.locality_domain." + x = tvm_tirx.runtime.empty((n,), "float32", dev, mem_scope=scope) + assert locality_domain.get_locality_domain_id(x) == k + +The query names and their semantics follow ``torch.cuda.green_contexts`` +(``get_num_locality_domains``, ``is_localization_supported``) and torch's +``locality_domain_id``. Each query wraps the tvm-ffi function +``runtime.cuda.`` of the same name, e.g. ``get_num_locality_domains`` is +``runtime.cuda.GetNumLocalityDomains``. + +Like ``cudaMalloc`` memory, a localized tensor is ready on every stream once it +is allocated, and freeing it waits for the work on every stream of the device. + +A kernel reads the domain of the SM it runs on with +``T.cuda.mov_sreg(32, "locality_domain_id")`` (PTX ISA 9.5) and can then work on +the data of its own domain. Memory of any domain stays accessible from every +SM; only the access speed differs. A device without locality domains, or with +a CUDA build or driver older than 13.4, reports one domain: +``global.locality_domain.0`` then behaves like ``global``, and +``get_locality_domain_id`` returns None for its memory. +""" + +import tvm_ffi + +from tvm_tirx.runtime import Device + +from . import _ffi_api + +__all__ = [ + "MAX_LOCALITY_DOMAINS", + "get_locality_domain_id", + "get_num_locality_domains", + "get_num_sms_per_locality_domain", + "is_localization_supported", + "locality_domain_scope", +] + +#: Number of ``global.locality_domain.`` memory scopes registered (k < this). +MAX_LOCALITY_DOMAINS = 8 + + +def _device_id(device: Device | int) -> int: + if isinstance(device, int): + return device + if device.dlpack_device_type() != tvm_ffi.DLDeviceType.kDLCUDA: + raise ValueError(f"Expected a CUDA device, got {device}") + return device.index + + +def get_num_locality_domains(device: Device | int = 0) -> int: + """Return the number of locality domains of a CUDA device. + + A device without locality domains, or a CUDA driver older than 13.4, + reports 1. + + Parameters + ---------- + device : tvm_tirx.runtime.Device or int + A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id. + """ + return int(_ffi_api.GetNumLocalityDomains(_device_id(device))) + + +def is_localization_supported(device: Device | int = 0) -> bool: + """Return whether memory can be localized on a CUDA device. + + True when the CUDA driver supports locality domains and the device has two + or more of them. + + Parameters + ---------- + device : tvm_tirx.runtime.Device or int + A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id. + """ + return bool(_ffi_api.IsLocalizationSupported(_device_id(device))) + + +def get_num_sms_per_locality_domain(device: Device | int = 0) -> int: + """Return the number of SMs guaranteed to each locality domain of a CUDA device. + + A device without locality domains reports all of its SMs. On a device with + domains, SMs beyond ``num_domains * sms_per_domain`` are supplemental SMs; a + kernel tells them apart with + ``T.cuda.mov_sreg(32, "locality_domain_supplemental_sm")``. + + Parameters + ---------- + device : tvm_tirx.runtime.Device or int + A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id. + """ + return int(_ffi_api.GetNumSMsPerLocalityDomain(_device_id(device))) + + +def get_locality_domain_id(tensor) -> int | None: + """Return the locality domain a CUDA tensor's memory belongs to. + + Returns None for memory that is not localized, e.g. memory allocated with + the plain ``global`` scope, or with ``global.locality_domain.0`` on a device + that reports a single locality domain. + + Parameters + ---------- + tensor : tvm_tirx.runtime.Tensor or any DLPack-compatible tensor + A tensor in CUDA device memory. + """ + domain = _ffi_api.GetLocalityDomainId(tensor) + return None if domain is None else int(domain) + + +def locality_domain_scope(locality_domain_id: int) -> str: + """Return the memory scope that places an allocation in a locality domain. + + Parameters + ---------- + locality_domain_id : int + The locality domain, in ``[0, get_num_locality_domains(device))``. + """ + if not 0 <= locality_domain_id < MAX_LOCALITY_DOMAINS: + raise ValueError( + f"locality domain {locality_domain_id} is outside the registered scopes " + f"[0, {MAX_LOCALITY_DOMAINS})" + ) + return f"global.locality_domain.{locality_domain_id}" diff --git a/python/tvm_tirx/testing/env.py b/python/tvm_tirx/testing/env.py index fbba72148..5f6a8dd5f 100644 --- a/python/tvm_tirx/testing/env.py +++ b/python/tvm_tirx/testing/env.py @@ -31,6 +31,7 @@ "has_cuda_arch", # version / capability "has_cuda_compute", + "has_cuda_locality_domain_nvrtc_support", "has_cudagraph", "has_gpu", "has_llvm", @@ -221,6 +222,31 @@ def has_nvcc_version(major: int, minor: int = 0, release: int = 0) -> bool: return has_cuda() and _nvcc_version() >= (major, minor, release) +@functools.cache +def has_cuda_locality_domain_nvrtc_support() -> bool: + """True if locality-domain kernels can run with the default NVRTC compiler. + + Requires a CUDA runtime and an sm_100+ device with multiple locality domains, + plus loaded NVRTC 13.5+ for the PTX 9.5 locality-domain special registers. + This probes kernel-test capability, not just localized allocation support. + Missing bindings, libraries, or failed capability queries return False. + """ + try: + if not has_cuda_compute(10): + return False + get_num_domains = tvm_ffi.get_global_func( + "runtime.cuda.GetNumLocalityDomains", allow_missing=True + ) + if get_num_domains is None or get_num_domains(0) < 2: + return False + from cuda.bindings import nvrtc # pylint: disable=import-outside-toplevel + + status, major, minor = nvrtc.nvrtcVersion() + return status == nvrtc.nvrtcResult.NVRTC_SUCCESS and (major, minor) >= (13, 5) + except Exception: # pylint: disable=broad-except + return False + + @functools.cache def _llvm_version_major() -> int: """Return the major LLVM version, or 0 if LLVM is unavailable.""" diff --git a/src/backend/cuda/runtime/cuda_device_api.cc b/src/backend/cuda/runtime/cuda_device_api.cc index 745d10be5..4828973e8 100644 --- a/src/backend/cuda/runtime/cuda_device_api.cc +++ b/src/backend/cuda/runtime/cuda_device_api.cc @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,12 @@ #include #include +#include +#include +#include +#include +#include +#include #include "../../../runtime/workspace_pool.h" @@ -52,6 +59,182 @@ namespace runtime { } #endif +namespace { + +/*! + * \brief Device memory localized to one CUDA locality domain. + * + * A device with several locality domains (CU_DEVICE_ATTRIBUTE_LOCALITY_DOMAIN_COUNT) + * splits its memory between them; SMs read their own domain's memory faster. + * `Empty` places a tensor in domain k for the memory scope + * `global.locality_domain.`. The scopes are registered for k in + * [0, kMaxLocalityDomains); a device validates k against its own domain count. + */ +constexpr int kMaxLocalityDomains = 8; +constexpr const char* kLocalityDomainScopePrefix = "global.locality_domain."; + +/*! \brief Memory-scope attribute: the locality domain k of `global.locality_domain.`. */ +constexpr ffi::reflection::AttrKey kLocalityDomainIdAttr{"cuda.locality_domain_id"}; + +void RegisterLocalityDomainScopes() { + for (int k = 0; k < kMaxLocalityDomains; ++k) { + std::string name = kLocalityDomainScopePrefix + std::to_string(k); + if (!MemoryScope::IsDefined(name)) { + ffi::reflection::EnumDef(name.c_str()) + .set_attr(attr::kMemoryAllocProperty, static_cast(MemoryAllocProperty::kNormal)) + .set_attr(kLocalityDomainIdAttr, k); + } + } +} + +/*! \brief The locality domain of a memory scope, or nullopt if it is not a locality-domain scope. + */ +std::optional LocalityDomainOf(const MemoryScope& scope) { + static auto column = [] { + ffi::reflection::TypeAttrColumn state_column(ffi::type_attr::kEnumState); + auto state = state_column[MemoryScopeObj::RuntimeTypeIndex()].cast(); + return state->attrs.at(kLocalityDomainIdAttr.name); + }(); + if (auto value = column.Get(scope)) return static_cast(value->cast()); + return std::nullopt; +} + +// Locality domains go through the driver API, not the CUDA runtime: the +// runtime library loaded at run time may predate CUDA 13.4 (e.g. the one a +// PyTorch wheel ships) and then knows neither the attributes nor the new +// cudaPointerAttributes field, while the driver always matches the device. +#if CUDA_VERSION >= 13040 +/*! \brief Whether the installed driver implements locality domains (CUDA 13.4+). */ +bool DriverHasLocalityDomains() { + static const bool supported = [] { + int version = 0; + return cuDriverGetVersion(&version) == CUDA_SUCCESS && version >= 13040; + }(); + return supported; +} + +int GetDriverAttribute(CUdevice_attribute attribute, int device_id) { + CUDA_DRIVER_CALL(cuInit(0)); + CUdevice device; + CUDA_DRIVER_CALL(cuDeviceGet(&device, device_id)); + int value = 0; + CUDA_DRIVER_CALL(cuDeviceGetAttribute(&value, attribute, device)); + return value; +} +#endif + +// The functions below back `tvm_tirx.backend.cuda.runtime.locality_domain`; each +// one is registered as `runtime.cuda.` for the Python function of the same name. + +/*! \brief Number of locality domains of a device; 1 when it has none or CUDA predates them. */ +int GetNumLocalityDomains(int device_id) { + // Queried once per device: every localized allocation and free needs it. + static std::mutex mutex; + static std::vector cache; // 0 = not queried yet + std::lock_guard lock(mutex); + TVM_FFI_CHECK(device_id >= 0, ValueError) << "Invalid CUDA device id " << device_id; + if (static_cast(device_id) < cache.size() && cache[device_id] > 0) { + return cache[device_id]; + } + int count = 0; +#if CUDA_VERSION >= 13040 + if (DriverHasLocalityDomains()) { + count = GetDriverAttribute(CU_DEVICE_ATTRIBUTE_LOCALITY_DOMAIN_COUNT, device_id); + } +#endif + if (count <= 0) count = 1; + if (static_cast(device_id) >= cache.size()) cache.resize(device_id + 1, 0); + cache[device_id] = count; + return count; +} + +/*! \brief Whether memory can be localized on a device: it has two or more locality domains. */ +bool IsLocalizationSupported(int device_id) { return GetNumLocalityDomains(device_id) > 1; } + +/*! \brief SMs guaranteed to each locality domain; the device's SM count without domains. */ +int GetNumSMsPerLocalityDomain(int device_id) { + int count = 0; +#if CUDA_VERSION >= 13040 + if (DriverHasLocalityDomains()) { + count = GetDriverAttribute(CU_DEVICE_ATTRIBUTE_LOCALITY_DOMAIN_MULTIPROCESSOR_COUNT, device_id); + } +#endif + if (count <= 0) { + TVM_FFI_CHECK_CUDA_ERROR( + cudaDeviceGetAttribute(&count, cudaDevAttrMultiProcessorCount, device_id)); + } + return count; +} + +/*! \brief The locality domain a tensor's memory belongs to, or None if it is not localized. */ +ffi::Optional GetLocalityDomainId(ffi::TensorView tensor) { + TVM_FFI_CHECK(tensor.device().device_type == kDLCUDA, ValueError) + << "Expected a tensor in CUDA device memory, got device type " << tensor.device().device_type; + if (tensor.data_ptr() == nullptr) return std::nullopt; +#if CUDA_VERSION >= 13040 + if (DriverHasLocalityDomains()) { + // The driver call needs a current context, which a fresh thread may lack. + ffi::CUDADeviceGuard device_guard(tensor.device().device_id); + TVM_FFI_CHECK_CUDA_ERROR(cudaSetDevice(tensor.device().device_id)); + int ordinal = -1; + CUDA_DRIVER_CALL(cuPointerGetAttribute(&ordinal, CU_POINTER_ATTRIBUTE_LOCALITY_DOMAIN_ORDINAL, + reinterpret_cast(tensor.data_ptr()))); + if (ordinal >= 0) return static_cast(ordinal); + } +#endif + return std::nullopt; +} + +#if CUDA_VERSION >= 13040 +/*! + * \brief The memory pools of CUDA locality domains, one per (device, domain), + * each created on first use. + * + * Pools live until process exit (they are never destroyed) so that tensors freed + * during static destruction still have a valid pool. The release threshold stays + * at its default of 0: freed memory goes back to the device at the next + * synchronization instead of staying reserved by the pool. + */ +class LocalityDomainPoolManager { + public: + static LocalityDomainPoolManager* Global() { + // Never destroyed, for the same reason as the pools. + static auto* manager = new LocalityDomainPoolManager(); + return manager; + } + + CUmemoryPool GetLocalityDomainPool(int device_id, int locality_domain_id) { + TVM_FFI_CHECK(device_id >= 0, ValueError) << "Invalid CUDA device id " << device_id; + TVM_FFI_CHECK(0 <= locality_domain_id && locality_domain_id < kMaxLocalityDomains, ValueError) + << "Invalid locality domain " << locality_domain_id; + std::lock_guard lock(mutex_); + if (static_cast(device_id) >= pools_.size()) { + pools_.resize(device_id + 1, std::vector(kMaxLocalityDomains, nullptr)); + } + CUmemoryPool& pool = pools_[device_id][locality_domain_id]; + if (pool == nullptr) { + CUmemPoolProps props{}; + props.allocType = CU_MEM_ALLOCATION_TYPE_PINNED; + props.handleTypes = CU_MEM_HANDLE_TYPE_NONE; + props.location.type = CU_MEM_LOCATION_TYPE_DEVICE_LOCALITY_DOMAIN; + props.location.localized.deviceId = static_cast(device_id); + props.location.localized.localityDomainId = static_cast(locality_domain_id); + CUmemoryPool created = nullptr; + CUDA_DRIVER_CALL(cuMemPoolCreate(&created, &props)); + pool = created; + } + return pool; + } + + private: + std::mutex mutex_; + /*! \brief pools_[device][domain]; nullptr until created. */ + std::vector> pools_; +}; +#endif + +} // namespace + class CUDADeviceAPI final : public DeviceAPI { public: void SetDevice(Device dev) final { TVM_FFI_CHECK_CUDA_ERROR(cudaSetDevice(dev.device_id)); } @@ -154,22 +337,46 @@ class CUDADeviceAPI final : public DeviceAPI { } ffi::Tensor Empty(ffi::ShapeView shape, DLDataType dtype, DLDevice device, ffi::Optional mem_scope = std::nullopt) final { - TVM_FFI_CHECK(!mem_scope.has_value() || mem_scope.value() == runtime::MemoryScope::Global(), - ValueError) - << "Unsupported memory scope: " - << mem_scope.value_or(runtime::MemoryScope::Global()).name(); + // `global` (the default) is a plain device allocation; + // `global.locality_domain.` places the tensor in locality domain k. + std::optional locality_domain; + if (mem_scope.has_value() && !(mem_scope.value() == runtime::MemoryScope::Global())) { + locality_domain = LocalityDomainOf(mem_scope.value()); + TVM_FFI_CHECK(locality_domain.has_value(), ValueError) + << "Unsupported memory scope: " << mem_scope.value().name(); + TVM_FFI_CHECK(device.device_type == kDLCUDA, ValueError) + << "Memory scope " << mem_scope.value().name() << " requires a CUDA device"; + int count = GetNumLocalityDomains(device.device_id); + TVM_FFI_CHECK(*locality_domain < count, ValueError) + << "Memory scope " << mem_scope.value().name() << " is out of range: CUDA device " + << device.device_id << " has " << count << " locality domain(s)"; + // Without locality domains, or without CUDA 13.4 support for them, domain 0 + // is the whole device: allocate it like `global`. + if (count == 1) locality_domain = std::nullopt; + } struct Allocator { CUDADeviceAPI* api; + std::optional locality_domain; void AllocData(DLTensor* tensor) { + size_t nbytes = ffi::GetDataSize(*tensor); + if (locality_domain.has_value()) { + tensor->data = api->AllocLocalityDomainSpace(tensor->device, nbytes, *locality_domain); + return; + } size_t alignment = std::max(static_cast(kAllocAlignment), static_cast((tensor->dtype.bits / 8) * tensor->dtype.lanes)); - tensor->data = api->AllocDataSpace(tensor->device, ffi::GetDataSize(*tensor), alignment, - tensor->dtype); + tensor->data = api->AllocDataSpace(tensor->device, nbytes, alignment, tensor->dtype); + } + void FreeData(DLTensor* tensor) { + if (locality_domain.has_value()) { + api->FreeLocalityDomainSpace(tensor->device, tensor->data); + return; + } + api->FreeDataSpace(tensor->device, tensor->data); } - void FreeData(DLTensor* tensor) { api->FreeDataSpace(tensor->device, tensor->data); } }; - return ffi::Tensor::FromNDAlloc(Allocator{this}, shape, dtype, device); + return ffi::Tensor::FromNDAlloc(Allocator{this, locality_domain}, shape, dtype, device); } private: @@ -190,6 +397,42 @@ class CUDADeviceAPI final : public DeviceAPI { return ret; } + /*! \brief Allocate in locality domain `domain` of a device with several domains. */ + void* AllocLocalityDomainSpace(Device dev, size_t nbytes, int domain) { +#if CUDA_VERSION >= 13040 + ffi::CUDADeviceGuard device_guard(dev.device_id); + // Ensure a current context even when the guard did not switch devices: the + // driver calls below need one, and this may be the first CUDA call. Since + // CUDA 12.0, cudaSetDevice initializes the device's primary context. + TVM_FFI_CHECK_CUDA_ERROR(cudaSetDevice(dev.device_id)); + CUdeviceptr ret; + CUDA_DRIVER_CALL(cuMemAllocFromPoolAsync( + &ret, nbytes, + LocalityDomainPoolManager::Global()->GetLocalityDomainPool(dev.device_id, domain), + /*hStream=*/nullptr)); + // Like cudaMalloc, the allocation is usable from any stream once this returns. + CUDA_DRIVER_CALL(cuStreamSynchronize(nullptr)); + return reinterpret_cast(ret); +#else + TVM_FFI_THROW(InternalError) + << "A device reports several locality domains only with CUDA 13.4+"; +#endif + } + + /*! \brief Free memory from AllocLocalityDomainSpace. */ + void FreeLocalityDomainSpace(Device dev, void* ptr) { + // As in FreeDataSpace, skip the free while unwinding from a sticky CUDA error. + if (std::uncaught_exceptions() && cudaPeekAtLastError() == cudaErrorIllegalAddress) return; + // cudaFree of cudaMalloc memory waits for all work on the device, but + // cudaFree of pool memory does not. Synchronize the device first so that, + // as for global memory, no kernel on any stream still uses the memory. + { + ffi::CUDADeviceGuard device_guard(dev.device_id); + TVM_FFI_CHECK_CUDA_ERROR(cudaDeviceSynchronize()); + } + FreeDataSpace(dev, ptr); + } + void FreeDataSpace(Device dev, void* ptr) { if (std::uncaught_exceptions() && cudaPeekAtLastError() == cudaErrorIllegalAddress) { // For most CUDA calls, an error from an API call will be @@ -420,6 +663,16 @@ TVM_FFI_STATIC_INIT_BLOCK() { refl::GlobalDef().def("runtime.GetCudaDeviceCount", GetCudaDeviceCount); } +TVM_FFI_STATIC_INIT_BLOCK() { + namespace refl = tvm::ffi::reflection; + RegisterLocalityDomainScopes(); + refl::GlobalDef() + .def("runtime.cuda.GetNumLocalityDomains", GetNumLocalityDomains) + .def("runtime.cuda.IsLocalizationSupported", IsLocalizationSupported) + .def("runtime.cuda.GetNumSMsPerLocalityDomain", GetNumSMsPerLocalityDomain) + .def("runtime.cuda.GetLocalityDomainId", GetLocalityDomainId); +} + #if (CUDA_VERSION >= 12000) /** * \brief FFI wrapper for cuTensorMapEncodeTiled. diff --git a/tests/python/backend/cuda/test_locality_domain.py b/tests/python/backend/cuda/test_locality_domain.py new file mode 100644 index 000000000..f7639e96b --- /dev/null +++ b/tests/python/backend/cuda/test_locality_domain.py @@ -0,0 +1,378 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Locality-domain memory scopes of the CUDA device API.""" + +import numpy as np +import pytest +import tvm_ffi + +import tvm_tirx +import tvm_tirx.testing +from tvm_tirx.backend.cuda.runtime import locality_domain +from tvm_tirx.script import tirx as T +from tvm_tirx.testing import env + + +def test_locality_domain_scope_names(): + for k in range(locality_domain.MAX_LOCALITY_DOMAINS): + assert locality_domain.locality_domain_scope(k) == f"global.locality_domain.{k}" + with pytest.raises(ValueError): + locality_domain.locality_domain_scope(locality_domain.MAX_LOCALITY_DOMAINS) + with pytest.raises(ValueError): + locality_domain.locality_domain_scope(-1) + + +@pytest.mark.skipif( + tvm_ffi.get_global_func("runtime.cuda.GetNumLocalityDomains", allow_missing=True) is None, + reason="TVM-TIRx is built without the CUDA runtime", +) +def test_locality_domain_scopes_registered(): + """The CUDA runtime registers the scopes, even on a machine without a GPU.""" + for k in range(locality_domain.MAX_LOCALITY_DOMAINS): + assert tvm_tirx.runtime.MemoryScope(locality_domain.locality_domain_scope(k)).name == ( + f"global.locality_domain.{k}" + ) + with pytest.raises(ValueError): + tvm_tirx.runtime.MemoryScope( + f"global.locality_domain.{locality_domain.MAX_LOCALITY_DOMAINS}" + ) + + +@pytest.mark.gpu +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") +def test_locality_domain_topology(): + dev = tvm_tirx.cuda(0) + count = locality_domain.get_num_locality_domains(dev) + per_domain = locality_domain.get_num_sms_per_locality_domain(dev) + assert count >= 1 + assert 0 < per_domain * count <= dev.multi_processor_count + assert locality_domain.is_localization_supported(dev) == (count > 1) + # A device id works in place of a Device, as in torch.cuda.green_contexts. + assert locality_domain.get_num_locality_domains(0) == count + assert locality_domain.get_num_sms_per_locality_domain(0) == per_domain + assert locality_domain.is_localization_supported(0) == (count > 1) + with pytest.raises(ValueError, match="CUDA device"): + locality_domain.get_num_locality_domains(tvm_tirx.cpu(0)) + + +@pytest.mark.gpu +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") +def test_locality_domain_allocation_roundtrip(): + """Each scope places memory in its domain; plain `global` memory is not localized.""" + dev = tvm_tirx.cuda(0) + count = locality_domain.get_num_locality_domains(dev) + plain = tvm_tirx.runtime.empty((1024,), "float32", dev) + assert locality_domain.get_locality_domain_id(plain) is None + for k in range(count): + data = np.arange(4096, dtype=np.float32) + k + x = tvm_tirx.runtime.tensor(data, dev, mem_scope=locality_domain.locality_domain_scope(k)) + np.testing.assert_array_equal(x.numpy(), data) + y = tvm_tirx.runtime.empty((4096,), "float32", dev, mem_scope=f"global.locality_domain.{k}") + y.copyfrom(x) + np.testing.assert_array_equal(y.numpy(), data) + z = plain.copyto(dev, mem_scope=locality_domain.locality_domain_scope(k)) + if count > 1: + # On a device without domains, domain 0 is the whole device and reports None. + assert locality_domain.get_locality_domain_id(x) == k + assert locality_domain.get_locality_domain_id(y) == k + assert locality_domain.get_locality_domain_id(z) == k + + +@pytest.mark.gpu +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") +@pytest.mark.parametrize("shape", [(0,), (2, 0, 3)]) +def test_locality_domain_id_empty_tensor(shape): + """Empty CUDA tensors have no localized allocation to query.""" + tensor = tvm_tirx.runtime.empty(shape, "float32", tvm_tirx.cuda(0)) + assert locality_domain.get_locality_domain_id(tensor) is None + + +@pytest.mark.gpu +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") +def test_locality_domain_allocation_as_first_cuda_call(): + """A localized allocation works as the very first CUDA operation of a process.""" + import subprocess + import sys + + code = ( + "import tvm_tirx\n" + "dev = tvm_tirx.cuda(0)\n" + "x = tvm_tirx.runtime.empty((256,), 'float32', dev, mem_scope='global.locality_domain.0')\n" + "print('ok', x.shape)\n" + ) + out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=False) + assert out.returncode == 0, out.stderr + assert out.stdout.startswith("ok") + + +@pytest.mark.gpu +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") +def test_locality_domain_out_of_range(): + dev = tvm_tirx.cuda(0) + count = locality_domain.get_num_locality_domains(dev) + if count >= locality_domain.MAX_LOCALITY_DOMAINS: + pytest.skip("every registered scope is a valid domain on this device") + with pytest.raises(ValueError, match=f"has {count} locality domain"): + tvm_tirx.runtime.empty( + (16,), "float32", dev, mem_scope=locality_domain.locality_domain_scope(count) + ) + + +@pytest.mark.gpu +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") +def test_locality_domain_free_returns_memory(): + """Freed localized tensors go back to the device rather than staying in the pool.""" + dev = tvm_tirx.cuda(0) + nbytes = 1 << 30 + scope = locality_domain.locality_domain_scope(locality_domain.get_num_locality_domains(dev) - 1) + + def run_and_check(): + for _ in range(3): # warm up the pool and the allocator + tvm_tirx.runtime.empty((nbytes // 4,), "float32", dev, mem_scope=scope) + before = dev.available_global_memory + for _ in range(8): + x = tvm_tirx.runtime.empty((nbytes // 4,), "float32", dev, mem_scope=scope) + del x + dev.sync() + assert dev.available_global_memory >= before - nbytes + + tvm_tirx.testing.run_with_gpu_lock(run_and_check) + + +@pytest.mark.gpu +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") +def test_locality_domain_tensor_freed_while_kernel_runs(): + """Dropping a tensor right after launching a kernel on it is safe. + + The free waits for the kernel, so the memory is not handed out again, + or returned to the device, while the kernel still writes it. + """ + n, steps = 1 << 24, 256 + + @T.function + def bump(A: T.Tensor((n,), "int32")): + T.device_entry(launch=T.cuda.LaunchConfig(grid=n // 256, block=256)) + i = T.cuda.block_idx("x") * 256 + T.cuda.thread_idx("x") + for _ in T.serial(steps): + A[i] = A[i] + 1 + + mod = tvm_tirx.compile( + tvm_tirx.IRModule({"main": bump}), + target=tvm_tirx.target.Target({"kind": "cuda", "arch": env.cuda_arch()}), + pipeline="tirx", + ) + dev = tvm_tirx.cuda(0) + + def run_and_check(): + for k in range(locality_domain.get_num_locality_domains(dev)): + scope = locality_domain.locality_domain_scope(k) + for _ in range(4): + x = tvm_tirx.runtime.empty((n,), "int32", dev, mem_scope=scope) + mod(x) + del x # no synchronization: the kernel may still be running + y = tvm_tirx.runtime.tensor(np.zeros(n, np.int32), dev, mem_scope=scope) + mod(y) + np.testing.assert_array_equal(y.numpy(), np.full(n, steps, np.int32)) + dev.sync() + + tvm_tirx.testing.run_with_gpu_lock(run_and_check) + + +@pytest.mark.gpu +@pytest.mark.skipif(not env.has_cuda(), reason="CUDA GPU not available") +def test_locality_domain_tensor_in_kernel_and_dlpack(): + """Localized tensors are ordinary CUDA tensors for kernels and DLPack consumers.""" + torch = pytest.importorskip("torch") + + @T.function + def double(A: T.Tensor((1024,), "float32"), B: T.Tensor((1024,), "float32")): + T.device_entry(launch=T.cuda.LaunchConfig(grid=4, block=256)) + i = T.cuda.block_idx("x") * 256 + T.cuda.thread_idx("x") + B[i] = A[i] * T.float32(2) + + mod = tvm_tirx.compile( + tvm_tirx.IRModule({"main": double}), + target=tvm_tirx.target.Target({"kind": "cuda", "arch": env.cuda_arch()}), + pipeline="tirx", + ) + dev = tvm_tirx.cuda(0) + last = locality_domain.get_num_locality_domains(dev) - 1 + data = np.random.default_rng(0).standard_normal(1024).astype(np.float32) + + def run_and_check(): + a = tvm_tirx.runtime.tensor(data, dev, mem_scope=locality_domain.locality_domain_scope(0)) + b = tvm_tirx.runtime.empty( + (1024,), "float32", dev, mem_scope=locality_domain.locality_domain_scope(last) + ) + mod(a, b) + np.testing.assert_array_equal(b.numpy(), data * 2) + t = torch.from_dlpack(b) + assert t.is_cuda + np.testing.assert_array_equal(t.cpu().numpy(), data * 2) + # The query accepts any DLPack tensor, e.g. torch's own (plain) allocations. + assert locality_domain.get_locality_domain_id(torch.zeros(4, device="cuda")) is None + + tvm_tirx.testing.run_with_gpu_lock(run_and_check) + + +@pytest.mark.gpu +@pytest.mark.skipif( + not env.has_cuda_locality_domain_nvrtc_support(), + reason="needs multiple CUDA locality domains and NVRTC 13.5+", +) +def test_locality_domain_aware_kernel(): + """Each CTA writes the half of the output that lives in the domain it runs on.""" + dev = tvm_tirx.cuda(0) + + @T.function + def fill( + O0: T.Tensor((4096,), "int32"), + O1: T.Tensor((4096,), "int32"), + C: T.Tensor((2,), "int32"), + ): + T.device_entry(launch=T.cuda.LaunchConfig(grid=64, block=64)) + dom = T.cuda.mov_sreg(32, "locality_domain_id") + tx = T.cuda.thread_idx("x") + if tx == 0: + if dom == 0: + T.cuda.atomic_add(C.ptr_to([0]), T.int32(1)) + else: + T.cuda.atomic_add(C.ptr_to([1]), T.int32(1)) + for i in T.serial(64): + if dom == 0: + O0[i * 64 + tx] = dom + else: + O1[i * 64 + tx] = dom + + mod = tvm_tirx.compile( + tvm_tirx.IRModule({"main": fill}), + target=tvm_tirx.target.Target({"kind": "cuda", "arch": env.cuda_arch()}), + pipeline="tirx", + ) + + def run_and_check(): + o = [ + tvm_tirx.runtime.tensor( + np.full(4096, -1, np.int32), dev, mem_scope=locality_domain.locality_domain_scope(k) + ) + for k in range(2) + ] + c = tvm_tirx.runtime.tensor(np.zeros(2, np.int32), dev) + mod(o[0], o[1], c) + ctas = c.numpy() + assert ctas.sum() == 64 + for k in range(2): + got = o[k].numpy() + # A domain that received no CTA leaves its output untouched. + assert (got == (k if ctas[k] else -1)).all() + + tvm_tirx.testing.run_with_gpu_lock(run_and_check) + + +def _domain_queue_reader(n_words, chunk_words, grid, threads=256): + """A persistent kernel: CTAs read the buffer of their own domain, then help the other. + + B lives in locality domain k and Ctr[k] is its chunk queue. Every word read + is XORed into Sink[0], so the result proves each chunk was read exactly once. + """ + steps = chunk_words // (threads * 4) + n_chunks = n_words // chunk_words + + @T.macro + def drain(B, Ctr, k, tx, chunk, c, r, acc): + if tx == 0: + chunk[0] = T.cuda.atomic_add(Ctr.ptr_to([k]), T.uint32(1)) + T.cuda.cta_sync() + c[0] = chunk[0] + T.cuda.cta_sync() + while c[0] < T.uint32(n_chunks): + for i in T.unroll(steps): + idx = T.Cast("int32", c[0]) * chunk_words + (i * threads + tx) * 4 + T.ptx.ld.global_.cs.v4.u32(r[0], r[1], r[2], r[3], B.ptr_to([idx])) + acc[0] = acc[0] ^ r[0] ^ r[1] ^ r[2] ^ r[3] + if tx == 0: + chunk[0] = T.cuda.atomic_add(Ctr.ptr_to([k]), T.uint32(1)) + T.cuda.cta_sync() + c[0] = chunk[0] + T.cuda.cta_sync() + + @T.function + def read( + B0: T.Tensor((n_words,), "uint32"), + B1: T.Tensor((n_words,), "uint32"), + Ctr: T.Tensor((2,), "uint32"), + Sink: T.Tensor((1,), "uint32"), + ): + T.device_entry(launch=T.cuda.LaunchConfig(grid=grid, block=threads)) + dom = T.cuda.mov_sreg(32, "locality_domain_id") + tx = T.cuda.thread_idx("x") + chunk = T.alloc_shared([1], "uint32") + c = T.alloc_local([1], "uint32") + r = T.alloc_local([4], "uint32") + acc = T.alloc_local([1], "uint32") + acc[0] = T.uint32(0) + # The hardware decides which domain a CTA runs in, so a domain may get + # few CTAs or none: drain the own queue first, then the other one. + if dom == 0: + drain(B0, Ctr, 0, tx, chunk, c, r, acc) + drain(B1, Ctr, 1, tx, chunk, c, r, acc) + else: + drain(B1, Ctr, 1, tx, chunk, c, r, acc) + drain(B0, Ctr, 0, tx, chunk, c, r, acc) + T.ptx.red.global_.xor.b32(Sink.ptr_to([0]), acc[0]) + + return read + + +@pytest.mark.gpu +@pytest.mark.skipif( + not env.has_cuda_locality_domain_nvrtc_support(), + reason="needs multiple CUDA locality domains and NVRTC 13.5+", +) +@pytest.mark.parametrize("grid", ["full", 1]) +def test_locality_domain_work_queue_kernel(grid): + """A domain-aware persistent kernel reads every chunk exactly once, however CTAs land.""" + dev = tvm_tirx.cuda(0) + if locality_domain.get_num_locality_domains(dev) != 2: + pytest.skip("needs a GPU with exactly two locality domains") + n_words, chunk_words = 1 << 22, 1 << 14 + if grid == "full": + grid = dev.multi_processor_count * 2 + mod = tvm_tirx.compile( + tvm_tirx.IRModule({"main": _domain_queue_reader(n_words, chunk_words, grid)}), + target=tvm_tirx.target.Target({"kind": "cuda", "arch": env.cuda_arch()}), + pipeline="tirx", + ) + rng = np.random.default_rng(0) + data = [rng.integers(0, 1 << 32, n_words, dtype=np.uint32) for _ in range(2)] + + def run_and_check(): + b = [ + tvm_tirx.runtime.tensor( + data[k], dev, mem_scope=locality_domain.locality_domain_scope(k) + ) + for k in range(2) + ] + ctr = tvm_tirx.runtime.tensor(np.zeros(2, np.uint32), dev) + sink = tvm_tirx.runtime.tensor(np.zeros(1, np.uint32), dev) + mod(b[0], b[1], ctr, sink) + want = np.bitwise_xor.reduce(data[0]) ^ np.bitwise_xor.reduce(data[1]) + assert sink.numpy()[0] == want + # Each queue hands out all its chunks, then one end marker to every CTA. + assert (ctr.numpy() == n_words // chunk_words + grid).all() + + tvm_tirx.testing.run_with_gpu_lock(run_and_check)