Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/reference/python/backend.cuda.rst
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ Namespaces
* :doc:`tvm_tirx.backend.cuda.iket <backend.cuda.iket>`
* :doc:`tvm_tirx.backend.cuda.launch <backend.cuda.launch>`
* :doc:`tvm_tirx.backend.cuda.ptx <backend.cuda.ptx>`
* :doc:`tvm_tirx.backend.cuda.runtime.locality_domain <backend.cuda.runtime.locality_domain>`
* :doc:`tvm_tirx.backend.cuda.thread_scope <backend.cuda.thread_scope>`
* :doc:`tvm_tirx.backend.cuda.tile <backend.cuda.tile>`

Expand All @@ -53,6 +54,7 @@ Namespaces
backend.cuda.iket
backend.cuda.launch
backend.cuda.ptx
backend.cuda.runtime.locality_domain
backend.cuda.thread_scope
backend.cuda.tile

Expand Down
36 changes: 36 additions & 0 deletions docs/reference/python/backend.cuda.runtime.locality_domain.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
.. Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at

.. http://www.apache.org/licenses/LICENSE-2.0

.. Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.

tvm_tirx.backend.cuda.runtime.locality_domain
=============================================

.. automodule:: tvm_tirx.backend.cuda.runtime.locality_domain
:no-members:

.. currentmodule:: tvm_tirx.backend.cuda.runtime.locality_domain

Functions
---------

.. autosummary::
:toctree: generated/

get_num_locality_domains
is_localization_supported
get_num_sms_per_locality_domain
get_locality_domain_id
locality_domain_scope
2 changes: 2 additions & 0 deletions python/tvm_tirx/backend/cuda/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
"launch",
"op",
"ptx",
"runtime",
"script",
"target_tags",
"tile",
Expand Down Expand Up @@ -140,6 +141,7 @@ def __getattr__(name: str):
"op",
"ptx",
"register_backend",
"runtime",
"script",
"script_namespace",
"script_namespaces",
Expand Down
19 changes: 19 additions & 0 deletions python/tvm_tirx/backend/cuda/runtime/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""CUDA runtime extensions of TIRx."""

from . import locality_domain
22 changes: 22 additions & 0 deletions python/tvm_tirx/backend/cuda/runtime/_ffi_api.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""FFI APIs for tvm_tirx.backend.cuda.runtime"""

import tvm_ffi

# Exports functions registered in the runtime.cuda namespace.
tvm_ffi.init_ffi_api("runtime.cuda", __name__)
150 changes: 150 additions & 0 deletions python/tvm_tirx/backend/cuda/runtime/locality_domain.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""CUDA locality domains: per-domain device memory and topology queries.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

move to backend.cuda.runtime.locality_domain.py


Some GPUs (e.g. Rubin) split their SMs and device memory into locality domains;
an SM reads memory of its own domain faster than memory of another one. A
tensor is placed in domain ``k`` by allocating it with the memory scope
``global.locality_domain.<k>``. On a GPU with multiple locality domains::

from tvm_tirx.backend.cuda.runtime import locality_domain

dev = tvm_tirx.cuda(0)
for k in range(locality_domain.get_num_locality_domains(dev)):
scope = locality_domain.locality_domain_scope(k) # "global.locality_domain.<k>"
x = tvm_tirx.runtime.empty((n,), "float32", dev, mem_scope=scope)
assert locality_domain.get_locality_domain_id(x) == k

The query names and their semantics follow ``torch.cuda.green_contexts``
(``get_num_locality_domains``, ``is_localization_supported``) and torch's
``locality_domain_id``. Each query wraps the tvm-ffi function
``runtime.cuda.<Name>`` of the same name, e.g. ``get_num_locality_domains`` is
``runtime.cuda.GetNumLocalityDomains``.

Like ``cudaMalloc`` memory, a localized tensor is ready on every stream once it
is allocated, and freeing it waits for the work on every stream of the device.

A kernel reads the domain of the SM it runs on with
``T.cuda.mov_sreg(32, "locality_domain_id")`` (PTX ISA 9.5) and can then work on
the data of its own domain. Memory of any domain stays accessible from every
SM; only the access speed differs. A device without locality domains, or with
a CUDA build or driver older than 13.4, reports one domain:
``global.locality_domain.0`` then behaves like ``global``, and
``get_locality_domain_id`` returns None for its memory.
"""

import tvm_ffi

from tvm_tirx.runtime import Device

from . import _ffi_api

__all__ = [
"MAX_LOCALITY_DOMAINS",
"get_locality_domain_id",
"get_num_locality_domains",
"get_num_sms_per_locality_domain",
"is_localization_supported",
"locality_domain_scope",
]

#: Number of ``global.locality_domain.<k>`` memory scopes registered (k < this).
MAX_LOCALITY_DOMAINS = 8


def _device_id(device: Device | int) -> int:
if isinstance(device, int):
return device
if device.dlpack_device_type() != tvm_ffi.DLDeviceType.kDLCUDA:
raise ValueError(f"Expected a CUDA device, got {device}")
return device.index


def get_num_locality_domains(device: Device | int = 0) -> int:
"""Return the number of locality domains of a CUDA device.

A device without locality domains, or a CUDA driver older than 13.4,
reports 1.

Parameters
----------
device : tvm_tirx.runtime.Device or int
A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id.
"""
return int(_ffi_api.GetNumLocalityDomains(_device_id(device)))


def is_localization_supported(device: Device | int = 0) -> bool:
"""Return whether memory can be localized on a CUDA device.

True when the CUDA driver supports locality domains and the device has two
or more of them.

Parameters
----------
device : tvm_tirx.runtime.Device or int
A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id.
"""
return bool(_ffi_api.IsLocalizationSupported(_device_id(device)))


def get_num_sms_per_locality_domain(device: Device | int = 0) -> int:
"""Return the number of SMs guaranteed to each locality domain of a CUDA device.

A device without locality domains reports all of its SMs. On a device with
domains, SMs beyond ``num_domains * sms_per_domain`` are supplemental SMs; a
kernel tells them apart with
``T.cuda.mov_sreg(32, "locality_domain_supplemental_sm")``.

Parameters
----------
device : tvm_tirx.runtime.Device or int
A CUDA device, e.g. ``tvm_tirx.cuda(0)``, or its device id.
"""
return int(_ffi_api.GetNumSMsPerLocalityDomain(_device_id(device)))


def get_locality_domain_id(tensor) -> int | None:
"""Return the locality domain a CUDA tensor's memory belongs to.

Returns None for memory that is not localized, e.g. memory allocated with
the plain ``global`` scope, or with ``global.locality_domain.0`` on a device
that reports a single locality domain.

Parameters
----------
tensor : tvm_tirx.runtime.Tensor or any DLPack-compatible tensor
A tensor in CUDA device memory.
"""
domain = _ffi_api.GetLocalityDomainId(tensor)
return None if domain is None else int(domain)


def locality_domain_scope(locality_domain_id: int) -> str:
"""Return the memory scope that places an allocation in a locality domain.

Parameters
----------
locality_domain_id : int
The locality domain, in ``[0, get_num_locality_domains(device))``.
"""
if not 0 <= locality_domain_id < MAX_LOCALITY_DOMAINS:
raise ValueError(
f"locality domain {locality_domain_id} is outside the registered scopes "
f"[0, {MAX_LOCALITY_DOMAINS})"
)
return f"global.locality_domain.{locality_domain_id}"
26 changes: 26 additions & 0 deletions python/tvm_tirx/testing/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
"has_cuda_arch",
# version / capability
"has_cuda_compute",
"has_cuda_locality_domain_nvrtc_support",
"has_cudagraph",
"has_gpu",
"has_llvm",
Expand Down Expand Up @@ -221,6 +222,31 @@ def has_nvcc_version(major: int, minor: int = 0, release: int = 0) -> bool:
return has_cuda() and _nvcc_version() >= (major, minor, release)


@functools.cache
def has_cuda_locality_domain_nvrtc_support() -> bool:
"""True if locality-domain kernels can run with the default NVRTC compiler.

Requires a CUDA runtime and an sm_100+ device with multiple locality domains,
plus loaded NVRTC 13.5+ for the PTX 9.5 locality-domain special registers.
This probes kernel-test capability, not just localized allocation support.
Missing bindings, libraries, or failed capability queries return False.
"""
try:
if not has_cuda_compute(10):
return False
get_num_domains = tvm_ffi.get_global_func(
"runtime.cuda.GetNumLocalityDomains", allow_missing=True
)
if get_num_domains is None or get_num_domains(0) < 2:
return False
from cuda.bindings import nvrtc # pylint: disable=import-outside-toplevel

status, major, minor = nvrtc.nvrtcVersion()
return status == nvrtc.nvrtcResult.NVRTC_SUCCESS and (major, minor) >= (13, 5)
except Exception: # pylint: disable=broad-except
return False


@functools.cache
def _llvm_version_major() -> int:
"""Return the major LLVM version, or 0 if LLVM is unavailable."""
Expand Down
Loading
Loading