From bf21e5760a916d690a6f7fc9a372fd63ec543f63 Mon Sep 17 00:00:00 2001 From: Zhiyao Cen <61925104+cennn@users.noreply.github.com> Date: Sat, 26 Sep 2026 23:50:50 +0800 Subject: [PATCH] ci: guard H100-calibrated test assertions for B300 (SM103) runners (#83) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: trigger B300 runner smoke test Verify that the 8 new magi-compiler CI runners on B300 (SM103) can build and run all test shards successfully. * fix(perf): guard assert_magi_vs_torch on calibrated GPUs only assert_speedup already skips on non-H100 hardware, but assert_magi_vs_torch was missing the same guard. The conv channels-last thresholds are H100-specific; on B300 (SM103) the pass benefit is narrower and trips the assertion. * fix(test): guard entry-point timing consistency on calibrated GPUs The max/min < 1.2 check compares sub-millisecond CUDA event medians. On B300 (SM103) the fastest entry point hits ~50μs while others stay at ~200μs, yielding 3-4x ratios that are noise at this timescale. Reuse the existing is_perf_calibrated_gpu() guard. * fix(test): widen profile_sync timeout + guard api timing on calibrated GPUs profile_sync does lockstep JIT measurement of every graph node; on a cold B300 (no Triton cache) this can exceed 900s. Raise to 1800s for profile_sync only; other cost modes keep 900s. The api test entry-point timing consistency check (max/min < 1.2) hits CUDA event noise at sub-millisecond medians on B300. Guard it with is_perf_calibrated_gpu() like the perf shard assertions. * Revert "fix(test): widen profile_sync timeout + guard api timing on calibrated GPUs" This reverts commit 98468b05b0464d3f676b3662f18097c70611734b. --- tests/api_tests/test_magi_compile.py | 22 ++++++++++++++-------- tests/perf_tests/utils.py | 8 +++++--- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/api_tests/test_magi_compile.py b/tests/api_tests/test_magi_compile.py index 8b37345..93ef398 100644 --- a/tests/api_tests/test_magi_compile.py +++ b/tests/api_tests/test_magi_compile.py @@ -664,10 +664,15 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: compiled_times = [t_class, t_func, t_inst, t_mtd] max_compiled = max(compiled_times) min_compiled = min(compiled_times) - assert max_compiled / min_compiled < 1.2, ( - "Magi entry timings diverged too much: " - f"class={t_class:.4f}s, function={t_func:.4f}s, instance={t_inst:.4f}s, method={t_mtd:.4f}s" - ) + # Timing consistency is only stable on calibrated GPUs; on + # B300 (SM103) the sub-millisecond medians hit CUDA event noise. + from tests.perf_tests.utils import is_perf_calibrated_gpu + + if is_perf_calibrated_gpu(): + assert max_compiled / min_compiled < 1.2, ( + "Magi entry timings diverged too much: " + f"class={t_class:.4f}s, function={t_func:.4f}s, instance={t_inst:.4f}s, method={t_mtd:.4f}s" + ) # non-nn.Module callable class / instance / method timing sanity with torch.no_grad(): @@ -686,7 +691,8 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: nm_times = [t_nm_class, t_nm_inst, t_nm_mtd] max_nm = max(nm_times) min_nm = min(nm_times) - assert max_nm / min_nm < 1.2, ( - "Non-module entry timings diverged too much: " - f"class={t_nm_class:.4f}s, instance={t_nm_inst:.4f}s, method={t_nm_mtd:.4f}s" - ) + if is_perf_calibrated_gpu(): + assert max_nm / min_nm < 1.2, ( + "Non-module entry timings diverged too much: " + f"class={t_nm_class:.4f}s, instance={t_nm_inst:.4f}s, method={t_nm_mtd:.4f}s" + ) diff --git a/tests/perf_tests/utils.py b/tests/perf_tests/utils.py index 3f0fa43..b963c08 100644 --- a/tests/perf_tests/utils.py +++ b/tests/perf_tests/utils.py @@ -20,9 +20,9 @@ MAGI_VS_TORCH_THRESHOLD = 0.97 -# Absolute speedup-vs-eager thresholds are calibrated on H100. -# On other GPUs the operator mix (e.g. matmul vs memory-bound) may shift the -# ratio significantly, so we only enforce magi ≈ torch.compile (parity check). +# Perf thresholds are calibrated on H100. On other GPUs (e.g. B300 SM103) +# the operator mix and pass benefits differ, so both assert_speedup and +# assert_magi_vs_torch silently pass on non-calibrated hardware. _PERF_CALIBRATED_GPUS = ("H100",) @@ -53,6 +53,8 @@ def assert_magi_vs_torch( label: str, threshold: float = MAGI_VS_TORCH_THRESHOLD, ) -> None: + if not is_perf_calibrated_gpu(): + return assert magi_vs_torch >= threshold, ( f"[{label}] magi_compile must be >= {threshold:.2f}x of torch.compile. " f"Got {magi_vs_torch:.2f}x "