From 7877ab37cbe79124a495152dd3d06461265dfaeb Mon Sep 17 00:00:00 2001 From: cangtianhuang <1903374751@qq.com> Date: Thu, 16 Jul 2026 13:49:51 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Fix=20wrong=20classify?= =?UTF-8?q?,=20use=20report=5Fcompare=5Ferror?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tester/accuracy.py | 14 +------- tester/accuracy_stable.py | 56 +++----------------------------- tester/base.py | 8 +++++ tester/paddle_cinn_vs_dygraph.py | 13 ++------ tester/paddle_device_vs_cpu.py | 12 +++---- tester/paddle_device_vs_gpu.py | 25 ++++---------- 6 files changed, 27 insertions(+), 101 deletions(-) diff --git a/tester/accuracy.py b/tester/accuracy.py index cb13385e..3c1be418 100644 --- a/tester/accuracy.py +++ b/tester/accuracy.py @@ -334,20 +334,8 @@ def compare_paddle_and_torch(paddle_tensor, torch_tensor, idx=0) -> bool: paddle_tensor, torch_tensor, atol=self.get_atol(), rtol=self.get_rtol() ) except Exception as err: - err_str = str(err) phase = "backward" if self.is_backward else "forward" - if err_str.startswith("[torch_assert_OOM]"): - print( - f"[oom] phase={phase} idx={idx} {self.api_config.config}\n{err_str}", - flush=True, - ) - write_to_log("oom", self.api_config.config) - else: - print( - f"[paddle_accuracy] phase={phase} idx={idx} {self.api_config.config}\n{err_str}", - flush=True, - ) - write_to_log("paddle_accuracy", self.api_config.config) + self.report_compare_error(err, f"{phase} idx={idx}") if self.exit_on_error: raise return False diff --git a/tester/accuracy_stable.py b/tester/accuracy_stable.py index 21c72190..9f331dad 100644 --- a/tester/accuracy_stable.py +++ b/tester/accuracy_stable.py @@ -403,19 +403,7 @@ def compare(self, input1, input2, comp): try: self.assert_accuracy(input1, input2, comp) except Exception as err: - err_str = str(err) - if err_str.startswith("[torch_assert_OOM]"): - print( - f"[oom] comp={comp} {self.api_config.config}\n{err_str}", - flush=True, - ) - write_to_log("oom", self.api_config.config) - else: - print( - f"[paddle_accuracy] comp={comp} {self.api_config.config}\n{err_str}", - flush=True, - ) - write_to_log("paddle_accuracy", self.api_config.config) + self.report_compare_error(err, f"comp={comp}") return else: print( @@ -450,19 +438,7 @@ def compare(self, input1, input2, comp): try: self.assert_accuracy(item1, item2, comp, idx) except Exception as err: - err_str = str(err) - if err_str.startswith("[torch_assert_OOM]"): - print( - f"[oom] comp={comp} {self.api_config.config}\n{err_str}", - flush=True, - ) - write_to_log("oom", self.api_config.config) - else: - print( - f"[paddle_accuracy] comp={comp} {self.api_config.config}\n{err_str}", - flush=True, - ) - write_to_log("paddle_accuracy", self.api_config.config) + self.report_compare_error(err, f"comp={comp} idx={idx}") return elif not isinstance(item1, (paddle.Tensor, torch.Tensor)) and not isinstance( item2, (paddle.Tensor, torch.Tensor) @@ -470,19 +446,7 @@ def compare(self, input1, input2, comp): try: self.assert_accuracy(torch.tensor(item1), torch.tensor(item2), comp, idx) except Exception as err: - err_str = str(err) - if err_str.startswith("[torch_assert_OOM]"): - print( - f"[oom] comp={comp} {self.api_config.config}\n{err_str}", - flush=True, - ) - write_to_log("oom", self.api_config.config) - else: - print( - f"[paddle_accuracy] comp={comp} {self.api_config.config}\n{err_str}", - flush=True, - ) - write_to_log("paddle_accuracy", self.api_config.config) + self.report_compare_error(err, f"comp={comp} idx={idx}") return else: print( @@ -496,19 +460,7 @@ def compare(self, input1, input2, comp): try: self.assert_accuracy(torch.tensor(input1), torch.tensor(input2), comp) except Exception as err: - err_str = str(err) - if err_str.startswith("[torch_assert_OOM]"): - print( - f"[oom] comp={comp} {self.api_config.config}\n{err_str}", - flush=True, - ) - write_to_log("oom", self.api_config.config) - else: - print( - f"[paddle_accuracy] comp={comp} {self.api_config.config}\n{err_str}", - flush=True, - ) - write_to_log("paddle_accuracy", self.api_config.config) + self.report_compare_error(err, f"comp={comp}") return def assert_accuracy(self, tensor1, tensor2, comp, idx=0): diff --git a/tester/base.py b/tester/base.py index 285d9f61..6b34675d 100644 --- a/tester/base.py +++ b/tester/base.py @@ -68,6 +68,8 @@ def __getattr__(self, name): def classify_runtime_error(error_msg): error_msg_lower = error_msg.lower() + if error_msg.startswith("[torch_assert_OOM]"): + return "oom", False oom_markers = tuple(marker.lower() for marker in CUDA_OOM) + ( "cannot allocate memory", "std::bad_alloc", @@ -256,6 +258,12 @@ def report_runtime_error(self, err, default_log_type, phase="", allow_ignore_pad write_to_log(log_type, self.api_config.config) return log_type, fatal + def report_compare_error(self, err, phase="", default_log_type="paddle_accuracy"): + log_type, fatal = self.report_runtime_error(err, default_log_type, phase) + if fatal: + raise err + return log_type, fatal + def need_skip(self, paddle_only=False): # not support if "sparse" in self.api_config.api_name: diff --git a/tester/paddle_cinn_vs_dygraph.py b/tester/paddle_cinn_vs_dygraph.py index c5180b09..ee190539 100644 --- a/tester/paddle_cinn_vs_dygraph.py +++ b/tester/paddle_cinn_vs_dygraph.py @@ -298,11 +298,7 @@ def compare(self, dygraph_output, static_output, is_backward=False): try: self.paddle_assert_accuracy(dygraph_output, static_output) except Exception as err: - print( - f"[accuracy error] {backward_str}{self.api_config.config}\n{err!s}", - flush=True, - ) - write_to_log("paddle_accuracy", self.api_config.config) + self.report_compare_error(err, backward_str.strip()) return False elif isinstance(dygraph_output, (list, tuple)): if not isinstance(static_output, (list, tuple)): @@ -341,11 +337,8 @@ def compare(self, dygraph_output, static_output, is_backward=False): try: self.paddle_assert_accuracy(dygraph_item, static_item) except Exception as err: - print( - f"[accuracy error] {backward_str}{self.api_config.config}\n{err!s}", - flush=True, - ) - write_to_log("paddle_accuracy", self.api_config.config) + phase = f"{backward_str.strip()} idx={i}" if backward_str else f"idx={i}" + self.report_compare_error(err, phase) return False elif dygraph_output is None and static_output is None: pass diff --git a/tester/paddle_device_vs_cpu.py b/tester/paddle_device_vs_cpu.py index 5e3e39ef..30e58cff 100644 --- a/tester/paddle_device_vs_cpu.py +++ b/tester/paddle_device_vs_cpu.py @@ -3,7 +3,7 @@ import paddle import torch -from .api_config.log_writer import write_to_log +from .api_config.log_writer import has_terminal_log, write_to_log from .base import APITestBase @@ -125,11 +125,8 @@ def _compare_single_tensor(self, cpu_tensor, custom_tensor, tensor_name=""): return True except Exception as err: - error_msg = "[accuracy error]" - if tensor_name: - error_msg += f" {tensor_name}" - error_msg += f"\n{self.api_config.config}\n{err!s}" - print(error_msg, flush=True) + phase = f"compare {tensor_name}" if tensor_name else "compare" + self.report_compare_error(err, phase) return False def compare_outputs(self, cpu_output, custom_output): @@ -355,7 +352,8 @@ def test(self): write_to_log("pass", self.api_config.config) else: print("[Fail]", self.api_config.config, flush=True) - write_to_log("paddle_accuracy", self.api_config.config) + if not has_terminal_log(self.api_config.config): + write_to_log("paddle_accuracy", self.api_config.config) # 生成可复现的单测文件 if self.generate_failed_tests: try: diff --git a/tester/paddle_device_vs_gpu.py b/tester/paddle_device_vs_gpu.py index 202d27f4..57b748a6 100644 --- a/tester/paddle_device_vs_gpu.py +++ b/tester/paddle_device_vs_gpu.py @@ -185,11 +185,9 @@ def _run_paddle(self, device_type: str): return paddle_output, paddle_grads except Exception as e: - print( - f"[paddle {paddle_device_type} error] {self.api_config.config}: {e}", - flush=True, - ) - write_to_log("paddle_error", self.api_config.config) + _, fatal = self.report_runtime_error(e, "paddle_error", f"paddle {paddle_device_type}") + if fatal: + raise return None, None def _compare_with_downloaded(self, local_output, local_grads, downloaded_tensor): @@ -258,11 +256,7 @@ def _compare_with_downloaded(self, local_output, local_grads, downloaded_tensor) flush=True, ) except Exception as e: - print( - f"[compare] Forward accuracy check failed for {self.api_config.config}, error: {e}", - flush=True, - ) - write_to_log("paddle_accuracy", self.api_config.config) + self.report_compare_error(e, "download forward") return False # 对比Backward梯度（如果存在且Forward通过） @@ -306,10 +300,7 @@ def _compare_with_downloaded(self, local_output, local_grads, downloaded_tensor) flush=True, ) except Exception as e: - print( - f"[compare] Backward gradient check failed for {self.api_config.config}, error: {e}", - flush=True, - ) + self.report_compare_error(e, "download backward") return False print( @@ -320,11 +311,7 @@ def _compare_with_downloaded(self, local_output, local_grads, downloaded_tensor) return True except Exception as e: - print( - f"[compare] Comparison failed for {self.api_config.config}, error: {e}", - flush=True, - ) - write_to_log("paddle_accuracy", self.api_config.config) + self.report_compare_error(e, "download compare") return False def test(self): From ffa16477f5265a80ffe7aefa12350e1db8623892 Mon Sep 17 00:00:00 2001 From: cangtianhuang <1903374751@qq.com> Date: Thu, 16 Jul 2026 13:50:17 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=90=9B=20Fix=20numpy=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tester/api_config/config_analyzer.py | 158 ++++++++++++++++----------- 1 file changed, 93 insertions(+), 65 deletions(-) diff --git a/tester/api_config/config_analyzer.py b/tester/api_config/config_analyzer.py index 319eff6e..9da8ebce 100644 --- a/tester/api_config/config_analyzer.py +++ b/tester/api_config/config_analyzer.py @@ -1738,9 +1738,12 @@ def get_padding_offset(bsz, max_seq_len, seq_lens_this_time): vocab_size = numpy.random.randint(10, 1000) if isinstance(weight_config, TensorConfig) and weight_config.shape: vocab_size = weight_config.shape[0] - self.numpy_tensor = numpy.random.randint(0, vocab_size, size=self.shape).astype( - self.dtype - ) + if vocab_size == 0: + self.numpy_tensor = numpy.zeros(self.shape, dtype=self.dtype) + else: + self.numpy_tensor = numpy.random.randint( + 0, vocab_size, size=self.shape + ).astype(self.dtype) elif self.check_arg(api_config, 1, "weight"): if self.dtype.startswith("complex"): real_dtype = "float32" if self.dtype == "complex64" else "float64" @@ -1778,9 +1781,12 @@ def get_padding_offset(bsz, max_seq_len, seq_lens_this_time): soft_labels = soft_labels / soft_labels.sum(axis=1, keepdims=True) self.numpy_tensor = soft_labels.astype(self.dtype) else: - self.numpy_tensor = numpy.random.randint( - 0, num_classes, size=self.shape - ).astype(self.dtype) + if num_classes == 0: + self.numpy_tensor = numpy.zeros(self.shape, dtype=self.dtype) + else: + self.numpy_tensor = numpy.random.randint( + 0, num_classes, size=self.shape + ).astype(self.dtype) elif self.check_arg(api_config, 3, "weight"): self.numpy_tensor = numpy.random.random(size=self.shape) self.numpy_tensor = self.numpy_tensor / self.numpy_tensor.sum() @@ -2635,9 +2641,12 @@ def get_padding_offset(bsz, max_seq_len, seq_lens_this_time): if axis is None: axis = 0 inputs = self.get_arg(api_config, 0, "x") - self.numpy_tensor = numpy.random.randint( - 0, inputs.shape[axis], size=self.shape - ).astype(self.dtype) + if inputs.shape[axis] == 0: + self.numpy_tensor = numpy.zeros(self.shape, dtype=self.dtype) + else: + self.numpy_tensor = numpy.random.randint( + 0, inputs.shape[axis], size=self.shape + ).astype(self.dtype) elif api_config.api_name in {"paddle.Tensor.index_put", "paddle.index_put"}: if self.check_arg(api_config, 1, "indices") and not self.get_arg( @@ -2981,38 +2990,50 @@ def get_exponent_max(value, dtype_max, default_max=5): seqlen, topk = self.shape[0], self.shape[1] # Generate valid routemap vectorized for large seqlen routemap = numpy.full(self.shape, -1, dtype="int32") - # Each row randomly assigns 1~topk experts to random positions - n_assigned = numpy.random.randint(1, topk + 1, size=seqlen) - # For each possible n_assigned value, batch process all rows with that count - for n in range(1, topk + 1): - mask = n_assigned == n - count = int(mask.sum()) - if count == 0: - continue - # Generate random expert indices for these rows - expert_indices = numpy.array( - [ - numpy.random.choice(num_experts, size=n, replace=False) - for _ in range(count) - ], - dtype="int32", - ) - # Generate random positions for these rows - position_indices = numpy.array( - [ - numpy.random.choice(topk, size=n, replace=False) - for _ in range(count) - ], - dtype="int32", - ) - row_indices = numpy.where(mask)[0] - for j in range(n): - routemap[row_indices, position_indices[:, j]] = expert_indices[:, j] - self.numpy_tensor = routemap - # Update tokens_per_expert to match the generated routemap - tokens_count = [int(numpy.sum(routemap == e)) for e in range(num_experts)] - tokens_per_expert = self.get_arg(api_config, 5, "tokens_per_expert") - tokens_per_expert[:] = tokens_count + if topk == 0: + # 0-size topk dimension: routemap stays all -1 + self.numpy_tensor = routemap + tokens_per_expert = self.get_arg(api_config, 5, "tokens_per_expert") + if tokens_per_expert is not None: + tokens_per_expert[:] = [0] * num_experts + else: + # Each row randomly assigns 1~min(topk, num_experts) experts + max_assign = min(topk, num_experts) + n_assigned = numpy.random.randint(1, max_assign + 1, size=seqlen) + # For each possible n_assigned value, batch process all rows with that count + for n in range(1, max_assign + 1): + mask = n_assigned == n + count = int(mask.sum()) + if count == 0: + continue + # Generate random expert indices for these rows + expert_indices = numpy.array( + [ + numpy.random.choice(num_experts, size=n, replace=False) + for _ in range(count) + ], + dtype="int32", + ) + # Generate random positions for these rows + position_indices = numpy.array( + [ + numpy.random.choice(topk, size=n, replace=False) + for _ in range(count) + ], + dtype="int32", + ) + row_indices = numpy.where(mask)[0] + for j in range(n): + routemap[row_indices, position_indices[:, j]] = expert_indices[ + :, j + ] + self.numpy_tensor = routemap + # Update tokens_per_expert to match the generated routemap + tokens_count = [ + int(numpy.sum(routemap == e)) for e in range(num_experts) + ] + tokens_per_expert = self.get_arg(api_config, 5, "tokens_per_expert") + tokens_per_expert[:] = tokens_count # expert_prob_topk (arg3): float32, shape [seqlen, topk], value in [0, 1] elif self.check_arg(api_config, 3, "expert_prob_topk"): routemap_config = self.get_arg(api_config, 2, "expert_routemap_topk") @@ -3047,30 +3068,37 @@ def get_exponent_max(value, dtype_max, default_max=5): seqlen, topk = self.shape[0], self.shape[1] # Generate valid routemap vectorized for large seqlen routemap = numpy.full(self.shape, -1, dtype="int32") - n_assigned = numpy.random.randint(1, topk + 1, size=seqlen) - for n in range(1, topk + 1): - mask = n_assigned == n - count = int(mask.sum()) - if count == 0: - continue - expert_indices = numpy.array( - [ - numpy.random.choice(num_experts, size=n, replace=False) - for _ in range(count) - ], - dtype="int32", - ) - position_indices = numpy.array( - [ - numpy.random.choice(topk, size=n, replace=False) - for _ in range(count) - ], - dtype="int32", - ) - row_indices = numpy.where(mask)[0] - for j in range(n): - routemap[row_indices, position_indices[:, j]] = expert_indices[:, j] - self.numpy_tensor = routemap + if topk == 0: + # 0-size topk dimension: routemap stays all -1 + self.numpy_tensor = routemap + else: + max_assign = min(topk, num_experts) + n_assigned = numpy.random.randint(1, max_assign + 1, size=seqlen) + for n in range(1, max_assign + 1): + mask = n_assigned == n + count = int(mask.sum()) + if count == 0: + continue + expert_indices = numpy.array( + [ + numpy.random.choice(num_experts, size=n, replace=False) + for _ in range(count) + ], + dtype="int32", + ) + position_indices = numpy.array( + [ + numpy.random.choice(topk, size=n, replace=False) + for _ in range(count) + ], + dtype="int32", + ) + row_indices = numpy.where(mask)[0] + for j in range(n): + routemap[row_indices, position_indices[:, j]] = expert_indices[ + :, j + ] + self.numpy_tensor = routemap # zipped_expertwise_rowmap (arg1): int32, shape [seqlen, num_experts] # Needs to be valid rowmap based on routemap elif self.check_arg(api_config, 1, "zipped_expertwise_rowmap"): From 62a6360ec28eeb570c84cf52b8cc01aa489e2f14 Mon Sep 17 00:00:00 2001 From: cangtianhuang <1903374751@qq.com> Date: Thu, 16 Jul 2026 13:51:06 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E2=9C=A8=20Impl=20TE=20moe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tester/paddle_to_torch/rules.py | 171 +++++++++++++++++++++++++++++--- 1 file changed, 155 insertions(+), 16 deletions(-) diff --git a/tester/paddle_to_torch/rules.py b/tester/paddle_to_torch/rules.py index 62be9bd7..a8aa2c63 100644 --- a/tester/paddle_to_torch/rules.py +++ b/tester/paddle_to_torch/rules.py @@ -1906,13 +1906,116 @@ class Fp8QuantBlockwiseRule(BaseRule): Aligns with phi fp8_quant_blockwise kernel: quant_scale = fp8_max / amax (optionally power-of-2), stored scale = 1/quant_scale, quantized = cast(x * quant_scale, float8_e4m3fn). + + Reference implementation is selected via PADDLEAPITEST_FLPGA_IMPL env var: + - "te" (default): Transformer Engine Float8BlockQuantizer + - "torch": Manual torch scatter-op implementation """ def apply(self, paddle_api: str) -> ConvertResult: + import os + defaults_code, _map_code = self.apply_generic() - # Single function body: converter exec uses separate globals/locals, so - # nested defs cannot close over sibling locals. - core = """ + impl = os.environ.get("PADDLEAPITEST_FLPGA_IMPL", "te") + if impl == "torch": + core = self._torch_code() + else: + core = self._te_code() + code = Code( + preprocess=defaults_code, + core=core.splitlines(), + ) + return ConvertResult.success(paddle_api, code, is_torch_corresponding=False) + + @staticmethod + def _te_code() -> str: + return """ +import os as _os +_fp8_max = 448.0 +_eps = float(epsilon) if epsilon is not None else 0.0 +_input_transpose = bool(input_transpose) +_output_scale_transpose = bool(output_scale_transpose) +_return_transpose_only = bool(return_transpose_only) +_using_pow2_scale = bool(using_pow2_scale) +_using_ue8m0_scale = bool(using_ue8m0_scale) +_quant_method = quant_method +if _quant_method not in ("1x128", "128x128"): + raise ValueError(f"Unsupported quantization method: {_quant_method}") +if output_type != "e4m3": + raise ValueError(f"Unsupported output type: {output_type}") + +try: + import transformer_engine.pytorch as _te + from transformer_engine.pytorch.quantization import DType as _teDType +except Exception as _err: + raise RuntimeError( + "PADDLEAPITEST_FLPGA_IMPL=te: cannot import transformer_engine; " + f"error={type(_err).__name__}: {_err}" + ) from _err + +def _te_fp8_quant_blockwise(inp, eps, power2, scale_transpose, ue8m0, method): + import transformer_engine.pytorch as _te + from transformer_engine.pytorch.quantization import DType as _teDType + m, n = inp.shape + _block_dim = 1 if method == "1x128" else 2 + _quantizer = _te.Float8BlockQuantizer( + fp8_dtype=_teDType.kFloat8E4M3, + rowwise=True, + columnwise=False, + force_pow_2_scales=power2, + amax_epsilon=eps, + block_scaling_dim=_block_dim, + ) + _result = _quantizer.quantize(inp.to(torch.bfloat16) if inp.dtype not in (torch.bfloat16, torch.float16, torch.float32) else inp) + q = _result._rowwise_data.view(torch.float8_e4m3fn) + deq_scale = _result._rowwise_scale_inv # float32 + + # Handle scale transpose: + # TE 1D: already (scale_cols, m) = transposed format + # TE 2D: (sm, sn) = non-transposed format + if method == "1x128": + # TE gives (scale_cols, m) which IS the transposed layout + scale = deq_scale if scale_transpose else deq_scale.t().contiguous() + else: + # TE gives (sm, sn) which is NON-transposed layout + scale = deq_scale.t().contiguous() if scale_transpose else deq_scale + + # Handle ue8m0 packing: 4 float32 exponents -> 1 packed int32 + if ue8m0: + base = deq_scale.contiguous() if method == "1x128" else (deq_scale.t().contiguous() if scale_transpose else deq_scale.contiguous()) + # base is in the final layout orientation already; pack along last dim + # Actually re-derive from the non-transposed deq_scale for consistent packing + if method == "1x128": + _base = deq_scale.t().contiguous() # (m, scale_cols) + else: + _base = deq_scale.contiguous() # (sm, sn) + cols_s = _base.shape[-1] + pad_c = (4 - (cols_s % 4)) % 4 + if pad_c: + _base = torch.nn.functional.pad(_base, (0, pad_c)) + b = _base.reshape(*_base.shape[:-1], -1, 4) + safe = torch.clamp(b, min=torch.finfo(torch.float32).tiny) + exp = torch.floor(torch.log2(safe)).to(torch.int32) + 127 + exp = torch.clamp(exp, 0, 255) + packed = (exp[..., 0] | (exp[..., 1] << 8) | (exp[..., 2] << 16) | (exp[..., 3] << 24)).to(torch.int32) + scale = packed.transpose(0, 1).contiguous() if scale_transpose else packed.contiguous() + return q, scale + +if not _input_transpose: + result = _te_fp8_quant_blockwise(x, _eps, _using_pow2_scale, _output_scale_transpose, _using_ue8m0_scale, _quant_method) +else: + x_t = x.transpose(0, 1).contiguous() + q_t, s_t = _te_fp8_quant_blockwise(x_t, _eps, _using_pow2_scale, _output_scale_transpose, _using_ue8m0_scale, _quant_method) + if _return_transpose_only: + result = (q_t, s_t) + else: + q, s = _te_fp8_quant_blockwise(x, _eps, _using_pow2_scale, _output_scale_transpose, _using_ue8m0_scale, _quant_method) + result = (q, s, q_t, s_t) +""" + + @staticmethod + def _torch_code() -> str: + return """ _fp8_max = 448.0 _eps = float(epsilon) if epsilon is not None else 0.0 _input_transpose = bool(input_transpose) @@ -1932,12 +2035,27 @@ def _fp8_quant_blockwise_impl(inp, eps, power2, scale_transpose, ue8m0, method, q = torch.empty((m, n), dtype=torch.float8_e4m3fn, device=inp.device) if method == "1x128": scale_cols = (n + 127) // 128 - scale = torch.empty((scale_cols, m) if scale_transpose else (m, scale_cols), dtype=torch.float32, device=inp.device) else: sm, sn = (m + 127) // 128, (n + 127) // 128 - scale = torch.empty((sn, sm) if scale_transpose else (sm, sn), dtype=torch.float32, device=inp.device) + scale_cols = sn if scale_transpose else sm + # for 128x128: scale shape before transpose is (sm, sn) + scale_rows_128 = sm if ue8m0: - scale = torch.empty_like(scale, dtype=torch.int32) + if method == "1x128": + packed_cols = (scale_cols + 3) // 4 + scale = torch.empty((packed_cols, m) if scale_transpose else (m, packed_cols), dtype=torch.int32, device=inp.device) + else: + packed_sm = (sm + 3) // 4 if not scale_transpose else sm + packed_sn = (sn + 3) // 4 if scale_transpose else sn + if scale_transpose: + scale = torch.empty((packed_sn, sm), dtype=torch.int32, device=inp.device) + else: + scale = torch.empty((sm, packed_sn), dtype=torch.int32, device=inp.device) + else: + if method == "1x128": + scale = torch.empty((scale_cols, m) if scale_transpose else (m, scale_cols), dtype=torch.float32, device=inp.device) + else: + scale = torch.empty((sn, sm) if scale_transpose else (sm, sn), dtype=torch.float32, device=inp.device) return q, scale _need_chunk = (m * n * 4) > (2 * 1024 * 1024 * 1024) @@ -2087,10 +2205,7 @@ def _fp8_quant_blockwise_impl(inp, eps, power2, scale_transpose, ue8m0, method, q, s = _fp8_quant_blockwise_impl(x, _eps, _using_pow2_scale, _output_scale_transpose, _using_ue8m0_scale, _quant_method, _fp8_max) result = (q, s, q_t, s_t) """ - code = Code( - preprocess=defaults_code, - core=core.splitlines(), - ) + return ConvertResult.success(paddle_api, code, is_torch_corresponding=False) return ConvertResult.success(paddle_api, code, is_torch_corresponding=False) @@ -4303,10 +4418,16 @@ def apply(self, paddle_api: str) -> ConvertResult: if scale is None: scale_unzipped = torch.empty(0, dtype=torch.float32, device=_dev) else: - scale_fp = scale.float() if using_ue8m0_scale else scale - scale_unzipped = torch.zeros(total_rows, scale_fp.shape[1], dtype=scale_fp.dtype, device=_dev) - _valid_rows_mask = (_gather_src >= 0) - scale_unzipped[_valid_rows_mask] = scale_fp[_gather_src[_valid_rows_mask]] + if using_ue8m0_scale: + # ue8m0 scale is int32 packed; just scatter the packed rows as-is + scale_unzipped = torch.zeros(total_rows, scale.shape[1], dtype=scale.dtype, device=_dev) + _valid_rows_mask = (_gather_src >= 0) + scale_unzipped[_valid_rows_mask] = scale[_gather_src[_valid_rows_mask]] + else: + scale_fp = scale.float() + scale_unzipped = torch.zeros(total_rows, scale_fp.shape[1], dtype=scale_fp.dtype, device=_dev) + _valid_rows_mask = (_gather_src >= 0) + scale_unzipped[_valid_rows_mask] = scale_fp[_gather_src[_valid_rows_mask]] # build hidden_states_unzipped via index_select if do_gather: hidden_states_unzipped = torch.zeros(total_rows, token_dim, dtype=hidden_states.dtype, device=_dev) @@ -7931,7 +8052,19 @@ def apply(self, paddle_api: str) -> ConvertResult: _scale_out[_rs:_re, _cb] = _inv_scale del _act - result = [_output_fp8, _scale_out] + if _use_ue8m0: + # Pack 4 exponent columns into 1 int32 (ue8m0 format) + _log2_inv = torch.log2(_scale_out).round().to(torch.int32) + 127 + _log2_inv = _log2_inv.clamp(0, 254) + _pack_cols = _log2_inv.shape[-1] + _pad_c = (4 - (_pack_cols % 4)) % 4 + if _pad_c: + _log2_inv = torch.nn.functional.pad(_log2_inv, (0, _pad_c)) + _log2_inv = _log2_inv.reshape(_log2_inv.shape[0], -1, 4) + _scale_packed = (_log2_inv[..., 0] | (_log2_inv[..., 1] << 8) | (_log2_inv[..., 2] << 16) | (_log2_inv[..., 3] << 24)).to(torch.int32) + result = [_output_fp8, _scale_packed] + else: + result = [_output_fp8, _scale_out] elif op_name in ("fuse_stack_fp8_quant", "fuse_stack_transpose_fp8_quant"): # fuse_stack[_transpose]_fp8_quant(X_list, using_pow2_scaling, using_ue8m0_scale, output_scale_transpose) @@ -7984,9 +8117,15 @@ def apply(self, paddle_api: str) -> ConvertResult: _log2_inv = torch.log2(_inv_scale).round().to(torch.int32) + 127 _log2_inv = _log2_inv.clamp(0, 254) _scale_out = _log2_inv.unsqueeze(1).expand(-1, _TILE, -1).reshape(_out_rows, _num_col_blocks) + # Pack 4 exponent columns into 1 int32 (ue8m0 format) + _pack_cols = _scale_out.shape[-1] + _pad_c = (4 - (_pack_cols % 4)) % 4 + if _pad_c: + _scale_out = torch.nn.functional.pad(_scale_out, (0, _pad_c)) + _scale_out = _scale_out.reshape(_scale_out.shape[0], -1, 4) + _scale_out = (_scale_out[..., 0] | (_scale_out[..., 1] << 8) | (_scale_out[..., 2] << 16) | (_scale_out[..., 3] << 24)).to(torch.int32) if _output_scale_transpose: _scale_out = _scale_out.t().contiguous() - _scale_out = _scale_out.to(torch.int32) else: _scale_out = _inv_scale if _output_scale_transpose: From 68d7ceb46d98d9c2369deff65029a8ba3ce0525c Mon Sep 17 00:00:00 2001 From: cangtianhuang <1903374751@qq.com> Date: Fri, 17 Jul 2026 10:55:01 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Update=20gpu=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- engineV2.py | 157 ++++++++++++++++++--------- engineV4.py | 62 ++++++++++- tester/accuracy.py | 22 +++- tester/api_config/config_analyzer.py | 53 ++++----- tester/base.py | 84 ++++++++++++-- tester/runtime_config.py | 53 +++++++++ 6 files changed, 323 insertions(+), 108 deletions(-) create mode 100644 tester/runtime_config.py diff --git a/engineV2.py b/engineV2.py index ddc3f751..a6105b04 100644 --- a/engineV2.py +++ b/engineV2.py @@ -38,6 +38,7 @@ ) from tester.api_config.log_writer import * +from tester.runtime_config import TestRuntimeConfig, runtime_config_for_gpu os.environ["FLAGS_use_system_allocator"] = "1" os.environ["NVIDIA_TF32_OVERRIDE"] = "0" @@ -58,6 +59,7 @@ "bitwise_alignment", "exit_on_error", "use_gpu_mode", + "runtime_config", } DEVICE_TYPE = None @@ -71,6 +73,11 @@ FATAL_TORCH_EXIT_CODE = 97 MEMORY_WAIT_SECONDS = 10 MEMORY_WAIT_LOG_INTERVAL = 60 +MEMORY_WAIT_MAX_SECONDS = 300 + + +class GpuMemoryDeferred(Exception): + """Raised when a GPU-mode case should wait for more free memory.""" def cleanup(pool): @@ -457,6 +464,9 @@ def parse_bool(value): def _prepare_single_config_gpu(options): if options.test_cpu: + options.gpu_workers_per_gpu_map = {} + options.gpu_total_memory_map = {} + options.runtime_config = TestRuntimeConfig.from_options(options) return None gpu_ids = validate_gpu_options(options) @@ -465,6 +475,13 @@ def _prepare_single_config_gpu(options): f"single --api_config run supports exactly one GPU; got {len(gpu_ids)} GPUs: {gpu_ids}" ) os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_ids[0]) + try: + options.gpu_total_memory_map = {gpu_ids[0]: get_memory_info(gpu_ids[0])[0]} + except Exception: + options.gpu_total_memory_map = {} + options.gpu_workers_per_gpu_map = {gpu_ids[0]: 1} + options.runtime_config = TestRuntimeConfig.from_options(options) + options.runtime_config = runtime_config_for_gpu(options, gpu_ids[0]) return gpu_ids[0] @@ -573,6 +590,7 @@ def run_test_case(api_config_str, options): ) last_memory_log_time = 0 + memory_wait_start = time.time() while True: total_memory, used_memory = get_memory_info(gpu_id) free_memory = total_memory - used_memory @@ -589,8 +607,18 @@ def run_test_case(api_config_str, options): flush=True, ) last_memory_log_time = now + if ( + getattr(options, "use_gpu_mode", False) + and now - memory_wait_start >= MEMORY_WAIT_MAX_SECONDS + ): + raise GpuMemoryDeferred( + f"device {gpu_id} Memory wait timeout after {MEMORY_WAIT_MAX_SECONDS}s " + f"(free={free_memory:.1f} GB, required={options.required_memory:.1f} GB)" + ) time.sleep(MEMORY_WAIT_SECONDS) + runtime_config = runtime_config_for_gpu(options, gpu_id) + if options.show_runtime_status: total_memory, used_memory_before = get_memory_info(gpu_id) print( @@ -610,6 +638,7 @@ def run_test_case(api_config_str, options): test_class = _select_test_class(options) kwargs = {k: v for k, v in vars(options).items() if k in VALID_TEST_ARGS} + kwargs["runtime_config"] = runtime_config case = test_class(api_config, **kwargs) try: case.test() @@ -976,7 +1005,6 @@ def main(): options.use_cached_numpy = False os.environ["USE_CACHED_NUMPY"] = str(options.use_cached_numpy) os.environ["USE_GPU_MODE"] = str(options.use_gpu_mode) - os.environ["SKIP_GPU_CLEANUP"] = str(options.use_gpu_mode) if options.use_gpu_mode: print( "[gpu_mode] enabled: GPU tensor generation, GPU compare, and allocator reuse are active.", @@ -1051,6 +1079,7 @@ def main(): bitwise_alignment=options.bitwise_alignment, exit_on_error=options.exit_on_error, use_gpu_mode=options.use_gpu_mode, + runtime_config=options.runtime_config, ) else: case = test_class(api_config, test_amp=options.test_amp) @@ -1161,6 +1190,15 @@ def main(): return total_workers = sum(max_workers_per_gpu.values()) + gpu_total_memory_map = {} + for gpu_id in available_gpus: + try: + gpu_total_memory_map[gpu_id] = get_memory_info(gpu_id)[0] + except Exception: + pass + options.gpu_workers_per_gpu_map = dict(max_workers_per_gpu) + options.gpu_total_memory_map = gpu_total_memory_map + options.runtime_config = TestRuntimeConfig.from_options(options) print( f"Using {len(available_gpus)} GPU(s) with max workers per GPU: {max_workers_per_gpu}. Total workers: {total_workers}.", flush=True, @@ -1200,7 +1238,8 @@ def cleanup_handler(*args): for batch_start in range(0, len(api_configs), BATCH_SIZE): batch = api_configs[batch_start : batch_start + BATCH_SIZE] futures = {} - for config in batch: + + def schedule_config(config): # timeout = estimate_timeout(config) timeout = options.timeout future = pool.schedule( @@ -1210,67 +1249,79 @@ def cleanup_handler(*args): ) futures[future] = config - for future in as_completed(futures): - config = futures[future] - checkpoint_ready = True - try: - future.result() - if options.show_runtime_status or tested_case % 10000 == 0: - print(f"[info] Test case succeeded for {config}", flush=True) - except TimeoutError as err: - write_terminal_log("timeout", config) - print( - f"[timeout] {config}: {err}", - flush=True, - ) - except ProcessExpired as err: - # CUDA, OOM, and Torch fatal errors may also expire the subprocess; - # classify them by dedicated terminal log types instead of falling through to crash. - if err.exitcode == FATAL_CUDA_EXIT_CODE: - write_terminal_log("paddle_cuda", config) - print( - f"[paddle_cuda] {config}: {err}", - flush=True, - ) - elif err.exitcode == FATAL_OOM_EXIT_CODE: - write_terminal_log("oom", config) - print( - f"[oom] {config}: {err}", - flush=True, - ) - elif err.exitcode == FATAL_TORCH_EXIT_CODE: - write_terminal_log("torch_error", config) + for config in batch: + schedule_config(config) + + while futures: + done_futures = list(as_completed(futures)) + for future in done_futures: + config = futures.pop(future) + checkpoint_ready = True + try: + future.result() + if options.show_runtime_status or tested_case % 10000 == 0: + print(f"[info] Test case succeeded for {config}", flush=True) + except TimeoutError as err: + write_terminal_log("timeout", config) print( - f"[torch_error] {config}: {err}", + f"[timeout] {config}: {err}", flush=True, ) - elif err.exitcode in (-signal.SIGKILL, -signal.SIGTERM): + except ProcessExpired as err: + # CUDA, OOM, and Torch fatal errors may also expire the subprocess; + # classify them by dedicated terminal log types instead of falling through to crash. + if err.exitcode == FATAL_CUDA_EXIT_CODE: + write_terminal_log("paddle_cuda", config) + print( + f"[paddle_cuda] {config}: {err}", + flush=True, + ) + elif err.exitcode == FATAL_OOM_EXIT_CODE: + write_terminal_log("oom", config) + print( + f"[oom] {config}: {err}", + flush=True, + ) + elif err.exitcode == FATAL_TORCH_EXIT_CODE: + write_terminal_log("torch_error", config) + print( + f"[torch_error] {config}: {err}", + flush=True, + ) + elif err.exitcode in (-signal.SIGKILL, -signal.SIGTERM): + checkpoint_ready = False + print( + f"[warn] Worker was externally killed for {config} " + f"(exit={err.exitcode}); case will be retried on next run.", + flush=True, + ) + else: + write_terminal_log("paddle_crash", config) + print( + f"[paddle_crash] {config}: {err}", + flush=True, + ) + except GpuMemoryDeferred as err: checkpoint_ready = False print( - f"[warn] Worker was externally killed for {config} " - f"(exit={err.exitcode}); case will be retried on next run.", + f"[gpu_mode] Deferred {config}: {err}", flush=True, ) - else: - write_terminal_log("paddle_crash", config) + schedule_config(config) + except Exception as err: + write_terminal_log("config_parse", config) print( - f"[paddle_crash] {config}: {err}", - flush=True, - ) - except Exception as err: - write_terminal_log("config_parse", config) - print( - f"[config_parse] {config}: {err}", - flush=True, - ) - checkpoint_ready = False # checkpoint already written by write_terminal_log - if checkpoint_ready: - tested_case += 1 - if options.show_runtime_status or tested_case % 10000 == 0: - print( - f"[{tested_case}/{all_case}] Testing {config}", + f"[config_parse] {config}: {err}", flush=True, ) + checkpoint_ready = False + if checkpoint_ready: + tested_case += 1 + if options.show_runtime_status or tested_case % 10000 == 0: + print( + f"[{tested_case}/{all_case}] Testing {config}", + flush=True, + ) aggregate_logs() pool.close() pool.join() diff --git a/engineV4.py b/engineV4.py index 0eeb3086..5e6b5663 100644 --- a/engineV4.py +++ b/engineV4.py @@ -43,6 +43,7 @@ ) from tester.api_config.log_writer import * +from tester.runtime_config import TestRuntimeConfig, runtime_config_for_gpu os.environ["FLAGS_use_system_allocator"] = "1" os.environ["NVIDIA_TF32_OVERRIDE"] = "0" @@ -51,6 +52,11 @@ FATAL_OOM_EXIT_CODE = 98 FATAL_TORCH_EXIT_CODE = 97 + +class GpuMemoryDeferred(Exception): + """Raised when a GPU-mode case should wait for more free memory.""" + + VALID_TEST_ARGS = { "test_amp", "test_backward", @@ -67,8 +73,10 @@ "bitwise_alignment", "exit_on_error", "use_gpu_mode", + "runtime_config", } + SANITIZER_FORWARD_ARGS = { "accuracy", "paddle_only", @@ -211,6 +219,8 @@ def _worker_loop(slot_index, gpu_id, input_queue, result_queue, options): try: run_test_case(api_config_str, options) result_queue.put(("done", slot_index, api_config_str)) + except GpuMemoryDeferred as e: + result_queue.put(("deferred", slot_index, api_config_str, str(e))) except SystemExit: # run_test_case calls os._exit for CUDA errors, this shouldn't reach here # but if it does via sys.exit, let it propagate @@ -412,6 +422,15 @@ def __init__(self, available_gpus, max_workers_per_gpu, options): self.options = SimpleNamespace(**vars(options)) else: self.options = options + self.options.gpu_workers_per_gpu_map = dict(max_workers_per_gpu) + gpu_total_memory_map = {} + for gpu_id in available_gpus: + try: + gpu_total_memory_map[gpu_id] = get_memory_info(gpu_id)[0] + except Exception: + pass + self.options.gpu_total_memory_map = gpu_total_memory_map + self.options.runtime_config = TestRuntimeConfig.from_options(self.options) self.result_queue = mp.Queue() self.slots: list[WorkerSlot] = [] self._shutdown_event = threading.Event() @@ -1042,6 +1061,9 @@ def parse_bool(value): def _prepare_single_config_gpu(options): if options.test_cpu: + options.gpu_workers_per_gpu_map = {} + options.gpu_total_memory_map = {} + options.runtime_config = TestRuntimeConfig.from_options(options) return None gpu_ids = validate_gpu_options(options) @@ -1050,6 +1072,13 @@ def _prepare_single_config_gpu(options): f"single --api_config run supports exactly one GPU; got {len(gpu_ids)} GPUs: {gpu_ids}" ) os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_ids[0]) + try: + options.gpu_total_memory_map = {gpu_ids[0]: get_memory_info(gpu_ids[0])[0]} + except Exception: + options.gpu_total_memory_map = {} + options.gpu_workers_per_gpu_map = {gpu_ids[0]: 1} + options.runtime_config = TestRuntimeConfig.from_options(options) + options.runtime_config = runtime_config_for_gpu(options, gpu_ids[0]) return gpu_ids[0] @@ -1268,12 +1297,19 @@ def run_test_case(api_config_str, options): ) time.sleep(10) else: + message = ( + f"device {gpu_id} Memory wait timeout after {max_memory_wait * 10}s " + f"(free={free_memory:.1f} GB, required={options.required_memory:.1f} GB)" + ) + if getattr(options, "use_gpu_mode", False): + raise GpuMemoryDeferred(message) print( - f"{datetime.now()} device {gpu_id} Memory wait timeout after {max_memory_wait * 10}s. " - f"Proceeding anyway (free={free_memory:.1f} GB, required={options.required_memory:.1f} GB).", + f"{datetime.now()} {message}. Proceeding anyway.", flush=True, ) + runtime_config = runtime_config_for_gpu(options, gpu_id) + try: api_config = APIConfig(api_config_str) except Exception as err: @@ -1283,6 +1319,7 @@ def run_test_case(api_config_str, options): test_class = _select_test_class(options) kwargs = {k: v for k, v in vars(options).items() if k in VALID_TEST_ARGS} + kwargs["runtime_config"] = runtime_config case = test_class(api_config, **kwargs) try: case.test() @@ -1663,7 +1700,6 @@ def main(): options.use_cached_numpy = False os.environ["USE_CACHED_NUMPY"] = str(options.use_cached_numpy) os.environ["USE_GPU_MODE"] = str(options.use_gpu_mode) - os.environ["SKIP_GPU_CLEANUP"] = str(options.use_gpu_mode) if options.use_gpu_mode: print( "[gpu_mode] enabled: GPU tensor generation, GPU compare, and allocator reuse are active.", @@ -1757,6 +1793,7 @@ def main(): bitwise_alignment=options.bitwise_alignment, exit_on_error=options.exit_on_error, use_gpu_mode=options.use_gpu_mode, + runtime_config=options.runtime_config, ) else: case = test_class(api_config, test_amp=options.test_amp) @@ -1917,6 +1954,13 @@ def cleanup_handler(*args): while active_tasks > 0 or pending_dispatch: msg = pool.collect_one(timeout=5.0) if msg is None: + if pending_dispatch: + for slot in pool.idle_slots(): + if not pending_dispatch: + break + config = pending_dispatch.pop(0) + pool.dispatch(slot.index, config) + active_tasks += 1 continue # watchdog handles timeouts/crashes msg_type = msg[0] @@ -1951,7 +1995,7 @@ def cleanup_handler(*args): config = msg[2] exitcode = msg[3] if msg_type == "crashed" and len(msg) > 3 else None crash_source = msg[5] if msg_type == "crashed" and len(msg) > 5 else "worker" - worker_reusable = msg_type in ("done", "error") or ( + worker_reusable = msg_type in ("done", "error", "deferred") or ( msg_type == "crashed" and options.use_compute_sanitizer and crash_source == "child" @@ -1981,6 +2025,16 @@ def cleanup_handler(*args): if worker_reusable: pool.mark_idle(slot_idx) + + if msg_type == "deferred": + reason = msg[3] if len(msg) > 3 else "insufficient GPU memory" + pending_dispatch.append(config) + print( + f"[gpu_mode] Deferred {config}: {reason}", + flush=True, + ) + continue + tested_case += 1 if options.show_runtime_status or tested_case % 10000 == 0: diff --git a/tester/accuracy.py b/tester/accuracy.py index 3c1be418..c361b911 100644 --- a/tester/accuracy.py +++ b/tester/accuracy.py @@ -9,7 +9,7 @@ import yaml from .api_config.log_writer import write_to_log -from .base import APITestBase +from .base import APITestBase, gpu_mode_maybe_empty_cache from .paddle_to_torch import get_converter # from func_timeout import func_set_timeout @@ -17,14 +17,16 @@ class APITestAccuracy(APITestBase): def __init__(self, api_config, **kwargs): - super().__init__(api_config) + super().__init__(api_config, runtime_config=kwargs.get("runtime_config")) self.test_amp = kwargs.get("test_amp", False) self.atol = kwargs.get("atol", 0) self.rtol = kwargs.get("rtol", 0) self.test_tol = kwargs.get("test_tol", False) - self.exit_on_error = kwargs.get("exit_on_error", False) - self.bitwise_alignment = kwargs.get("bitwise_alignment", False) - self.use_gpu_mode = kwargs.get("use_gpu_mode", False) + self.exit_on_error = kwargs.get("exit_on_error", self.runtime_config.exit_on_error) + self.bitwise_alignment = kwargs.get( + "bitwise_alignment", self.runtime_config.bitwise_alignment + ) + self.use_gpu_mode = self.gpu_mode_config.enabled self.manual_threshold_config_file = kwargs.get("manual_threshold_config_file", "") self.manual_threshold_config = self._load_manual_threshold_config( self.manual_threshold_config_file @@ -266,7 +268,10 @@ def process_torch_outputs(obj): torch_out_grads = process_torch_outputs(torch_out_grads) gc.collect() - if not self.use_gpu_mode: + if self.use_gpu_mode: + self.clear_torch_tensor() + gpu_mode_maybe_empty_cache(self.gpu_mode_config, "before_paddle") + else: torch.cuda.empty_cache() try: @@ -325,6 +330,8 @@ def process_torch_outputs(obj): def compare_paddle_and_torch(paddle_tensor, torch_tensor, idx=0) -> bool: try: + if self.use_gpu_mode: + gpu_mode_maybe_empty_cache(self.gpu_mode_config, "before_compare") # if paddle_tensor.dtype == paddle.bfloat16: # paddle_tensor = paddle.cast(paddle_tensor, dtype="float32") # if torch_tensor.dtype == torch.bfloat16: @@ -444,6 +451,9 @@ def compare_paddle_and_torch(paddle_tensor, torch_tensor, idx=0) -> bool: # Forward check now pass. # Then do paddle backward and backward result check. + if self.use_gpu_mode: + del torch_output + gpu_mode_maybe_empty_cache(self.gpu_mode_config, "after_forward_compare") if torch_grad_success: self.is_backward = True try: diff --git a/tester/api_config/config_analyzer.py b/tester/api_config/config_analyzer.py index 9da8ebce..28f68afd 100644 --- a/tester/api_config/config_analyzer.py +++ b/tester/api_config/config_analyzer.py @@ -25,7 +25,6 @@ def __getattr__(self, name): USE_CACHED_NUMPY = os.getenv("USE_CACHED_NUMPY", "False").lower() == "true" TEST_NON_CONTIGUOUS = os.getenv("TEST_NON_CONTIGUOUS", "0").lower() in ("true", "1") USE_GPU_MODE = os.getenv("USE_GPU_MODE", "False").lower() == "true" -SKIP_GPU_CLEANUP = os.getenv("SKIP_GPU_CLEANUP", "False").lower() == "true" cached_numpy = {} AUTOGRAD_DTYPES = frozenset( ["float32", "float64", "float16", "complex64", "complex128", "bfloat16"] @@ -43,21 +42,8 @@ def _load_forward_only_apis(): FORWARD_ONLY_APIS = _load_forward_only_apis() -def _env_bool(name, default=False): - return os.getenv(name, str(default)).lower() in ("true", "1", "yes", "y") - - -def set_gpu_mode(enabled): - os.environ["USE_GPU_MODE"] = str(bool(enabled)) - os.environ["SKIP_GPU_CLEANUP"] = str(bool(enabled)) - - def is_gpu_mode(): - return _env_bool("USE_GPU_MODE", USE_GPU_MODE) - - -def should_skip_gpu_cleanup(): - return _env_bool("SKIP_GPU_CLEANUP", SKIP_GPU_CLEANUP) + return os.getenv("USE_GPU_MODE", str(USE_GPU_MODE)).lower() in ("true", "1", "yes", "y") def _shape_tuple(shape): @@ -189,7 +175,6 @@ def __init__(self, shape, dtype, place=None, is_contiguous=True, strides=None): self.numpy_tensor = None self.paddle_tensor = None self.torch_tensor = None - self.gpu_source_tensor = None self.shuffle_dims = None def __deepcopy__(self, memo): @@ -249,7 +234,7 @@ def numel(self): def get_cached_numpy(self, dtype, shape, generation_kind="input", scale=1.2): return get_cached_numpy_array(dtype, shape, generation_kind=generation_kind, scale=scale) - def _use_gpu(self, dtype=None): + def _use_gpu(self, api_config=None, dtype=None): if not is_gpu_mode(): return False if self.place is not None and "cpu" in str(self.place).lower(): @@ -370,20 +355,21 @@ def _make_gpu_torch_tensor(self, api_config, dtype=None): def _make_gpu_tensor_pair(self, api_config, dtype=None): dtype = dtype or self.dtype source_dtype = "float16" if dtype in FLOAT8_DTYPES else dtype - if self.gpu_source_tensor is None: - self.gpu_source_tensor = self._make_gpu_torch_dense_tensor(source_dtype) - torch_source = self.gpu_source_tensor + torch_source = self._make_gpu_torch_dense_tensor(source_dtype) paddle_source = paddle.utils.dlpack.from_dlpack( torch.utils.dlpack.to_dlpack(torch_source.detach().clone()) ) if not self.is_contiguous and self.strides is not None: - paddle_tensor = self._make_gpu_strided_paddle_tensor_from_values( - paddle_source, api_config, dtype - ) - torch_tensor = self._make_gpu_strided_torch_tensor_from_values( - torch_source, api_config, dtype - ) - return paddle_tensor, torch_tensor + try: + paddle_tensor = self._make_gpu_strided_paddle_tensor_from_values( + paddle_source, api_config, dtype + ) + torch_tensor = self._make_gpu_strided_torch_tensor_from_values( + torch_source, api_config, dtype + ) + return paddle_tensor, torch_tensor + finally: + del torch_source, paddle_source if dtype in FLOAT8_DTYPES: paddle_tensor = paddle_source.cast(dtype) torch_tensor = torch_source.to(dtype=self.convert_dtype_to_torch_type(dtype)) @@ -392,6 +378,7 @@ def _make_gpu_tensor_pair(self, api_config, dtype=None): torch_tensor = torch_source.detach().clone() paddle_tensor = self._set_paddle_autograd(paddle_tensor, api_config, dtype) torch_tensor = self._set_torch_autograd(torch_tensor, api_config, dtype) + del torch_source return paddle_tensor, torch_tensor def get_gpu_paddle_tensor(self, api_config, dtype=None): @@ -3145,7 +3132,7 @@ def get_exponent_max(value, dtype_max, default_max=5): self.numpy_tensor = numpy.random.random(self.shape).astype("float32") if self.numpy_tensor is None: - if self._use_gpu(original_dtype): + if self._use_gpu(api_config, original_dtype): self.get_gpu_paddle_tensor(api_config, original_dtype) elif self.shape == []: if "int" in self.dtype: @@ -3188,7 +3175,7 @@ def get_exponent_max(value, dtype_max, default_max=5): def get_paddle_tensor(self, api_config): if self.paddle_tensor is None: - if self.numpy_tensor is None and self._use_gpu(): + if self.numpy_tensor is None and self._use_gpu(api_config): return self.get_gpu_paddle_tensor(api_config) if not self.is_contiguous and self.strides is not None: self.paddle_tensor = self._create_strided_paddle_tensor(api_config) @@ -3267,7 +3254,7 @@ def get_torch_tensor(self, api_config): device = torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu") torch.set_default_device(device) if self.torch_tensor is None: - if self.numpy_tensor is None and self._use_gpu(): + if self.numpy_tensor is None and self._use_gpu(api_config): return self.get_gpu_torch_tensor(api_config) if not self.is_contiguous and self.strides is not None: self.torch_tensor = self._create_strided_torch_tensor(api_config) @@ -3336,14 +3323,14 @@ def clear_tensor(self): self.torch_tensor = None self.paddle_tensor = None self.numpy_tensor = None - if not should_skip_gpu_cleanup(): + if not is_gpu_mode(): torch.cuda.empty_cache() paddle.device.cuda.empty_cache() def clear_paddle_tensor(self): del self.paddle_tensor self.paddle_tensor = None - if not should_skip_gpu_cleanup(): + if not is_gpu_mode(): paddle.device.cuda.empty_cache() def clear_numpy_tensor(self): @@ -3353,7 +3340,7 @@ def clear_numpy_tensor(self): def clear_torch_tensor(self): del self.torch_tensor self.torch_tensor = None - if not should_skip_gpu_cleanup(): + if not is_gpu_mode(): torch.cuda.empty_cache() def fill_numpy_tensor(self, full_value): diff --git a/tester/base.py b/tester/base.py index 6b34675d..adf9e09b 100644 --- a/tester/base.py +++ b/tester/base.py @@ -1,6 +1,7 @@ from __future__ import annotations import collections +import gc import inspect import os @@ -12,10 +13,9 @@ USE_CACHED_NUMPY, TensorConfig, get_cached_numpy_array, - is_gpu_mode, - should_skip_gpu_cleanup, ) from .api_config.log_writer import log_accuracy_tolerance, write_to_log +from .runtime_config import TestRuntimeConfig with open("tester/base_config.yaml", encoding="utf-8") as f: config = yaml.safe_load(f) @@ -66,6 +66,56 @@ def __getattr__(self, name): ) +def gpu_mode_maybe_empty_cache(gpu_config, phase="", force=False): + if not gpu_config.enabled: + return False + try: + if "gpu" not in paddle.device.get_device(): + return False + except Exception: + return False + + should_cleanup = bool(force) + if not should_cleanup: + total_memory = float(gpu_config.total_memory or 0.0) + workers_on_gpu = max(1, int(gpu_config.workers_on_gpu or 1)) + required_memory = float(gpu_config.required_memory or 10.0) + memory_fraction = float(gpu_config.memory_fraction or 0.85) + pressure_ratio = float(gpu_config.cleanup_pressure_ratio or 0.25) + used_ratio = float(gpu_config.cleanup_used_ratio or 0.90) + + try: + torch_reserved = torch.cuda.memory_reserved() / (1024**3) + except Exception: + torch_reserved = 0.0 + try: + torch_allocated = torch.cuda.memory_allocated() / (1024**3) + except Exception: + torch_allocated = 0.0 + + if total_memory <= 0: + should_cleanup = ( + torch_reserved > 0 and torch_allocated / max(torch_reserved, 1e-6) < 0.5 + ) + else: + per_worker_budget = total_memory * memory_fraction / workers_on_gpu + cleanup_reserved_threshold = max(required_memory, per_worker_budget * used_ratio) + cleanup_idle_threshold = max(1.0, per_worker_budget * pressure_ratio) + idle_reserved = max(0.0, torch_reserved - torch_allocated) + should_cleanup = ( + torch_reserved >= cleanup_reserved_threshold + or idle_reserved >= cleanup_idle_threshold + ) + + if not should_cleanup: + return False + + gc.collect() + torch.cuda.empty_cache() + paddle.device.cuda.empty_cache() + return True + + def classify_runtime_error(error_msg): error_msg_lower = error_msg.lower() if error_msg.startswith("[torch_assert_OOM]"): @@ -232,9 +282,11 @@ def get_arg(api_config, arg_pos, arg_name, default=None): class APITestBase: - def __init__(self, api_config, use_torch=True): + def __init__(self, api_config, use_torch=True, runtime_config=None): self.api_config = api_config self.api_config.use_torch = use_torch + self.runtime_config = runtime_config or TestRuntimeConfig() + self.gpu_mode_config = self.runtime_config.gpu_mode self.outputs_grad_numpy = [] self.outputs_grad_paddleonly = [] if use_torch: @@ -833,7 +885,7 @@ def _make_paddle_output_grad(self, shape, dtype): return (paddle.rand(tuple(shape), dtype=base_dtype) - 0.5).cast(dtype) def _use_gpu_output_grad(self, output): - if not is_gpu_mode() or "gpu" not in paddle.device.get_device(): + if not self.gpu_mode_config.enabled or "gpu" not in paddle.device.get_device(): return False dtype = str(output.dtype).split(".")[-1] return dtype in ["float32", "float64", "float16", "bfloat16", "complex64", "complex128"] @@ -849,12 +901,12 @@ def _get_gpu_output_grad_paddleonly(self, output, index): def _get_gpu_output_grad_pair(self, output, index): if len(self.outputs_grad_numpy) <= index: dtype = str(output.dtype).split(".")[-1] - torch_grad = self._make_torch_output_grad(output.shape, dtype) + torch_grad = self._make_torch_output_grad(output.shape, dtype).detach() paddle_grad = paddle.utils.dlpack.from_dlpack( - torch.utils.dlpack.to_dlpack(torch_grad.detach().clone()) + torch.utils.dlpack.to_dlpack(torch_grad.clone()) ) paddle_grad.stop_gradient = False - self.outputs_grad_numpy.append((paddle_grad, torch_grad.detach().clone())) + self.outputs_grad_numpy.append((paddle_grad, torch_grad)) return self.outputs_grad_numpy[index] def gen_paddle_output_and_output_grad(self, outputs): @@ -1240,7 +1292,9 @@ def gen_torch_input(self): ) or self.api_config.api_name == "paddle.Tensor.__setitem__": self.torch_args, self.torch_kwargs = self.copy_torch_input() - if not should_skip_gpu_cleanup(): + if self.gpu_mode_config.enabled: + gpu_mode_maybe_empty_cache(self.gpu_mode_config, "clear_torch_tensor") + else: torch.cuda.empty_cache() return True @@ -1532,7 +1586,7 @@ def torch_assert_accuracy(self, paddle_tensor, torch_tensor, atol=1e-2, rtol=1e- if test_tol: atol, rtol = 0.0, 0.0 - if test_tol or not is_gpu_mode(): + if test_tol or not self.gpu_mode_config.enabled: self._torch_assert_accuracy_cpu( paddle_tensor, torch_tensor, @@ -1564,7 +1618,9 @@ def clear_tensor(self): for i in range(len(arg_config)): if isinstance(arg_config[i], TensorConfig): arg_config[i].clear_tensor() - if not should_skip_gpu_cleanup(): + if self.gpu_mode_config.enabled: + gpu_mode_maybe_empty_cache(self.gpu_mode_config, "clear_tensor") + else: torch.cuda.empty_cache() paddle.device.cuda.empty_cache() @@ -1578,7 +1634,9 @@ def clear_paddle_tensor(self): for i in range(len(arg_config)): if isinstance(arg_config[i], TensorConfig): arg_config[i].clear_paddle_tensor() - if not should_skip_gpu_cleanup(): + if self.gpu_mode_config.enabled: + gpu_mode_maybe_empty_cache(self.gpu_mode_config, "clear_paddle_tensor") + else: paddle.device.cuda.empty_cache() def clear_torch_tensor(self): @@ -1591,7 +1649,9 @@ def clear_torch_tensor(self): for i in range(len(arg_config)): if isinstance(arg_config[i], TensorConfig): arg_config[i].clear_torch_tensor() - if not should_skip_gpu_cleanup(): + if self.gpu_mode_config.enabled: + gpu_mode_maybe_empty_cache(self.gpu_mode_config, "clear_torch_tensor") + else: torch.cuda.empty_cache() def clear_numpy_tensor(self): diff --git a/tester/runtime_config.py b/tester/runtime_config.py new file mode 100644 index 00000000..27760c80 --- /dev/null +++ b/tester/runtime_config.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from dataclasses import dataclass, field, replace + + +@dataclass(frozen=True) +class GpuModeConfig: + enabled: bool = False + required_memory: float = 10.0 + workers_on_gpu: int = 1 + total_memory: float = 0.0 + memory_fraction: float = 0.85 + cleanup_pressure_ratio: float = 0.25 + cleanup_used_ratio: float = 0.90 + + +@dataclass(frozen=True) +class TestRuntimeConfig: + random_seed: int = 0 + bitwise_alignment: bool = False + exit_on_error: bool = False + gpu_mode: GpuModeConfig = field(default_factory=GpuModeConfig) + + @classmethod + def from_options(cls, options): + gpu_mode = GpuModeConfig( + enabled=bool(options.use_gpu_mode), + required_memory=float(options.required_memory), + ) + return cls( + random_seed=int(options.random_seed), + bitwise_alignment=bool(options.bitwise_alignment), + exit_on_error=bool(options.exit_on_error), + gpu_mode=gpu_mode, + ) + + def for_gpu(self, gpu_id, workers_per_gpu, total_memory_per_gpu): + workers_on_gpu = workers_per_gpu.get(gpu_id, self.gpu_mode.workers_on_gpu) + total_memory = total_memory_per_gpu.get(gpu_id, self.gpu_mode.total_memory) + gpu_mode = replace( + self.gpu_mode, + workers_on_gpu=max(1, int(workers_on_gpu or 1)), + total_memory=float(total_memory or 0.0), + ) + return replace(self, gpu_mode=gpu_mode) + + +def runtime_config_for_gpu(options, gpu_id): + return options.runtime_config.for_gpu( + gpu_id, + options.gpu_workers_per_gpu_map, + options.gpu_total_memory_map, + ) From 485b225701a7405d98cc152e339508a7b96773cb Mon Sep 17 00:00:00 2001 From: cangtianhuang <1903374751@qq.com> Date: Fri, 17 Jul 2026 14:50:48 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Update=20gpu=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- engineV2-README.md | 3 +- engineV2.py | 76 ++++-------------- engineV4.py | 77 ++++--------------- run-example.sh | 2 - run-v4.sh | 2 - run.py | 3 - test_pipeline/CINN/README_test_cinn.md | 2 - test_pipeline/CINN/run_cinn.sh | 2 - test_pipeline/V2/cpu_0size.md | 4 +- test_pipeline/V2/cpu_accuracy.md | 2 - test_pipeline/V2/gpu_0size.md | 4 +- test_pipeline/V2/gpu_accuracy.md | 2 - test_pipeline/V2/gpu_accuracy_tolerance.md | 2 - test_pipeline/V2/gpu_bigtensor.md | 2 - .../V2/monitor_script/run_cpu_0size_full.sh | 2 - .../V2/monitor_script/run_gpu_0size_full.sh | 2 - .../monitor_script/run_gpu_accuracy_full.sh | 2 - .../monitor_script/run_gpu_bigtensor_full.sh | 2 - test_pipeline/V2/run_cpu_0size_full.sh | 2 - test_pipeline/V2/run_cpu_0size_regr.sh | 2 - test_pipeline/V2/run_cpu_accuracy_full.sh | 2 - test_pipeline/V2/run_cpu_accuracy_regr.sh | 2 - test_pipeline/V2/run_gpu_0size_full.sh | 2 - test_pipeline/V2/run_gpu_0size_regr.sh | 2 - test_pipeline/V2/run_gpu_accuracy_full.sh | 2 - test_pipeline/V2/run_gpu_accuracy_regr.sh | 2 - test_pipeline/V2/run_gpu_accuracy_stable.sh | 2 - .../V2/run_gpu_accuracy_tolerance.sh | 2 - test_pipeline/V2/run_gpu_bigtensor_full.sh | 2 - test_pipeline/V2/run_gpu_bigtensor_regr.sh | 2 - .../V2/run_paddle_gpu_performance.sh | 2 - test_pipeline/V4/run_cpu_0size_full.sh | 2 - test_pipeline/V4/run_dsv4_0size_accuracy.sh | 2 - test_pipeline/V4/run_dsv4_0size_paddleonly.sh | 2 - test_pipeline/V4/run_dsv4_1M_accuracy.sh | 2 - test_pipeline/V4/run_dsv4_1M_paddleonly.sh | 2 - .../run_dsv4_v2_accuracy_manual_threshold.sh | 2 - test_pipeline/V4/run_gpu_0size_full.sh | 2 - test_pipeline/V4/run_gpu_accuracy_full.sh | 2 - test_pipeline/V4/run_gpu_bigtensor_full.sh | 2 - test_pipeline/V4/run_v2_0size_accuracy.sh | 2 - test_pipeline/V4/run_v2_0size_paddleonly.sh | 2 - test_pipeline/V4/run_v2_1M_accuracy.sh | 2 - test_pipeline/V4/run_v2_1M_paddleonly.sh | 2 - test_pipeline/run_config.schema.json | 1 - test_pipeline/run_config.yaml | 1 - tester/accuracy.py | 15 +++- tester/api_config/config_analyzer.py | 41 +++++++++- tester/base.py | 62 +++++++-------- tester/runtime_config.py | 26 ++++--- 50 files changed, 135 insertions(+), 254 deletions(-) diff --git a/engineV2-README.md b/engineV2-README.md index e3091d08..362d1b2a 100644 --- a/engineV2-README.md +++ b/engineV2-README.md @@ -72,9 +72,8 @@ | `--accuracy_stable` | bool | 启用稳定性测试（默认 False） | | `--paddle_custom_device` | bool | 运行Custom Device或者XPU的API与CPU的精度对比（默认 False） | | `--num_gpus` | int | 使用的 GPU 数量（默认 -1，-1 动态最大） | -| `--num_workers_per_gpu` | int | 每 GPU 的 worker 进程数（默认 1，-1 动态最大） | +| `--num_workers_per_gpu` | int | 每 GPU 的 worker 进程数（默认 1；gpu_mode 下 -1 表示每 GPU 1 个 worker） | | `--gpu_ids` | str | 使用的 GPU 序号，以逗号分隔或横线范围（默认 ""，"-1" 动态最大） | -| `--required_memory` | float | 每 worker 进程预估使用显存 GB（默认 10.0） | | `--test_amp` | bool | 启用自动混合精度测试（默认 False） | | `--test_cpu` | bool | 启用 Paddle CPU 模式测试（默认 False） | | `--use_cached_numpy` | bool | 启用 Numpy 缓存（默认 False） | diff --git a/engineV2.py b/engineV2.py index 51acdf60..d88802ae 100644 --- a/engineV2.py +++ b/engineV2.py @@ -443,11 +443,6 @@ def validate_gpu_options(options) -> tuple: f"invalid --num_workers_per_gpu={options.num_workers_per_gpu}: " "expected -1 or a positive integer" ) - if options.required_memory <= 0: - raise ValueError( - f"invalid --required_memory={options.required_memory:g}: " - "expected a positive number of GiB" - ) return tuple(gpu_ids) @@ -485,26 +480,19 @@ def _prepare_single_config_gpu(options): return gpu_ids[0] -def check_gpu_memory(gpu_ids, num_workers_per_gpu, required_memory): # required_memory in GB +def check_gpu_memory(gpu_ids, num_workers_per_gpu): assert isinstance(gpu_ids, tuple) and len(gpu_ids) > 0 available_gpus = [] max_workers_per_gpu = {} for gpu_id in gpu_ids: try: - total_memory, used_memory = get_memory_info(gpu_id) - free_memory = total_memory - used_memory - max_workers = int(free_memory // required_memory) - if max_workers >= 1: - available_gpus.append(gpu_id) - max_workers_per_gpu[gpu_id] = ( - max_workers - if num_workers_per_gpu == -1 - else min(max_workers, num_workers_per_gpu) - ) + get_memory_info(gpu_id) except pynvml.NVMLError as e: print(f"[warn] Failed to check GPU {gpu_id}: {e!s}", flush=True) continue + available_gpus.append(gpu_id) + max_workers_per_gpu[gpu_id] = 1 if num_workers_per_gpu == -1 else num_workers_per_gpu return available_gpus, max_workers_per_gpu @@ -587,35 +575,16 @@ def run_test_case(api_config_str, options): flush=True, ) - last_memory_log_time = 0 - memory_wait_start = time.time() - while True: - total_memory, used_memory = get_memory_info(gpu_id) - free_memory = total_memory - used_memory - - if free_memory >= options.required_memory: - break - - now = time.time() - if now - last_memory_log_time >= MEMORY_WAIT_LOG_INTERVAL: - print( - f"{datetime.now()} device {gpu_id} Free: {free_memory:.1f} GB, " - f"Required: {options.required_memory:.1f} GB. ", - "Waiting for available memory...", - flush=True, - ) - last_memory_log_time = now - if ( - getattr(options, "use_gpu_mode", False) - and now - memory_wait_start >= MEMORY_WAIT_MAX_SECONDS - ): - raise GpuMemoryDeferred( - f"device {gpu_id} Memory wait timeout after {MEMORY_WAIT_MAX_SECONDS}s " - f"(free={free_memory:.1f} GB, required={options.required_memory:.1f} GB)" - ) - time.sleep(MEMORY_WAIT_SECONDS) - + total_memory, used_memory = get_memory_info(gpu_id) + free_memory = total_memory - used_memory runtime_config = runtime_config_for_gpu(options, gpu_id) + gpu_config = runtime_config.gpu_mode + print( + f"{datetime.now()} device {gpu_id} memory total={total_memory:.1f} GB, " + f"free={free_memory:.1f} GB, workers={gpu_config.workers_on_gpu}, " + f"budget={gpu_config.memory_budget:.1f} GB", + flush=True, + ) if options.show_runtime_status: total_memory, used_memory_before = get_memory_info(gpu_id) @@ -812,7 +781,7 @@ def main(): "--num_workers_per_gpu", type=int, default=1, - help="Workers per GPU. Use -1 to maximize workers based on free memory.", + help="Workers per GPU. In gpu_mode, -1 uses one worker per GPU.", ) parser.add_argument( "--gpu_ids", @@ -820,12 +789,6 @@ def main(): default="", help="GPU IDs to use, e.g. '0', '0,2', '0-3'. Use '-1' for all GPUs.", ) - parser.add_argument( - "--required_memory", - type=float, - default=10.0, - help="Minimum free memory required per worker, in GiB.", - ) parser.add_argument( "--test_cpu", type=parse_bool, @@ -1170,15 +1133,10 @@ def main(): print(all_case, "cases will be tested.", flush=True) del api_config_count, dup_case, finish_case - # validate GPU memory - available_gpus, max_workers_per_gpu = check_gpu_memory( - gpu_ids, options.num_workers_per_gpu, options.required_memory - ) + # validate GPU visibility and derive per-GPU worker counts + available_gpus, max_workers_per_gpu = check_gpu_memory(gpu_ids, options.num_workers_per_gpu) if not available_gpus: - print( - f"No GPUs with sufficient memory available. Current memory constraint is {options.required_memory} GB.", - flush=True, - ) + print("No usable GPUs available.", flush=True) return total_workers = sum(max_workers_per_gpu.values()) diff --git a/engineV4.py b/engineV4.py index 7312ab79..29c57427 100644 --- a/engineV4.py +++ b/engineV4.py @@ -92,7 +92,6 @@ class GpuMemoryDeferred(Exception): "test_cpu", "use_cached_numpy", "log_dir", - "required_memory", "atol", "rtol", "manual_threshold_config_file", @@ -1036,11 +1035,6 @@ def validate_gpu_options(options) -> tuple: f"invalid --num_workers_per_gpu={options.num_workers_per_gpu}: " "expected -1 or a positive integer" ) - if options.required_memory <= 0: - raise ValueError( - f"invalid --required_memory={options.required_memory:g}: " - "expected a positive number of GiB" - ) return tuple(gpu_ids) @@ -1242,26 +1236,19 @@ def _run_single_config_with_sanitizer(options): return result.returncode -def check_gpu_memory(gpu_ids, num_workers_per_gpu, required_memory): # required_memory in GB +def check_gpu_memory(gpu_ids, num_workers_per_gpu): assert isinstance(gpu_ids, tuple) and len(gpu_ids) > 0 available_gpus = [] max_workers_per_gpu = {} for gpu_id in gpu_ids: try: - total_memory, used_memory = get_memory_info(gpu_id) - free_memory = total_memory - used_memory - max_workers = int(free_memory // required_memory) - if max_workers >= 1: - available_gpus.append(gpu_id) - max_workers_per_gpu[gpu_id] = ( - max_workers - if num_workers_per_gpu == -1 - else min(max_workers, num_workers_per_gpu) - ) + get_memory_info(gpu_id) except pynvml.NVMLError as e: print(f"[WARNING] Failed to check GPU {gpu_id}: {e!s}", flush=True) continue + available_gpus.append(gpu_id) + max_workers_per_gpu[gpu_id] = 1 if num_workers_per_gpu == -1 else num_workers_per_gpu return available_gpus, max_workers_per_gpu @@ -1276,35 +1263,16 @@ def run_test_case(api_config_str, options): flush=True, ) - max_memory_wait = 30 # max 30 iterations × 10s = 5 minutes - for wait_iter in range(max_memory_wait): - total_memory, used_memory = get_memory_info(gpu_id) - free_memory = total_memory - used_memory - - if free_memory >= options.required_memory: - break - - if wait_iter % 6 == 0: # log every ~60s (every 6th iteration) - print( - f"{datetime.now()} device {gpu_id} Free: {free_memory:.1f} GB, " - f"Required: {options.required_memory:.1f} GB. " - f"Waiting for available memory... (attempt {wait_iter + 1}/{max_memory_wait})", - flush=True, - ) - time.sleep(10) - else: - message = ( - f"device {gpu_id} Memory wait timeout after {max_memory_wait * 10}s " - f"(free={free_memory:.1f} GB, required={options.required_memory:.1f} GB)" - ) - if getattr(options, "use_gpu_mode", False): - raise GpuMemoryDeferred(message) - print( - f"{datetime.now()} {message}. Proceeding anyway.", - flush=True, - ) - + total_memory, used_memory = get_memory_info(gpu_id) + free_memory = total_memory - used_memory runtime_config = runtime_config_for_gpu(options, gpu_id) + gpu_config = runtime_config.gpu_mode + print( + f"{datetime.now()} device {gpu_id} memory total={total_memory:.1f} GB, " + f"free={free_memory:.1f} GB, workers={gpu_config.workers_on_gpu}, " + f"budget={gpu_config.memory_budget:.1f} GB", + flush=True, + ) try: api_config = APIConfig(api_config_str) @@ -1479,7 +1447,7 @@ def main(): "--num_workers_per_gpu", type=int, default=1, - help="Workers per GPU. Use -1 to maximize workers based on free memory.", + help="Workers per GPU. In gpu_mode, -1 uses one worker per GPU.", ) parser.add_argument( "--gpu_ids", @@ -1487,12 +1455,6 @@ def main(): default="", help="GPU IDs to use, e.g. '0', '0,2', '0-3'. Use '-1' for all GPUs.", ) - parser.add_argument( - "--required_memory", - type=float, - default=10.0, - help="Minimum free memory required per worker, in GiB.", - ) parser.add_argument( "--test_cpu", type=parse_bool, @@ -1884,15 +1846,10 @@ def main(): print(all_case, "cases will be tested.", flush=True) del api_config_count, dup_case, finish_case - # validate GPU memory - available_gpus, max_workers_per_gpu = check_gpu_memory( - gpu_ids, options.num_workers_per_gpu, options.required_memory - ) + # validate GPU visibility and derive per-GPU worker counts + available_gpus, max_workers_per_gpu = check_gpu_memory(gpu_ids, options.num_workers_per_gpu) if not available_gpus: - print( - f"No GPUs with sufficient memory available. Current memory constraint is {options.required_memory} GB.", - flush=True, - ) + print("No usable GPUs available.", flush=True) return if ( diff --git a/run-example.sh b/run-example.sh index 40d7020e..adc38444 100755 --- a/run-example.sh +++ b/run-example.sh @@ -35,7 +35,6 @@ LOG_DIR="tester/api_config/test_log" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TIME_OUT=600 # ── 测试模式：必须且只能启用一种 ─────────────────────────────── @@ -99,7 +98,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( diff --git a/run-v4.sh b/run-v4.sh index e2ef9009..f277704c 100644 --- a/run-v4.sh +++ b/run-v4.sh @@ -52,7 +52,6 @@ LOG_DIR="tester/api_config/test_log" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TIME_OUT=600 # ── 测试模式：必须且只能启用一种 ─────────────────────────────── @@ -200,7 +199,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( diff --git a/run.py b/run.py index 5395b6bd..0af28471 100644 --- a/run.py +++ b/run.py @@ -56,7 +56,6 @@ "num_gpus": int, "num_workers_per_gpu": int, "gpu_ids": str, - "required_memory": (int, float), "test_cpu": bool, "use_cached_numpy": bool, "use_gpu_mode": bool, @@ -290,7 +289,6 @@ def apply_overrides(config: dict[str, Any], args: argparse.Namespace) -> None: "num_gpus": args.num_gpus, "num_workers_per_gpu": args.num_workers_per_gpu, "gpu_ids": args.gpu_ids, - "required_memory": args.required_memory, } for key, value in simple_engine_overrides.items(): if value is not None: @@ -676,7 +674,6 @@ def parse_args() -> argparse.Namespace: "--num-workers-per-gpu", type=int, help="覆盖 engine_args.num_workers_per_gpu" ) parser.add_argument("--gpu-ids", help="覆盖 engine_args.gpu_ids") - parser.add_argument("--required-memory", type=float, help="覆盖 engine_args.required_memory") parser.add_argument( "--set-env", action="append", default=[], help="追加或覆盖环境变量 KEY=VALUE" ) diff --git a/test_pipeline/CINN/README_test_cinn.md b/test_pipeline/CINN/README_test_cinn.md index 93c7c54a..4673f8db 100644 --- a/test_pipeline/CINN/README_test_cinn.md +++ b/test_pipeline/CINN/README_test_cinn.md @@ -62,7 +62,6 @@ LOG_DIR="tester/api_config/test_log_cinn" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=1 # 建议单卡单进程 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TEST_MODE_ARGS=( # --accuracy=True @@ -91,7 +90,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/CINN/run_cinn.sh b/test_pipeline/CINN/run_cinn.sh index b8b65b52..daf0bf61 100755 --- a/test_pipeline/CINN/run_cinn.sh +++ b/test_pipeline/CINN/run_cinn.sh @@ -11,7 +11,6 @@ LOG_DIR="tester/api_config/test_log_cinn" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=1 # 建议单卡单进程 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TEST_MODE_ARGS=( # --accuracy=True @@ -40,7 +39,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/cpu_0size.md b/test_pipeline/V2/cpu_0size.md index 3427383c..4b76ed36 100644 --- a/test_pipeline/V2/cpu_0size.md +++ b/test_pipeline/V2/cpu_0size.md @@ -67,7 +67,6 @@ LOG_DIR="tester/api_config/test_log_0_size_cpu_accuracy" # 测试日志目录 NUM_GPUS=-1 # 指定 GPU 数量，-1 表示使用所有 GPU NUM_WORKERS_PER_GPU=15 # 每个 GPU 使用 15 个 worker（建议不超过 15 个，否则内存会爆） GPU_IDS="-1" # 指定 GPU 列表，-1 表示使用所有 GPU -REQUIRED_MEMORY=5 # 每个 worker 预估所需显存 GB TEST_MODE_ARGS=( --accuracy=True @@ -88,7 +87,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { @@ -136,7 +134,7 @@ TEST_MODE_ARGS=( 若不使用 `run_0_cpu.sh`，以测试 0 size cpu accuracy 为例，可直接执行以下命令：（建议使用 nohup 避免终端终止时停止主进程） ```bash -python engineV2.py --api_config_file_pattern="tester/api_config/7_0_size/0_size_tensor_1_8_[1-2].txt" --accuracy=True --test_cpu=True --num_gpus=-1 --num_workers_per_gpu=15 --required_memory=5 --log_dir="tester/api_config/test_log_0_size_cpu_accuracy" >> "tester/api_config/test_log_0_size_cpu_accuracy/log.log" 2>&1 +python engineV2.py --api_config_file_pattern="tester/api_config/7_0_size/0_size_tensor_1_8_[1-2].txt" --accuracy=True --test_cpu=True --num_gpus=-1 --num_workers_per_gpu=15 --log_dir="tester/api_config/test_log_0_size_cpu_accuracy" >> "tester/api_config/test_log_0_size_cpu_accuracy/log.log" 2>&1 ``` 或者直接运行 run_0_cpu.sh： diff --git a/test_pipeline/V2/cpu_accuracy.md b/test_pipeline/V2/cpu_accuracy.md index c398d6b2..6981ab14 100644 --- a/test_pipeline/V2/cpu_accuracy.md +++ b/test_pipeline/V2/cpu_accuracy.md @@ -80,7 +80,6 @@ LOG_DIR="tester/api_config/test_log_cpu_accuracy" # 测试日志目录 NUM_GPUS=-1 # 指定 GPU 数量，-1 表示使用所有 GPU NUM_WORKERS_PER_GPU=-1 # 指定每个 GPU 的工作进程数，-1 表示使用最大进程数 GPU_IDS="-1" # 指定 GPU 列表，-1 表示使用所有 GPU -# REQUIRED_MEMORY=10 # 可通过调整单个工作进程预估显存 GB，修改最大进程数 TEST_MODE_ARGS=( --accuracy=True @@ -101,7 +100,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/gpu_0size.md b/test_pipeline/V2/gpu_0size.md index 7b88a01e..61d9c688 100644 --- a/test_pipeline/V2/gpu_0size.md +++ b/test_pipeline/V2/gpu_0size.md @@ -67,7 +67,6 @@ LOG_DIR="tester/api_config/test_log_0_size_gpu_accuracy" # 测试日志目录 NUM_GPUS=-1 # 指定 GPU 数量，-1 表示使用所有 GPU NUM_WORKERS_PER_GPU=15 # 每个 GPU 使用 15 个 worker（建议不超过 15 个，否则内存会爆） GPU_IDS="-1" # 指定 GPU 列表，-1 表示使用所有 GPU -REQUIRED_MEMORY=5 # 每个 worker 预估所需显存 GB TEST_MODE_ARGS=( --accuracy=True @@ -88,7 +87,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { @@ -135,7 +133,7 @@ TEST_MODE_ARGS=( 若不使用 `run_0_gpu.sh`，以测试 0 size gpu accuracy 为例，可直接执行以下命令：（建议使用 nohup 避免终端终止时停止主进程） ```bash -python engineV2.py --api_config_file_pattern="tester/api_config/7_0_size/0_size_tensor_1_8_[1-2].txt" --accuracy=True --num_gpus=-1 --num_workers_per_gpu=15 --required_memory=5 --log_dir="tester/api_config/test_log_0_size_gpu_accuracy" >> "tester/api_config/test_log_0_size_gpu_accuracy/log.log" 2>&1 +python engineV2.py --api_config_file_pattern="tester/api_config/7_0_size/0_size_tensor_1_8_[1-2].txt" --accuracy=True --num_gpus=-1 --num_workers_per_gpu=15 --log_dir="tester/api_config/test_log_0_size_gpu_accuracy" >> "tester/api_config/test_log_0_size_gpu_accuracy/log.log" 2>&1 ``` 或者直接运行 run_0_gpu.sh： diff --git a/test_pipeline/V2/gpu_accuracy.md b/test_pipeline/V2/gpu_accuracy.md index db2fab16..8766b1d4 100644 --- a/test_pipeline/V2/gpu_accuracy.md +++ b/test_pipeline/V2/gpu_accuracy.md @@ -80,7 +80,6 @@ LOG_DIR="tester/api_config/test_log_gpu_accuracy" # 测试日志目录 NUM_GPUS=-1 # 指定 GPU 数量，-1 表示使用所有 GPU NUM_WORKERS_PER_GPU=-1 # 指定每个 GPU 的工作进程数，-1 表示使用最大进程数 GPU_IDS="-1" # 指定 GPU 列表，-1 表示使用所有 GPU -# REQUIRED_MEMORY=10 # 可通过调整单个工作进程预估显存 GB，修改最大进程数 TEST_MODE_ARGS=( --accuracy=True @@ -101,7 +100,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/gpu_accuracy_tolerance.md b/test_pipeline/V2/gpu_accuracy_tolerance.md index 8189f870..5526c386 100644 --- a/test_pipeline/V2/gpu_accuracy_tolerance.md +++ b/test_pipeline/V2/gpu_accuracy_tolerance.md @@ -73,7 +73,6 @@ LOG_DIR="tester/api_config/test_log_tol" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=-1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TEST_MODE_ARGS=( --accuracy=True @@ -97,7 +96,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/gpu_bigtensor.md b/test_pipeline/V2/gpu_bigtensor.md index 37033ef6..b262ecc9 100644 --- a/test_pipeline/V2/gpu_bigtensor.md +++ b/test_pipeline/V2/gpu_bigtensor.md @@ -64,7 +64,6 @@ LOG_DIR="tester/api_config/test_log_big_tensor_accuracy" # 测试日志目录 NUM_GPUS=-1 # 指定 GPU 数量，-1 表示使用所有 GPU NUM_WORKERS_PER_GPU=1 # 每个 GPU 仅使用 1 个 worker，避免显存溢出 GPU_IDS="-1" # 指定 GPU 列表，-1 表示使用所有 GPU -# REQUIRED_MEMORY=10 TEST_MODE_ARGS=( --accuracy=True @@ -85,7 +84,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/monitor_script/run_cpu_0size_full.sh b/test_pipeline/V2/monitor_script/run_cpu_0size_full.sh index 001b4c5c..52e2aec5 100644 --- a/test_pipeline/V2/monitor_script/run_cpu_0size_full.sh +++ b/test_pipeline/V2/monitor_script/run_cpu_0size_full.sh @@ -11,7 +11,6 @@ LOG_DIR="tester/api_config/test_log_cpu_0size_full" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=15 GPU_IDS="-1" -REQUIRED_MEMORY=5 TIME_OUT=600 RANDOM_SEED=2025 @@ -34,7 +33,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( --timeout="$TIME_OUT" diff --git a/test_pipeline/V2/monitor_script/run_gpu_0size_full.sh b/test_pipeline/V2/monitor_script/run_gpu_0size_full.sh index 3198f951..39fa07e9 100644 --- a/test_pipeline/V2/monitor_script/run_gpu_0size_full.sh +++ b/test_pipeline/V2/monitor_script/run_gpu_0size_full.sh @@ -11,7 +11,6 @@ LOG_DIR="tester/api_config/test_log_gpu_0size_full" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=15 GPU_IDS="-1" -REQUIRED_MEMORY=5 TIME_OUT=300 RANDOM_SEED=2025 @@ -35,7 +34,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( --timeout="$TIME_OUT" diff --git a/test_pipeline/V2/monitor_script/run_gpu_accuracy_full.sh b/test_pipeline/V2/monitor_script/run_gpu_accuracy_full.sh index c2abf591..f9609921 100644 --- a/test_pipeline/V2/monitor_script/run_gpu_accuracy_full.sh +++ b/test_pipeline/V2/monitor_script/run_gpu_accuracy_full.sh @@ -12,7 +12,6 @@ LOG_DIR="tester/api_config/test_log_gpu_accuracy_full" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=-1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TEST_MODE_ARGS=( --accuracy=True @@ -39,7 +38,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/monitor_script/run_gpu_bigtensor_full.sh b/test_pipeline/V2/monitor_script/run_gpu_bigtensor_full.sh index 637e5748..20d61d15 100644 --- a/test_pipeline/V2/monitor_script/run_gpu_bigtensor_full.sh +++ b/test_pipeline/V2/monitor_script/run_gpu_bigtensor_full.sh @@ -11,7 +11,6 @@ LOG_DIR="tester/api_config/test_log_gpu_bigtensor_full" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TEST_MODE_ARGS=( --accuracy=True @@ -32,7 +31,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/run_cpu_0size_full.sh b/test_pipeline/V2/run_cpu_0size_full.sh index 01de9a06..e6357d6e 100644 --- a/test_pipeline/V2/run_cpu_0size_full.sh +++ b/test_pipeline/V2/run_cpu_0size_full.sh @@ -12,7 +12,6 @@ LOG_DIR="tester/api_config/test_log_cpu_0size_full" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=15 GPU_IDS="-1" -REQUIRED_MEMORY=5 TIME_OUT=600 TEST_MODE_ARGS=( @@ -34,7 +33,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( --timeout="$TIME_OUT" diff --git a/test_pipeline/V2/run_cpu_0size_regr.sh b/test_pipeline/V2/run_cpu_0size_regr.sh index 4c2fbaf1..c1b794ea 100644 --- a/test_pipeline/V2/run_cpu_0size_regr.sh +++ b/test_pipeline/V2/run_cpu_0size_regr.sh @@ -11,7 +11,6 @@ LOG_DIR="tester/api_config/test_log_cpu_0size_regr" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=15 GPU_IDS="-1" -REQUIRED_MEMORY=5 TIME_OUT=600 TEST_MODE_ARGS=( @@ -33,7 +32,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( --timeout="$TIME_OUT" diff --git a/test_pipeline/V2/run_cpu_accuracy_full.sh b/test_pipeline/V2/run_cpu_accuracy_full.sh index eae53744..aae784cc 100644 --- a/test_pipeline/V2/run_cpu_accuracy_full.sh +++ b/test_pipeline/V2/run_cpu_accuracy_full.sh @@ -14,7 +14,6 @@ LOG_DIR="tester/api_config/test_log_cpu_accuracy_full" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=-1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TEST_MODE_ARGS=( --accuracy=True @@ -35,7 +34,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/run_cpu_accuracy_regr.sh b/test_pipeline/V2/run_cpu_accuracy_regr.sh index 66c0a2ab..db676bd0 100644 --- a/test_pipeline/V2/run_cpu_accuracy_regr.sh +++ b/test_pipeline/V2/run_cpu_accuracy_regr.sh @@ -11,7 +11,6 @@ LOG_DIR="tester/api_config/test_log_cpu_accuracy_regr" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=-1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TEST_MODE_ARGS=( --accuracy=True @@ -32,7 +31,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/run_gpu_0size_full.sh b/test_pipeline/V2/run_gpu_0size_full.sh index de6fae4c..31899793 100644 --- a/test_pipeline/V2/run_gpu_0size_full.sh +++ b/test_pipeline/V2/run_gpu_0size_full.sh @@ -12,7 +12,6 @@ LOG_DIR="tester/api_config/test_log_gpu_0size_full" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=15 GPU_IDS="-1" -REQUIRED_MEMORY=5 TIME_OUT=600 TEST_MODE_ARGS=( @@ -34,7 +33,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( --timeout="$TIME_OUT" diff --git a/test_pipeline/V2/run_gpu_0size_regr.sh b/test_pipeline/V2/run_gpu_0size_regr.sh index 33c72333..ffaa2237 100644 --- a/test_pipeline/V2/run_gpu_0size_regr.sh +++ b/test_pipeline/V2/run_gpu_0size_regr.sh @@ -11,7 +11,6 @@ LOG_DIR="tester/api_config/test_log_gpu_0size_regr" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=15 GPU_IDS="-1" -REQUIRED_MEMORY=5 TIME_OUT=600 TEST_MODE_ARGS=( @@ -33,7 +32,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( --timeout="$TIME_OUT" diff --git a/test_pipeline/V2/run_gpu_accuracy_full.sh b/test_pipeline/V2/run_gpu_accuracy_full.sh index dd32a966..3a6b1d1a 100644 --- a/test_pipeline/V2/run_gpu_accuracy_full.sh +++ b/test_pipeline/V2/run_gpu_accuracy_full.sh @@ -11,7 +11,6 @@ LOG_DIR="tester/api_config/test_log_gpu_accuracy_full" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=-1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TEST_MODE_ARGS=( --accuracy=True @@ -32,7 +31,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/run_gpu_accuracy_regr.sh b/test_pipeline/V2/run_gpu_accuracy_regr.sh index 1522c797..72b0ccb0 100644 --- a/test_pipeline/V2/run_gpu_accuracy_regr.sh +++ b/test_pipeline/V2/run_gpu_accuracy_regr.sh @@ -11,7 +11,6 @@ LOG_DIR="tester/api_config/test_log_gpu_accuracy_regr" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=-1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 backprocess="${ppapitest_bp:-1}" TEST_MODE_ARGS=( @@ -33,7 +32,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/run_gpu_accuracy_stable.sh b/test_pipeline/V2/run_gpu_accuracy_stable.sh index 181683e5..9c05bced 100644 --- a/test_pipeline/V2/run_gpu_accuracy_stable.sh +++ b/test_pipeline/V2/run_gpu_accuracy_stable.sh @@ -13,7 +13,6 @@ LOG_DIR="tester/api_config/test_log_accuracy_stable" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=4 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TEST_MODE_ARGS=( # --accuracy=True @@ -41,7 +40,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/run_gpu_accuracy_tolerance.sh b/test_pipeline/V2/run_gpu_accuracy_tolerance.sh index d08fe50e..3ce6292d 100644 --- a/test_pipeline/V2/run_gpu_accuracy_tolerance.sh +++ b/test_pipeline/V2/run_gpu_accuracy_tolerance.sh @@ -13,7 +13,6 @@ LOG_DIR="tester/api_config/test_log_tol" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=-1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TEST_MODE_ARGS=( --accuracy=True @@ -37,7 +36,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/run_gpu_bigtensor_full.sh b/test_pipeline/V2/run_gpu_bigtensor_full.sh index 27bbf79f..029307cf 100644 --- a/test_pipeline/V2/run_gpu_bigtensor_full.sh +++ b/test_pipeline/V2/run_gpu_bigtensor_full.sh @@ -12,7 +12,6 @@ LOG_DIR="tester/api_config/test_log_gpu_bigtensor_full" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TEST_MODE_ARGS=( --accuracy=True @@ -33,7 +32,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/run_gpu_bigtensor_regr.sh b/test_pipeline/V2/run_gpu_bigtensor_regr.sh index 0ff2e28a..b4099957 100644 --- a/test_pipeline/V2/run_gpu_bigtensor_regr.sh +++ b/test_pipeline/V2/run_gpu_bigtensor_regr.sh @@ -11,7 +11,6 @@ LOG_DIR="tester/api_config/test_log_gpu_bigtensor_regr" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TEST_MODE_ARGS=( --accuracy=True @@ -32,7 +31,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V2/run_paddle_gpu_performance.sh b/test_pipeline/V2/run_paddle_gpu_performance.sh index 4570bc2b..44594f35 100644 --- a/test_pipeline/V2/run_paddle_gpu_performance.sh +++ b/test_pipeline/V2/run_paddle_gpu_performance.sh @@ -11,7 +11,6 @@ LOG_DIR="tester/api_config/test_log_paddle_performance" NUM_GPUS=8 NUM_WORKERS_PER_GPU=1 GPU_IDS="0,1,2,3,4,5,6,7" -# REQUIRED_MEMORY=10 backprocess="${ppapitest_bp:-1}" TEST_MODE_ARGS=( @@ -34,7 +33,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) mkdir -p "$LOG_DIR" || { diff --git a/test_pipeline/V4/run_cpu_0size_full.sh b/test_pipeline/V4/run_cpu_0size_full.sh index 50ede083..8ffa79fe 100755 --- a/test_pipeline/V4/run_cpu_0size_full.sh +++ b/test_pipeline/V4/run_cpu_0size_full.sh @@ -30,7 +30,6 @@ LOG_DIR="tester/api_config/test_log_cpu_0size_full" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=10 GPU_IDS="-1" -# REQUIRED_MEMORY=5 TIME_OUT=600 RANDOM_SEED=2025 @@ -135,7 +134,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( diff --git a/test_pipeline/V4/run_dsv4_0size_accuracy.sh b/test_pipeline/V4/run_dsv4_0size_accuracy.sh index c4b05f1e..a55eec45 100755 --- a/test_pipeline/V4/run_dsv4_0size_accuracy.sh +++ b/test_pipeline/V4/run_dsv4_0size_accuracy.sh @@ -41,7 +41,6 @@ LOG_DIR="tester/api_config/test_log_dsv4_0size_accuracy" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=4 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TIME_OUT=600 # ── 测试模式（取消注释启用）────────────────────────────────── @@ -155,7 +154,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( diff --git a/test_pipeline/V4/run_dsv4_0size_paddleonly.sh b/test_pipeline/V4/run_dsv4_0size_paddleonly.sh index 3a1a00fe..1521d3df 100755 --- a/test_pipeline/V4/run_dsv4_0size_paddleonly.sh +++ b/test_pipeline/V4/run_dsv4_0size_paddleonly.sh @@ -41,7 +41,6 @@ LOG_DIR="tester/api_config/test_log_dsv4_0size_paddleonly" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TIME_OUT=600 # ── 测试模式（取消注释启用）────────────────────────────────── @@ -155,7 +154,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( diff --git a/test_pipeline/V4/run_dsv4_1M_accuracy.sh b/test_pipeline/V4/run_dsv4_1M_accuracy.sh index 005fbb75..8a025df6 100755 --- a/test_pipeline/V4/run_dsv4_1M_accuracy.sh +++ b/test_pipeline/V4/run_dsv4_1M_accuracy.sh @@ -41,7 +41,6 @@ LOG_DIR="tester/api_config/test_log_dsv4_1M_accuracy" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TIME_OUT=1200 # ── 测试模式（取消注释启用）────────────────────────────────── @@ -155,7 +154,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( diff --git a/test_pipeline/V4/run_dsv4_1M_paddleonly.sh b/test_pipeline/V4/run_dsv4_1M_paddleonly.sh index d505ab72..8fc72a21 100755 --- a/test_pipeline/V4/run_dsv4_1M_paddleonly.sh +++ b/test_pipeline/V4/run_dsv4_1M_paddleonly.sh @@ -41,7 +41,6 @@ LOG_DIR="tester/api_config/test_log_dsv4_1M_paddleonly" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TIME_OUT=1200 # ── 测试模式（取消注释启用）────────────────────────────────── @@ -155,7 +154,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( diff --git a/test_pipeline/V4/run_dsv4_v2_accuracy_manual_threshold.sh b/test_pipeline/V4/run_dsv4_v2_accuracy_manual_threshold.sh index 261df946..ae632e57 100644 --- a/test_pipeline/V4/run_dsv4_v2_accuracy_manual_threshold.sh +++ b/test_pipeline/V4/run_dsv4_v2_accuracy_manual_threshold.sh @@ -42,7 +42,6 @@ LOG_DIR="tester/api_config/monitor_config/dsv4_v2/test9" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TIME_OUT=1200 # ── 测试模式（取消注释启用）────────────────────────────────── @@ -158,7 +157,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( diff --git a/test_pipeline/V4/run_gpu_0size_full.sh b/test_pipeline/V4/run_gpu_0size_full.sh index b5476880..f898b291 100755 --- a/test_pipeline/V4/run_gpu_0size_full.sh +++ b/test_pipeline/V4/run_gpu_0size_full.sh @@ -30,7 +30,6 @@ LOG_DIR="tester/api_config/test_log_gpu_0size_full" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=10 GPU_IDS="-1" -# REQUIRED_MEMORY=5 TIME_OUT=300 RANDOM_SEED=2025 @@ -135,7 +134,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( diff --git a/test_pipeline/V4/run_gpu_accuracy_full.sh b/test_pipeline/V4/run_gpu_accuracy_full.sh index 3d7659e5..825f6b51 100755 --- a/test_pipeline/V4/run_gpu_accuracy_full.sh +++ b/test_pipeline/V4/run_gpu_accuracy_full.sh @@ -39,7 +39,6 @@ LOG_DIR="tester/api_config/test_log_gpu_accuracy_full" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=10 GPU_IDS="-1" -# REQUIRED_MEMORY=10 # ── 测试模式 ────────────────────────────────────────────────── TEST_MODE_ARGS=( @@ -147,7 +146,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) SHOW_RUNTIME_STATUS_ARGS=( diff --git a/test_pipeline/V4/run_gpu_bigtensor_full.sh b/test_pipeline/V4/run_gpu_bigtensor_full.sh index 327bb4c1..8f85b89e 100755 --- a/test_pipeline/V4/run_gpu_bigtensor_full.sh +++ b/test_pipeline/V4/run_gpu_bigtensor_full.sh @@ -30,7 +30,6 @@ LOG_DIR="tester/api_config/test_log_gpu_bigtensor_full" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=10 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TEST_MODE_ARGS=( --accuracy=True @@ -133,7 +132,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) SANITIZER_ARGS=( diff --git a/test_pipeline/V4/run_v2_0size_accuracy.sh b/test_pipeline/V4/run_v2_0size_accuracy.sh index bd4720e0..93f2866d 100755 --- a/test_pipeline/V4/run_v2_0size_accuracy.sh +++ b/test_pipeline/V4/run_v2_0size_accuracy.sh @@ -41,7 +41,6 @@ LOG_DIR="tester/api_config/test_log_v2_0size_accuracy" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=4 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TIME_OUT=600 # ── 测试模式（取消注释启用）────────────────────────────────── @@ -155,7 +154,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( diff --git a/test_pipeline/V4/run_v2_0size_paddleonly.sh b/test_pipeline/V4/run_v2_0size_paddleonly.sh index fa80a2a4..b86d62db 100755 --- a/test_pipeline/V4/run_v2_0size_paddleonly.sh +++ b/test_pipeline/V4/run_v2_0size_paddleonly.sh @@ -41,7 +41,6 @@ LOG_DIR="tester/api_config/test_log_v2_0size_paddleonly" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TIME_OUT=600 # ── 测试模式（取消注释启用）────────────────────────────────── @@ -155,7 +154,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( diff --git a/test_pipeline/V4/run_v2_1M_accuracy.sh b/test_pipeline/V4/run_v2_1M_accuracy.sh index a446e17e..23c81a46 100755 --- a/test_pipeline/V4/run_v2_1M_accuracy.sh +++ b/test_pipeline/V4/run_v2_1M_accuracy.sh @@ -41,7 +41,6 @@ LOG_DIR="tester/api_config/test_log_v2_1M_accuracy" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TIME_OUT=1200 # ── 测试模式（取消注释启用）────────────────────────────────── @@ -155,7 +154,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( diff --git a/test_pipeline/V4/run_v2_1M_paddleonly.sh b/test_pipeline/V4/run_v2_1M_paddleonly.sh index e7e0727e..a061063d 100755 --- a/test_pipeline/V4/run_v2_1M_paddleonly.sh +++ b/test_pipeline/V4/run_v2_1M_paddleonly.sh @@ -41,7 +41,6 @@ LOG_DIR="tester/api_config/test_log_v2_1M_paddleonly" NUM_GPUS=-1 NUM_WORKERS_PER_GPU=1 GPU_IDS="-1" -# REQUIRED_MEMORY=10 TIME_OUT=1200 # ── 测试模式（取消注释启用）────────────────────────────────── @@ -155,7 +154,6 @@ PARALLEL_ARGS=( --num_gpus="$NUM_GPUS" --num_workers_per_gpu="$NUM_WORKERS_PER_GPU" --gpu_ids="$GPU_IDS" - # --required_memory="$REQUIRED_MEMORY" ) TIME_OUT_ARGS=( diff --git a/test_pipeline/run_config.schema.json b/test_pipeline/run_config.schema.json index 95fd96b2..670707fb 100644 --- a/test_pipeline/run_config.schema.json +++ b/test_pipeline/run_config.schema.json @@ -71,7 +71,6 @@ "num_gpus": {"type": "integer", "description": "对应 --num_gpus。"}, "num_workers_per_gpu": {"type": "integer", "description": "对应 --num_workers_per_gpu。"}, "gpu_ids": {"type": "string", "description": "对应 --gpu_ids。"}, - "required_memory": {"type": "number", "description": "对应 --required_memory。"}, "test_cpu": {"type": "boolean", "description": "对应 --test_cpu。"}, "use_cached_numpy": {"type": "boolean", "description": "对应 --use_cached_numpy。"}, "use_gpu_mode": {"type": "boolean", "description": "对应 --use_gpu_mode。"}, diff --git a/test_pipeline/run_config.yaml b/test_pipeline/run_config.yaml index d4b27b6a..3c2aedf1 100644 --- a/test_pipeline/run_config.yaml +++ b/test_pipeline/run_config.yaml @@ -119,7 +119,6 @@ engine_args: num_gpus: -1 num_workers_per_gpu: -1 gpu_ids: "4-7" - # required_memory: 10.0 # ========== 运行控制 ========== timeout: 600 diff --git a/tester/accuracy.py b/tester/accuracy.py index c361b911..8dbe034a 100644 --- a/tester/accuracy.py +++ b/tester/accuracy.py @@ -245,7 +245,14 @@ def test(self): else: del self.torch_args, self.torch_kwargs - keep_torch_outputs_on_device = self.use_gpu_mode + spill_torch_outputs = False + if self.use_gpu_mode: + spill_torch_outputs = gpu_mode_maybe_empty_cache( + self.gpu_mode_config, + "after_torch", + request_spill=True, + ) + keep_torch_outputs_on_device = self.use_gpu_mode and not spill_torch_outputs def process_torch_outputs(obj): if isinstance(obj, (torch.return_types.max, torch.return_types.min)): @@ -270,7 +277,11 @@ def process_torch_outputs(obj): gc.collect() if self.use_gpu_mode: self.clear_torch_tensor() - gpu_mode_maybe_empty_cache(self.gpu_mode_config, "before_paddle") + gpu_mode_maybe_empty_cache( + self.gpu_mode_config, + "before_paddle", + force=not keep_torch_outputs_on_device, + ) else: torch.cuda.empty_cache() diff --git a/tester/api_config/config_analyzer.py b/tester/api_config/config_analyzer.py index 28f68afd..ef58feda 100644 --- a/tester/api_config/config_analyzer.py +++ b/tester/api_config/config_analyzer.py @@ -278,6 +278,35 @@ def _make_gpu_paddle_dense_tensor(self, dtype=None): base_dtype = "float16" if dtype in FLOAT8_DTYPES else "float32" return ((paddle.rand(shape, dtype=base_dtype) - 0.5) * 1.2).cast(dtype) + def _make_gpu_torch_low_temp_dense_tensor(self, dtype, pattern_dtype=None): + pattern_dtype = pattern_dtype or torch.float32 + torch_dtype = self.convert_dtype_to_torch_type(dtype) + shape = tuple(self.shape) + device = torch.device("cuda", torch.cuda.current_device()) + tensor = torch.empty(shape, device=device, dtype=torch_dtype) + flat_tensor = tensor.reshape(-1) + numel = self.numel() + if numel == 0: + return tensor + + chunk_elems = min(numel, 8 * 1024 * 1024) + pattern_elems = min(chunk_elems, 1024 * 1024) + pattern = torch.linspace( + -0.6, + 0.6, + steps=pattern_elems, + device=device, + dtype=pattern_dtype, + ).to(dtype=torch_dtype) + for start in range(0, numel, chunk_elems): + end = min(start + chunk_elems, numel) + offset = start + while offset < end: + next_offset = min(offset + pattern_elems, end) + flat_tensor[offset:next_offset] = pattern[: next_offset - offset] + offset = next_offset + return tensor + def _make_gpu_torch_dense_tensor(self, dtype=None): dtype = dtype or self.dtype torch_dtype = self.convert_dtype_to_torch_type(dtype) @@ -294,8 +323,11 @@ def _make_gpu_torch_dense_tensor(self, dtype=None): real_part = (torch.rand(shape, device=device, dtype=real_dtype) - 0.5) * 1.2 imag_part = (torch.rand(shape, device=device, dtype=real_dtype) - 0.5) * 1.2 return (real_part + 1j * imag_part).to(dtype=torch_dtype) - base_dtype = torch.float16 if dtype in FLOAT8_DTYPES else torch.float32 - return ((torch.rand(shape, device=device, dtype=base_dtype) - 0.5) * 1.2).to( + if dtype in FLOAT8_DTYPES: + return self._make_gpu_torch_low_temp_dense_tensor(dtype, pattern_dtype=torch.float16) + if dtype in ("float16", "bfloat16"): + return self._make_gpu_torch_low_temp_dense_tensor(dtype, pattern_dtype=torch.float32) + return ((torch.rand(shape, device=device, dtype=torch.float32) - 0.5) * 1.2).to( dtype=torch_dtype ) @@ -373,12 +405,13 @@ def _make_gpu_tensor_pair(self, api_config, dtype=None): if dtype in FLOAT8_DTYPES: paddle_tensor = paddle_source.cast(dtype) torch_tensor = torch_source.to(dtype=self.convert_dtype_to_torch_type(dtype)) + del torch_source, paddle_source else: paddle_tensor = paddle_source - torch_tensor = torch_source.detach().clone() + torch_tensor = torch_source + del torch_source paddle_tensor = self._set_paddle_autograd(paddle_tensor, api_config, dtype) torch_tensor = self._set_torch_autograd(torch_tensor, api_config, dtype) - del torch_source return paddle_tensor, torch_tensor def get_gpu_paddle_tensor(self, api_config, dtype=None): diff --git a/tester/base.py b/tester/base.py index adf9e09b..d8c092c1 100644 --- a/tester/base.py +++ b/tester/base.py @@ -66,7 +66,7 @@ def __getattr__(self, name): ) -def gpu_mode_maybe_empty_cache(gpu_config, phase="", force=False): +def gpu_mode_maybe_empty_cache(gpu_config, phase="", force=False, request_spill=False): if not gpu_config.enabled: return False try: @@ -75,45 +75,45 @@ def gpu_mode_maybe_empty_cache(gpu_config, phase="", force=False): except Exception: return False - should_cleanup = bool(force) - if not should_cleanup: - total_memory = float(gpu_config.total_memory or 0.0) - workers_on_gpu = max(1, int(gpu_config.workers_on_gpu or 1)) - required_memory = float(gpu_config.required_memory or 10.0) - memory_fraction = float(gpu_config.memory_fraction or 0.85) - pressure_ratio = float(gpu_config.cleanup_pressure_ratio or 0.25) - used_ratio = float(gpu_config.cleanup_used_ratio or 0.90) + try: + torch_reserved = torch.cuda.memory_reserved() / (1024**3) + except Exception: + torch_reserved = 0.0 + try: + torch_allocated = torch.cuda.memory_allocated() / (1024**3) + except Exception: + torch_allocated = 0.0 - try: - torch_reserved = torch.cuda.memory_reserved() / (1024**3) - except Exception: - torch_reserved = 0.0 - try: - torch_allocated = torch.cuda.memory_allocated() / (1024**3) - except Exception: - torch_allocated = 0.0 + memory_budget = float(getattr(gpu_config, "memory_budget", 0.0) or 0.0) + pressure_ratio = float(getattr(gpu_config, "cleanup_pressure_ratio", 0.25) or 0.25) + used_ratio = float(getattr(gpu_config, "cleanup_used_ratio", 0.90) or 0.90) + idle_reserved = max(0.0, torch_reserved - torch_allocated) + over_budget = memory_budget > 0 and torch_reserved >= memory_budget * used_ratio + should_spill = bool(request_spill and over_budget) - if total_memory <= 0: + should_cleanup = bool(force or should_spill) + if not should_cleanup: + if memory_budget <= 0: should_cleanup = ( torch_reserved > 0 and torch_allocated / max(torch_reserved, 1e-6) < 0.5 ) else: - per_worker_budget = total_memory * memory_fraction / workers_on_gpu - cleanup_reserved_threshold = max(required_memory, per_worker_budget * used_ratio) - cleanup_idle_threshold = max(1.0, per_worker_budget * pressure_ratio) - idle_reserved = max(0.0, torch_reserved - torch_allocated) - should_cleanup = ( - torch_reserved >= cleanup_reserved_threshold - or idle_reserved >= cleanup_idle_threshold + should_cleanup = over_budget or idle_reserved >= max( + 1.0, memory_budget * pressure_ratio ) - if not should_cleanup: - return False + if should_cleanup: + action = "spill+empty_cache" if should_spill else "empty_cache" + print( + f"[gpu_mode_memory] {phase or 'unknown'} {action} " + f"budget={memory_budget:.1f}G reserved={torch_reserved:.1f}G allocated={torch_allocated:.1f}G", + flush=True, + ) + gc.collect() + torch.cuda.empty_cache() + paddle.device.cuda.empty_cache() - gc.collect() - torch.cuda.empty_cache() - paddle.device.cuda.empty_cache() - return True + return should_spill if request_spill else should_cleanup def classify_runtime_error(error_msg): diff --git a/tester/runtime_config.py b/tester/runtime_config.py index 27760c80..032e5283 100644 --- a/tester/runtime_config.py +++ b/tester/runtime_config.py @@ -6,9 +6,9 @@ @dataclass(frozen=True) class GpuModeConfig: enabled: bool = False - required_memory: float = 10.0 workers_on_gpu: int = 1 total_memory: float = 0.0 + memory_budget: float = 0.0 memory_fraction: float = 0.85 cleanup_pressure_ratio: float = 0.25 cleanup_used_ratio: float = 0.90 @@ -25,7 +25,6 @@ class TestRuntimeConfig: def from_options(cls, options): gpu_mode = GpuModeConfig( enabled=bool(options.use_gpu_mode), - required_memory=float(options.required_memory), ) return cls( random_seed=int(options.random_seed), @@ -35,19 +34,28 @@ def from_options(cls, options): ) def for_gpu(self, gpu_id, workers_per_gpu, total_memory_per_gpu): - workers_on_gpu = workers_per_gpu.get(gpu_id, self.gpu_mode.workers_on_gpu) - total_memory = total_memory_per_gpu.get(gpu_id, self.gpu_mode.total_memory) + workers_on_gpu = max(1, int(workers_per_gpu.get(gpu_id, self.gpu_mode.workers_on_gpu) or 1)) + total_memory = float(total_memory_per_gpu.get(gpu_id, self.gpu_mode.total_memory) or 0.0) + memory_budget = ( + total_memory * self.gpu_mode.memory_fraction / workers_on_gpu + if total_memory > 0 + else 0.0 + ) gpu_mode = replace( self.gpu_mode, - workers_on_gpu=max(1, int(workers_on_gpu or 1)), - total_memory=float(total_memory or 0.0), + workers_on_gpu=workers_on_gpu, + total_memory=total_memory, + memory_budget=memory_budget, ) return replace(self, gpu_mode=gpu_mode) def runtime_config_for_gpu(options, gpu_id): - return options.runtime_config.for_gpu( + runtime_config = getattr(options, "runtime_config", None) + if runtime_config is None: + runtime_config = TestRuntimeConfig.from_options(options) + return runtime_config.for_gpu( gpu_id, - options.gpu_workers_per_gpu_map, - options.gpu_total_memory_map, + getattr(options, "gpu_workers_per_gpu_map", {}) or {}, + getattr(options, "gpu_total_memory_map", {}) or {}, ) From 1cf7af7390ea2654c09e8fbca75881a47fd79e6d Mon Sep 17 00:00:00 2001 From: cangtianhuang <1903374751@qq.com> Date: Mon, 20 Jul 2026 16:04:53 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Fix=20accuracy=5Fstabl?= =?UTF-8?q?e=20oom?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- engineV4.py | 60 +--- tester/accuracy_stable.py | 214 +++++++------- tester/api_config/config_analyzer.py | 70 +++-- tester/base.py | 418 +++++++++++++++------------ tester/paddle_to_torch/rules.py | 8 +- 5 files changed, 391 insertions(+), 379 deletions(-) diff --git a/engineV4.py b/engineV4.py index 29c57427..c9732b33 100644 --- a/engineV4.py +++ b/engineV4.py @@ -1253,7 +1253,7 @@ def check_gpu_memory(gpu_ids, num_workers_per_gpu): return available_gpus, max_workers_per_gpu -def run_test_case(api_config_str, options): +def _execute_test_case(api_config_str, options): """Run a single test case for the given API configuration.""" cuda_visible = os.environ.get("CUDA_VISIBLE_DEVICES", "0") gpu_id = int(cuda_visible.split(",")[0]) @@ -1334,9 +1334,12 @@ def run_test_case(api_config_str, options): # if not fatal error, subprocess will be alive and report error print(f"[test error] {api_config_str}: {err}", flush=True) raise + + +def run_test_case(api_config_str, options): + try: + return _execute_test_case(api_config_str, options) finally: - del test_class, api_config, case - gc.collect() if not any( getattr(options, opt) for opt in ( @@ -1344,7 +1347,7 @@ def run_test_case(api_config_str, options): "torch_gpu_performance", "paddle_torch_gpu_performance", ) - ) and not getattr(options, "use_gpu_mode", False): + ): _clear_device_cache(options) @@ -1709,63 +1712,16 @@ def main(): init_log(options.log_dir, worker_tmp_logs=True) options.api_config = options.api_config.strip() - print( - f"{datetime.now()} [paddle {paddle_version}] test begin: {options.api_config}", - flush=True, - ) - try: - api_config = APIConfig(options.api_config) - except Exception as err: - print(f"[config_parse] {options.api_config} {err!s}", flush=True) - return - - test_class = _select_test_class(options) - - if options.test_cpu: - import paddle - - paddle.device.set_device("cpu") - if options.custom_device_vs_gpu: - # custom_device_vs_gpu 模式需要传递额外参数 - case = test_class( - api_config, - operation_mode=options.operation_mode, - bos_path=options.bos_path, - bos_conf_path=options.bos_conf_path, - bcecmd_path=options.bcecmd_path, - random_seed=options.random_seed, - atol=options.atol, - rtol=options.rtol, - ) - elif options.accuracy: - case = test_class( - api_config, - test_amp=options.test_amp, - atol=options.atol, - rtol=options.rtol, - manual_threshold_config_file=options.manual_threshold_config_file, - test_tol=options.test_tol, - bitwise_alignment=options.bitwise_alignment, - exit_on_error=options.exit_on_error, - use_gpu_mode=options.use_gpu_mode, - runtime_config=options.runtime_config, - ) - else: - case = test_class(api_config, test_amp=options.test_amp) try: - case.test() + run_test_case(options.api_config, options) except Exception as err: if ( "Tensor-likes are not equal" in str(err) or "Mismatched elements" in str(err) - or "Tensor-likes are not equal" in str(err) or "Error Message Summary" in str(err) ): exit(1) print(f"[test error] {options.api_config}: {err}", flush=True) - finally: - case.clear_tensor() - del case elif options.api_config_file or options.api_config_file_pattern: # validate GPU options gpu_ids = validate_gpu_options(options) diff --git a/tester/accuracy_stable.py b/tester/accuracy_stable.py index badda47d..0d6d6db4 100644 --- a/tester/accuracy_stable.py +++ b/tester/accuracy_stable.py @@ -1,6 +1,6 @@ from __future__ import annotations -import os +import gc import traceback import numpy @@ -17,7 +17,7 @@ write_to_comp_log, write_to_log, ) -from .base import CUDA_ERROR, CUDA_OOM, APITestBase +from .base import CUDA_ERROR, CUDA_OOM, APITestBase, gpu_mode_maybe_empty_cache from .paddle_to_torch import get_converter @@ -33,12 +33,52 @@ class APITestAccuracyStable(APITestBase): } def __init__(self, api_config, **kwargs): - super().__init__(api_config) + super().__init__(api_config, runtime_config=kwargs.get("runtime_config")) self.test_amp = kwargs.get("test_amp", False) + self.use_gpu_mode = self.gpu_mode_config.enabled self.converter = get_converter() torch.set_printoptions(profile="short", edgeitems=2, threshold=100, linewidth=120) torch.set_default_device("cuda") + def _detach_result(self, value): + if isinstance(value, (torch.Tensor, paddle.Tensor)): + return value.detach() + if isinstance(value, list): + return [self._detach_result(item) for item in value] + if isinstance(value, tuple): + return tuple(self._detach_result(item) for item in value) + if isinstance(value, dict): + return {key: self._detach_result(item) for key, item in value.items()} + return value + + def _move_result_to_cpu(self, value): + if isinstance(value, torch.Tensor): + return value.cpu() + if isinstance(value, paddle.Tensor): + return value.cpu() + if isinstance(value, list): + return [self._move_result_to_cpu(item) for item in value] + if isinstance(value, tuple): + return tuple(self._move_result_to_cpu(item) for item in value) + if isinstance(value, dict): + return {key: self._move_result_to_cpu(item) for key, item in value.items()} + return value + + def _clear_runtime_inputs(self, framework): + for attr_name in (f"{framework}_args", f"{framework}_kwargs"): + if hasattr(self, attr_name): + delattr(self, attr_name) + gc.collect() + if self.use_gpu_mode: + gpu_mode_maybe_empty_cache( + self.gpu_mode_config, + f"accuracy_stable_after_{framework}", + ) + elif framework == "torch": + torch.cuda.empty_cache() + else: + paddle.device.cuda.empty_cache() + def _broadcast_to_comp_dimensions(self, log_type, affected_comps): """将执行阶段错误广播到所有受影响的 comp 维度""" for comp in affected_comps: @@ -126,14 +166,18 @@ def test(self): ) if torch_output is None: return - torch.cuda.empty_cache() + torch_output = self._detach_result(torch_output) + torch_out_grads = self._detach_result(torch_out_grads) + self._clear_runtime_inputs("torch") # ======== paddle ======== self._reset_random_state() paddle_output, paddle_out_grads = self.get_paddle_output(torch_grad_success, _i) if paddle_output is None: return - paddle.device.cuda.empty_cache() + paddle_output = self._detach_result(paddle_output) + paddle_out_grads = self._detach_result(paddle_out_grads) + self._clear_runtime_inputs("paddle") # ======== format ======== paddle_output, torch_output = process_output( @@ -150,9 +194,25 @@ def test(self): paddle_output_pair.append(paddle_output) paddle_grad_pair.append(paddle_out_grads) + if _i == 0: + self.compare(torch_output_pair[0], paddle_output_pair[0], "T1P1") + self.compare(torch_grad_pair[0], paddle_grad_pair[0], "T1P1B") + if self.use_gpu_mode: + torch_output_pair[0] = self._move_result_to_cpu(torch_output_pair[0]) + paddle_output_pair[0] = self._move_result_to_cpu(paddle_output_pair[0]) + torch_grad_pair[0] = self._move_result_to_cpu(torch_grad_pair[0]) + paddle_grad_pair[0] = self._move_result_to_cpu(paddle_grad_pair[0]) + torch_output = None + paddle_output = None + torch_out_grads = None + paddle_out_grads = None + gc.collect() + gpu_mode_maybe_empty_cache( + self.gpu_mode_config, + "accuracy_stable_after_first_compare_spill", + ) + # ======== summary ======== - self.compare(torch_output_pair[0], paddle_output_pair[0], "T1P1") - self.compare(torch_grad_pair[0], paddle_grad_pair[0], "T1P1B") self.compare(torch_output_pair[1], paddle_output_pair[1], "T2P2") self.compare(torch_grad_pair[1], paddle_grad_pair[1], "T2P2B") self.compare(torch_output_pair[0], paddle_output_pair[1], "T1P2") @@ -160,9 +220,19 @@ def test(self): self.compare(torch_output_pair[1], paddle_output_pair[0], "T2P1") self.compare(torch_grad_pair[1], paddle_grad_pair[0], "T2P1B") self.compare(torch_output_pair[0], torch_output_pair[1], "T1T2") + torch_output_pair.clear() self.compare(torch_grad_pair[0], torch_grad_pair[1], "T1T2B") + torch_grad_pair.clear() + gc.collect() + if self.use_gpu_mode: + gpu_mode_maybe_empty_cache( + self.gpu_mode_config, + "accuracy_stable_after_torch_compare", + ) self.compare(paddle_output_pair[0], paddle_output_pair[1], "P1P2") + paddle_output_pair.clear() self.compare(paddle_grad_pair[0], paddle_grad_pair[1], "P1P2B") + paddle_grad_pair.clear() # 逐维度写 pass for dimension in ALL_DIMENSIONS: @@ -195,16 +265,17 @@ def get_torch_output(self, convert_result, iter_idx=0): exec_locals["fused_log_softmax"] = False code = convert_result.code - if code.preprocess_compiled: - exec(code.preprocess_compiled, exec_globals, exec_locals) - if code.core_compiled: - if self.test_amp: - with torch.autocast(device_type="cuda"): + with torch.set_grad_enabled(self.need_check_grad()): + if code.preprocess_compiled: + exec(code.preprocess_compiled, exec_globals, exec_locals) + if code.core_compiled: + if self.test_amp: + with torch.autocast(device_type="cuda"): + exec(code.core_compiled, exec_globals, exec_locals) + else: exec(code.core_compiled, exec_globals, exec_locals) - else: - exec(code.core_compiled, exec_globals, exec_locals) - if code.postprocess_compiled: - exec(code.postprocess_compiled, exec_globals, exec_locals) + if code.postprocess_compiled: + exec(code.postprocess_compiled, exec_globals, exec_locals) output_var = convert_result.output_var or "result" torch_output = exec_locals[output_var] @@ -321,20 +392,21 @@ def get_paddle_output(self, torch_grad_success, iter_idx=0): if len(self.paddle_args) > 0 else next(iter(self.paddle_kwargs.values())) ) - if self.api_config.api_name.startswith("paddle.Tensor."): - api_name = self.api_config.api_name.split(".")[-1] - api = getattr(self.paddle_args[0], api_name) - if self.test_amp: - with paddle.amp.auto_cast(): + with paddle.set_grad_enabled(self.need_check_grad()): + if self.api_config.api_name.startswith("paddle.Tensor."): + api_name = self.api_config.api_name.split(".")[-1] + api = getattr(self.paddle_args[0], api_name) + if self.test_amp: + with paddle.amp.auto_cast(): + paddle_output = api(*self.paddle_args[1:], **self.paddle_kwargs) + else: paddle_output = api(*self.paddle_args[1:], **self.paddle_kwargs) else: - paddle_output = api(*self.paddle_args[1:], **self.paddle_kwargs) - else: - if self.test_amp: - with paddle.amp.auto_cast(): + if self.test_amp: + with paddle.amp.auto_cast(): + paddle_output = self.paddle_api(*self.paddle_args, **self.paddle_kwargs) + else: paddle_output = self.paddle_api(*self.paddle_args, **self.paddle_kwargs) - else: - paddle_output = self.paddle_api(*self.paddle_args, **self.paddle_kwargs) if ( self.api_config.api_name[-1] == "_" and self.api_config.api_name[-2:] != "__" ) or self.api_config.api_name == "paddle.Tensor.__setitem__": @@ -514,11 +586,6 @@ def compare(self, input1, input2, comp): return def assert_accuracy(self, tensor1, tensor2, comp, idx=0): - if not tensor1.is_contiguous(): - tensor1 = tensor1.contiguous() - if not tensor2.is_contiguous(): - tensor2 = tensor2.contiguous() - api_name = self.api_config.api_name config = self.api_config.config[:120000] dtype = self.api_config.dtype @@ -527,33 +594,16 @@ def assert_accuracy(self, tensor1, tensor2, comp, idx=0): first = "Paddle" if comp[0] == "P" else "Torch" second = "Paddle" if comp[2] == "P" else "Torch" - if isinstance(tensor1, paddle.Tensor): - dlpack = paddle.utils.dlpack.to_dlpack(tensor1) # type: ignore[reportGeneralTypeIssues] - tensor1 = torch.utils.dlpack.from_dlpack(dlpack) # type: ignore[reportGeneralTypeIssues] - if isinstance(tensor2, paddle.Tensor): - dlpack = paddle.utils.dlpack.to_dlpack(tensor2) # type: ignore[reportGeneralTypeIssues] - tensor2 = torch.utils.dlpack.from_dlpack(dlpack) # type: ignore[reportGeneralTypeIssues] - - def error_msg(msg): - return ( - f"Not equal to tolerance rtol=0.0, atol=0.0\n" - f"{msg}\n" - f"{first}: (shape={tensor1.shape}, dtype={tensor1.dtype})\n" - f"{tensor1}\n" - f"{second}: (shape={tensor2.shape}, dtype={tensor2.dtype})\n" - f"{tensor2}" - ) - try: - torch.testing.assert_close( + self.torch_assert_accuracy( tensor1, tensor2, - rtol=0.0, atol=0.0, - equal_nan=True, - check_device=False, + rtol=0.0, check_dtype=check_dtype, - msg=error_msg, + actual_name=first, + expected_name=second, + apply_special_tolerance=False, ) log_accuracy_stable( "Identical", @@ -565,57 +615,11 @@ def error_msg(msg): except Exception as err: err_str = str(err) is_acc_err = False - if err_str.startswith("Comparing"): - # torch_assert internal error (OOM on large tensor) - if os.environ.get("PADDLEAPITEST_NP_FALLBACK", "0") == "1": - # fallback to np_assert - print( - f"[torch_assert_OOM] comp={comp} torch.testing.assert_close OOM, fallback to np_assert", - flush=True, - ) - t1_cpu = tensor1.cpu() - t2_cpu = tensor2.cpu() - # numpy does not support bfloat16/float8, cast to float32 - if t1_cpu.dtype in (torch.bfloat16,) or "float8" in str(t1_cpu.dtype): - t1_cpu = t1_cpu.float() - if t2_cpu.dtype in (torch.bfloat16,) or "float8" in str(t2_cpu.dtype): - t2_cpu = t2_cpu.float() - try: - numpy.testing.assert_allclose( - t1_cpu.numpy(), - t2_cpu.numpy(), - rtol=0.0, - atol=0.0, - equal_nan=True, - strict=True, - ) - log_accuracy_stable( - "Identical", - api_name, - config, - dtype, - comp, - ) - return - except Exception as err_np: - err_str = str(err_np) - err_list = err_str.split("\n", maxsplit=3) - if len(err_list) > 3 and err_list[3].startswith("Mismatched elements"): - is_acc_err = True - else: - # np_assert also failed with unexpected error, raise np error (not torch OOM) - raise err_np - else: - # report as OOM directly, raise with recognizable prefix for outer layer - raise RuntimeError( - "[torch_assert_OOM] torch.testing.assert_close OOM on large tensor comparison" - ) - else: - err_list = err_str.split("\n", maxsplit=1) - if len(err_list) > 1 and ( - err_list[1].startswith("Tensor-likes") or err_list[1].startswith("Scalars") - ): - is_acc_err = True + err_list = err_str.split("\n", maxsplit=1) + if len(err_list) > 1 and ( + err_list[1].startswith("Tensor-likes") or err_list[1].startswith("Scalars") + ): + is_acc_err = True if is_acc_err: log_accuracy_stable( err_str, diff --git a/tester/api_config/config_analyzer.py b/tester/api_config/config_analyzer.py index ef58feda..9e3b1dd1 100644 --- a/tester/api_config/config_analyzer.py +++ b/tester/api_config/config_analyzer.py @@ -3017,43 +3017,41 @@ def get_exponent_max(value, dtype_max, default_max=5): if tokens_per_expert is not None: tokens_per_expert[:] = [0] * num_experts else: - # Each row randomly assigns 1~min(topk, num_experts) experts - max_assign = min(topk, num_experts) - n_assigned = numpy.random.randint(1, max_assign + 1, size=seqlen) - # For each possible n_assigned value, batch process all rows with that count - for n in range(1, max_assign + 1): - mask = n_assigned == n - count = int(mask.sum()) - if count == 0: - continue - # Generate random expert indices for these rows - expert_indices = numpy.array( - [ - numpy.random.choice(num_experts, size=n, replace=False) - for _ in range(count) - ], - dtype="int32", - ) - # Generate random positions for these rows - position_indices = numpy.array( - [ - numpy.random.choice(topk, size=n, replace=False) - for _ in range(count) - ], - dtype="int32", - ) - row_indices = numpy.where(mask)[0] - for j in range(n): - routemap[row_indices, position_indices[:, j]] = expert_indices[ - :, j - ] - self.numpy_tensor = routemap - # Update tokens_per_expert to match the generated routemap - tokens_count = [ - int(numpy.sum(routemap == e)) for e in range(num_experts) - ] tokens_per_expert = self.get_arg(api_config, 5, "tokens_per_expert") - tokens_per_expert[:] = tokens_count + if ( + isinstance(tokens_per_expert, list) + and len(tokens_per_expert) == num_experts + ): + total_assignments = sum(tokens_per_expert) + if total_assignments > seqlen * topk or any( + count < 0 or count > seqlen for count in tokens_per_expert + ): + raise ValueError( + "tokens_per_expert cannot be represented by the " + "expert_routemap_topk shape" + ) + cursor = 0 + for expert, count in enumerate(tokens_per_expert): + positions = numpy.arange(cursor, cursor + count, dtype="int64") + rows = positions % seqlen + columns = (positions // seqlen) % topk + routemap[rows, columns] = expert + cursor += count + else: + max_assign = min(topk, num_experts) + n_assigned = numpy.random.randint(1, max_assign + 1, size=seqlen) + for row, count in enumerate(n_assigned): + columns = numpy.random.choice(topk, size=count, replace=False) + routemap[row, columns] = numpy.random.choice( + num_experts, size=count, replace=False + ) + tokens_count = [ + int(numpy.sum(routemap == expert)) + for expert in range(num_experts) + ] + if tokens_per_expert is not None: + tokens_per_expert[:] = tokens_count + self.numpy_tensor = routemap # expert_prob_topk (arg3): float32, shape [seqlen, topk], value in [0, 1] elif self.check_arg(api_config, 3, "expert_prob_topk"): routemap_config = self.get_arg(api_config, 2, "expert_routemap_topk") diff --git a/tester/base.py b/tester/base.py index d8c092c1..337ad27f 100644 --- a/tester/base.py +++ b/tester/base.py @@ -1398,212 +1398,270 @@ def error_msg(msg): else: raise - def _cast_float8_for_compare(self, tensor): - """torch.testing.assert_close cannot compare float8 with non-zero tol.""" - dtype = str(tensor.dtype) - if "float8" in dtype: - return tensor.to(dtype=torch.float32) - return tensor - - def _prepare_torch_compare_tensors(self, paddle_tensor, torch_tensor): - if not paddle_tensor.is_contiguous(): - paddle_tensor = paddle_tensor.contiguous() - paddle_tensor = paddle_tensor.detach() - - if not torch_tensor.is_contiguous(): - torch_tensor = torch_tensor.contiguous() - torch_tensor = torch_tensor.detach() - - paddle_dlpack = paddle.utils.dlpack.to_dlpack(paddle_tensor) # type: ignore[reportGeneralTypeIssues] - converted_paddle_tensor = torch.utils.dlpack.from_dlpack(paddle_dlpack) # type: ignore[reportGeneralTypeIssues] - if torch_tensor.device != converted_paddle_tensor.device: - torch_tensor = torch_tensor.to(device=converted_paddle_tensor.device) - converted_paddle_tensor = self._cast_float8_for_compare(converted_paddle_tensor) - torch_tensor = self._cast_float8_for_compare(torch_tensor) - return converted_paddle_tensor, torch_tensor - - def _torch_compare_error_msg(self, actual, expected, atol, rtol, suffix=""): - def error_msg(msg): - suffix_text = f"\n{suffix}" if suffix else "" - return ( - f"Not equal to tolerance rtol={rtol}, atol={atol}{suffix_text}\n" - f"{msg}\n" - f"ACTUAL: (shape={actual.shape}, dtype={actual.dtype})\n" - f"{actual}\n" - f"DESIRED: (shape={expected.shape}, dtype={expected.dtype})\n" - f"{expected}" - ) + def _torch_assert_accuracy_in_chunks( + self, actual, expected, atol, rtol, error_msg, working_bytes + ): + temp_bytes_per_element = max(32, actual.element_size() + expected.element_size() + 16) + chunk_numel = max(1, working_bytes // temp_bytes_per_element) + actual_flat = actual.reshape(-1) + expected_flat = expected.reshape(-1) + mismatch_count = 0 + max_abs_diff = -1.0 + max_abs_index = 0 + max_rel_diff = -1.0 + max_rel_index = 0 + exact_compare = atol == 0.0 and rtol == 0.0 - return error_msg + print( + f"[chunk_compare] numel={actual_flat.numel()} chunk_numel={chunk_numel} " + f"dtype={actual.dtype}", + flush=True, + ) + for start in range(0, actual_flat.numel(), chunk_numel): + end = min(actual_flat.numel(), start + chunk_numel) + actual_chunk = actual_flat[start:end] + expected_chunk = expected_flat[start:end] + if exact_compare: + equal = actual_chunk == expected_chunk + if actual_chunk.is_floating_point() or actual_chunk.is_complex(): + equal |= torch.isnan(actual_chunk) & torch.isnan(expected_chunk) + mismatch = ~equal + else: + if "float8" in str(actual_chunk.dtype): + actual_chunk = actual_chunk.float() + if "float8" in str(expected_chunk.dtype): + expected_chunk = expected_chunk.float() + mismatch = ~torch.isclose( + actual_chunk, + expected_chunk, + rtol=rtol, + atol=atol, + equal_nan=True, + ) - def _assert_torch_close(self, actual, expected, atol, rtol, is_check_dtype, suffix=""): - torch.testing.assert_close( - actual, - expected, - rtol=rtol, - atol=atol, - equal_nan=True, - check_dtype=is_check_dtype, - msg=self._torch_compare_error_msg(actual, expected, atol, rtol, suffix), + chunk_mismatch_count = int(mismatch.sum().item()) + mismatch_count += chunk_mismatch_count + if chunk_mismatch_count == 0: + continue + + if actual_chunk.is_complex() or expected_chunk.is_complex(): + actual_for_diff = actual_chunk + expected_for_diff = expected_chunk + elif actual_chunk.dtype == torch.float64 or expected_chunk.dtype == torch.float64: + actual_for_diff = actual_chunk.to(torch.float64) + expected_for_diff = expected_chunk.to(torch.float64) + else: + actual_for_diff = actual_chunk.float() + expected_for_diff = expected_chunk.float() + abs_diff = torch.abs(actual_for_diff - expected_for_diff) + abs_diff = abs_diff.masked_fill(~mismatch, -1.0) + chunk_max_abs, chunk_abs_index = torch.max(abs_diff, dim=0) + chunk_max_abs_value = float(chunk_max_abs.item()) + if chunk_max_abs_value > max_abs_diff: + max_abs_diff = chunk_max_abs_value + max_abs_index = start + int(chunk_abs_index.item()) + + rel_diff = abs_diff / torch.abs(expected_for_diff) + rel_diff = rel_diff.masked_fill(~mismatch, -1.0) + chunk_max_rel, chunk_rel_index = torch.max(rel_diff, dim=0) + chunk_max_rel_value = float(chunk_max_rel.item()) + if chunk_max_rel_value > max_rel_diff: + max_rel_diff = chunk_max_rel_value + max_rel_index = start + int(chunk_rel_index.item()) + + if mismatch_count == 0: + return + + abs_index = tuple(int(value) for value in numpy.unravel_index(max_abs_index, actual.shape)) + rel_index = tuple(int(value) for value in numpy.unravel_index(max_rel_index, actual.shape)) + mismatch_percent = 100.0 * mismatch_count / actual.numel() + raise AssertionError( + error_msg( + "Tensor-likes are not equal!\n\n" + f"Mismatched elements: {mismatch_count} / {actual.numel()} " + f"({mismatch_percent:.1f}%)\n" + f"Greatest absolute difference: {max_abs_diff} at index {abs_index}\n" + f"Greatest relative difference: {max_rel_diff} at index {rel_index}" + ) ) - def _torch_assert_accuracy_cpu( - self, paddle_tensor, torch_tensor, atol, rtol, is_check_dtype, test_tol, is_backward + def torch_assert_accuracy( + self, + actual, + expected, + atol=1e-2, + rtol=1e-2, + check_dtype=None, + actual_name="ACTUAL", + expected_name="DESIRED", + apply_special_tolerance=True, ): - if not paddle_tensor.is_contiguous(): - paddle_tensor = paddle_tensor.contiguous() - paddle_tensor = paddle_tensor.cpu().detach() + is_check_dtype = self.should_check_dtype() if check_dtype is None else check_dtype + bitwise_alignment = getattr(self, "bitwise_alignment", False) - if not torch_tensor.is_contiguous(): - torch_tensor = torch_tensor.contiguous() - torch_tensor = torch_tensor.cpu().detach() + if ( + apply_special_tolerance + and not bitwise_alignment + and self.api_config.api_name in special_accuracy_atol_rtol + ): + atol, rtol = special_accuracy_atol_rtol[self.api_config.api_name] + test_tol = getattr(self, "test_tol", False) + is_backward = getattr(self, "is_backward", False) + if test_tol: + atol, rtol = 0.0, 0.0 - paddle_dlpack = paddle.utils.dlpack.to_dlpack(paddle_tensor) # type: ignore[reportGeneralTypeIssues] - converted_paddle_tensor = torch.utils.dlpack.from_dlpack(paddle_dlpack) # type: ignore[reportGeneralTypeIssues] - converted_paddle_tensor = self._cast_float8_for_compare(converted_paddle_tensor) - torch_tensor = self._cast_float8_for_compare(torch_tensor) + compare_on_cpu = test_tol or not self.gpu_mode_config.enabled + if isinstance(actual, paddle.Tensor): + if not actual.is_contiguous(): + actual = actual.contiguous() + actual = actual.detach() + if compare_on_cpu: + actual = actual.cpu() + actual_tensor = torch.utils.dlpack.from_dlpack( + paddle.utils.dlpack.to_dlpack(actual) # type: ignore[reportGeneralTypeIssues] + ) + elif isinstance(actual, torch.Tensor): + if not actual.is_contiguous(): + actual = actual.contiguous() + actual_tensor = actual.detach() + if compare_on_cpu: + actual_tensor = actual_tensor.cpu() + else: + raise TypeError(f"Expected Paddle or Torch tensor, but got {type(actual)}") + + if isinstance(expected, paddle.Tensor): + if not expected.is_contiguous(): + expected = expected.contiguous() + expected = expected.detach() + if compare_on_cpu: + expected = expected.cpu() + expected_tensor = torch.utils.dlpack.from_dlpack( + paddle.utils.dlpack.to_dlpack(expected) # type: ignore[reportGeneralTypeIssues] + ) + elif isinstance(expected, torch.Tensor): + if not expected.is_contiguous(): + expected = expected.contiguous() + expected_tensor = expected.detach() + if compare_on_cpu: + expected_tensor = expected_tensor.cpu() + else: + raise TypeError(f"Expected Paddle or Torch tensor, but got {type(expected)}") + + if actual_tensor.device != expected_tensor.device: + expected_tensor = expected_tensor.to(device=actual_tensor.device) + + if actual_tensor.shape != expected_tensor.shape: + raise AssertionError( + f"shape mismatch: {actual_name} {actual_tensor.shape}, " + f"{expected_name} {expected_tensor.shape}" + ) + if is_check_dtype and actual_tensor.dtype != expected_tensor.dtype: + raise AssertionError( + f"dtype mismatch: {actual_name} {actual_tensor.dtype}, " + f"{expected_name} {expected_tensor.dtype}" + ) + + exact_compare = atol == 0.0 and rtol == 0.0 + + def error_msg(msg): + return ( + f"Not equal to tolerance rtol={rtol}, atol={atol}\n" + f"{msg}\n" + f"{actual_name}: (shape={actual_tensor.shape}, dtype={actual_tensor.dtype})\n" + f"{actual_tensor}\n" + f"{expected_name}: (shape={expected_tensor.shape}, dtype={expected_tensor.dtype})\n" + f"{expected_tensor}" + ) try: - self._assert_torch_close( - converted_paddle_tensor, - torch_tensor, - atol, - rtol, - is_check_dtype, + working_bytes = 1024**3 + if actual_tensor.is_cuda: + try: + free_bytes, _ = torch.cuda.mem_get_info(actual_tensor.device) + working_bytes = min(working_bytes, max(256 * 1024**2, free_bytes // 50)) + except Exception: + pass + estimated_temp_bytes = actual_tensor.numel() * max( + 8, actual_tensor.element_size() + expected_tensor.element_size() + 4 + ) + if estimated_temp_bytes > working_bytes: + self._torch_assert_accuracy_in_chunks( + actual_tensor, + expected_tensor, + atol, + rtol, + error_msg, + working_bytes, + ) + if test_tol: + log_accuracy_tolerance( + "Identical", + self.api_config.api_name, + self.api_config.config[:120000], + str(actual.dtype), + is_backward, + ) + return + + # PyTorch supports direct FP8 comparison only for exact tolerances. + if not exact_compare: + if "float8" in str(actual_tensor.dtype): + actual_tensor = actual_tensor.float() + if "float8" in str(expected_tensor.dtype): + expected_tensor = expected_tensor.float() + torch.testing.assert_close( + actual_tensor, + expected_tensor, + rtol=rtol, + atol=atol, + equal_nan=True, + check_device=False, + check_dtype=is_check_dtype, + msg=error_msg, ) if test_tol: - api_name = self.api_config.api_name - config = self.api_config.config[:120000] log_accuracy_tolerance( "Identical", - api_name, - config, - str(paddle_tensor.dtype), + self.api_config.api_name, + self.api_config.config[:120000], + str(actual.dtype), is_backward, ) - except Exception as e: - error_str = str(e) + except Exception as err: + error_str = str(err) if error_str.startswith("Comparing"): - if os.environ.get("PADDLEAPITEST_NP_FALLBACK", "0") == "1": - print( - "[torch_assert_OOM] torch.testing.assert_close OOM, fallback to np_assert", - flush=True, - ) - _p = self._cast_float8_for_compare( - torch.utils.dlpack.from_dlpack( - paddle.utils.dlpack.to_dlpack(paddle_tensor.detach().cpu().contiguous()) - ) - ) - _t = self._cast_float8_for_compare(torch_tensor.detach().cpu().contiguous()) - # numpy does not support bfloat16, cast to float32 - if _p.dtype == torch.bfloat16: - _p = _p.float() - if _t.dtype == torch.bfloat16: - _t = _t.float() - self.np_assert_accuracy( - _p.numpy(), - _t.numpy(), - atol, - rtol, - ) - else: + if os.environ.get("PADDLEAPITEST_NP_FALLBACK", "0") != "1": raise RuntimeError( "[torch_assert_OOM] torch.testing.assert_close OOM on large tensor comparison" - ) - elif test_tol: + ) from err + print( + "[torch_assert_OOM] torch.testing.assert_close OOM, fallback to np_assert", + flush=True, + ) + actual_cpu = actual_tensor.cpu() + expected_cpu = expected_tensor.cpu() + if actual_cpu.dtype == torch.bfloat16 or "float8" in str(actual_cpu.dtype): + actual_cpu = actual_cpu.float() + if expected_cpu.dtype == torch.bfloat16 or "float8" in str(expected_cpu.dtype): + expected_cpu = expected_cpu.float() + self.np_assert_accuracy( + actual_cpu.numpy(), expected_cpu.numpy(), atol=atol, rtol=rtol + ) + return + if test_tol: error_info = error_str.split("\n", maxsplit=2)[1] if "\n" in error_str else None if error_info and ( error_info.startswith("Tensor-likes") or error_info.startswith("Scalars") ): - api_name = self.api_config.api_name - config = self.api_config.config[:120000] log_accuracy_tolerance( error_str, - api_name, - config, - str(paddle_tensor.dtype), + self.api_config.api_name, + self.api_config.config[:120000], + str(actual.dtype), is_backward, ) - else: - raise - - def _torch_assert_accuracy_gpu(self, paddle_tensor, torch_tensor, atol, rtol, is_check_dtype): - converted_paddle_tensor, torch_tensor = self._prepare_torch_compare_tensors( - paddle_tensor, torch_tensor - ) - if converted_paddle_tensor.shape != torch_tensor.shape: - raise AssertionError( - f"shape mismatch: paddle {converted_paddle_tensor.shape}, torch {torch_tensor.shape}" - ) - if is_check_dtype and converted_paddle_tensor.dtype != torch_tensor.dtype: - raise AssertionError( - f"dtype mismatch: paddle {converted_paddle_tensor.dtype}, torch {torch_tensor.dtype}" - ) - - try: - self._assert_torch_close( - converted_paddle_tensor, - torch_tensor, - atol, - rtol, - is_check_dtype, - ) - except Exception as e: - error_str = str(e) - if error_str.startswith("Comparing"): - if os.environ.get("PADDLEAPITEST_NP_FALLBACK", "0") == "1": - print( - "[torch_assert_OOM] torch.testing.assert_close OOM, fallback to np_assert", - flush=True, - ) - _p = converted_paddle_tensor.cpu() - _t = torch_tensor.cpu() - if _p.dtype == torch.bfloat16: - _p = _p.float() - if _t.dtype == torch.bfloat16: - _t = _t.float() - self.np_assert_accuracy( - _p.numpy(), - _t.numpy(), - atol, - rtol, - ) - else: - raise RuntimeError( - "[torch_assert_OOM] torch.testing.assert_close OOM on large tensor comparison" - ) - else: - raise - - def torch_assert_accuracy(self, paddle_tensor, torch_tensor, atol=1e-2, rtol=1e-2): - is_check_dtype = self.api_config.api_name not in not_check_dtype - bitwise_alignment = getattr(self, "bitwise_alignment", False) - - if not bitwise_alignment and self.api_config.api_name in special_accuracy_atol_rtol: - atol, rtol = special_accuracy_atol_rtol[self.api_config.api_name] - test_tol = getattr(self, "test_tol", False) - is_backward = getattr(self, "is_backward", False) - if test_tol: - atol, rtol = 0.0, 0.0 - - if test_tol or not self.gpu_mode_config.enabled: - self._torch_assert_accuracy_cpu( - paddle_tensor, - torch_tensor, - atol, - rtol, - is_check_dtype, - test_tol, - is_backward, - ) - return - self._torch_assert_accuracy_gpu( - paddle_tensor, - torch_tensor, - atol, - rtol, - is_check_dtype, - ) + return + raise def test(self): pass diff --git a/tester/paddle_to_torch/rules.py b/tester/paddle_to_torch/rules.py index 49fd3d3c..012a6c5f 100644 --- a/tester/paddle_to_torch/rules.py +++ b/tester/paddle_to_torch/rules.py @@ -4353,10 +4353,8 @@ def apply(self, paddle_api: str) -> ConvertResult: class MoePermuteRule(BaseRule): def apply(self, paddle_api: str) -> ConvertResult: core = """ -_orig_hs_dtype = hidden_states.dtype -# Compute in a supported dtype; cast gathered tokens back to the original dtype -# so float8 inputs match paddle.nn.functional.moe_permute output dtype. -hidden_states = hidden_states.to(torch.bfloat16) if hidden_states.dtype not in (torch.bfloat16, torch.float32, torch.float16) else hidden_states +# FP8 zeros, advanced indexing, and assignment are supported on CUDA. Keep FP8 +# throughout this gather to avoid materializing a full BF16 copy of hidden_states. do_gather = locals().get('do_gather', True) using_ue8m0_scale = locals().get('using_ue8m0_scale', False) return_expert_indices = locals().get('return_expert_indices', False) @@ -4456,8 +4454,6 @@ def apply(self, paddle_api: str) -> ConvertResult: hidden_states_unzipped[_valid_mask_g] = hidden_states[_gather_src[_valid_mask_g]] else: hidden_states_unzipped = torch.empty(0, token_dim, dtype=hidden_states.dtype, device=_dev) -if _orig_hs_dtype != hidden_states_unzipped.dtype: - hidden_states_unzipped = hidden_states_unzipped.to(_orig_hs_dtype) if return_expert_indices: expert_indices_out = _gather_expert_id result = (hidden_states_unzipped, rowmap, token_prob_unzipped, scale_unzipped, expert_indices_out) From b2dcab623213a01231538fd4a0ef284765197fe0 Mon Sep 17 00:00:00 2001 From: cangtianhuang <1903374751@qq.com> Date: Mon, 20 Jul 2026 20:42:34 +0800 Subject: [PATCH 7/7] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Imp=20gpu=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- engineV2.py | 16 +--- engineV4.py | 16 +--- tester/accuracy_stable.py | 66 ++++++++++++++++ tester/api_config/config_analyzer.py | 2 +- tester/base.py | 110 +++++++++++++++++---------- 5 files changed, 144 insertions(+), 66 deletions(-) diff --git a/engineV2.py b/engineV2.py index d88802ae..8d55f4c2 100644 --- a/engineV2.py +++ b/engineV2.py @@ -575,16 +575,7 @@ def run_test_case(api_config_str, options): flush=True, ) - total_memory, used_memory = get_memory_info(gpu_id) - free_memory = total_memory - used_memory runtime_config = runtime_config_for_gpu(options, gpu_id) - gpu_config = runtime_config.gpu_mode - print( - f"{datetime.now()} device {gpu_id} memory total={total_memory:.1f} GB, " - f"free={free_memory:.1f} GB, workers={gpu_config.workers_on_gpu}, " - f"budget={gpu_config.memory_budget:.1f} GB", - flush=True, - ) if options.show_runtime_status: total_memory, used_memory_before = get_memory_info(gpu_id) @@ -967,10 +958,9 @@ def main(): os.environ["USE_CACHED_NUMPY"] = str(options.use_cached_numpy) os.environ["USE_GPU_MODE"] = str(options.use_gpu_mode) if options.use_gpu_mode: - print( - "[gpu_mode] enabled: GPU tensor generation, GPU compare, and allocator reuse are active.", - flush=True, - ) + print("[gpu_mode] enabled: use GPU tensors and comparison.", flush=True) + elif options.use_cached_numpy: + print("[use_cached_numpy] enabled: reuse cached NumPy inputs.", flush=True) if options.bitwise_alignment: options.atol = 0.0 options.rtol = 0.0 diff --git a/engineV4.py b/engineV4.py index c9732b33..2a0fbec5 100644 --- a/engineV4.py +++ b/engineV4.py @@ -1263,16 +1263,7 @@ def _execute_test_case(api_config_str, options): flush=True, ) - total_memory, used_memory = get_memory_info(gpu_id) - free_memory = total_memory - used_memory runtime_config = runtime_config_for_gpu(options, gpu_id) - gpu_config = runtime_config.gpu_mode - print( - f"{datetime.now()} device {gpu_id} memory total={total_memory:.1f} GB, " - f"free={free_memory:.1f} GB, workers={gpu_config.workers_on_gpu}, " - f"budget={gpu_config.memory_budget:.1f} GB", - flush=True, - ) try: api_config = APIConfig(api_config_str) @@ -1662,10 +1653,9 @@ def main(): os.environ["USE_CACHED_NUMPY"] = str(options.use_cached_numpy) os.environ["USE_GPU_MODE"] = str(options.use_gpu_mode) if options.use_gpu_mode: - print( - "[gpu_mode] enabled: GPU tensor generation, GPU compare, and allocator reuse are active.", - flush=True, - ) + print("[gpu_mode] enabled: use GPU tensors and comparison.", flush=True) + elif options.use_cached_numpy: + print("[use_cached_numpy] enabled: reuse cached NumPy inputs.", flush=True) if options.bitwise_alignment: options.atol = 0.0 options.rtol = 0.0 diff --git a/tester/accuracy_stable.py b/tester/accuracy_stable.py index 0d6d6db4..7ca1db06 100644 --- a/tester/accuracy_stable.py +++ b/tester/accuracy_stable.py @@ -64,6 +64,54 @@ def _move_result_to_cpu(self, value): return {key: self._move_result_to_cpu(item) for key, item in value.items()} return value + def _result_num_bytes(self, value): + if isinstance(value, (torch.Tensor, paddle.Tensor)): + try: + return int(value.numel()) * int(value.element_size()) + except Exception: + return 0 + if isinstance(value, (list, tuple)): + return sum(self._result_num_bytes(item) for item in value) + if isinstance(value, dict): + return sum(self._result_num_bytes(item) for item in value.values()) + return 0 + + def _move_result_to_gpu(self, value): + if isinstance(value, torch.Tensor): + return value.to(device="cuda", non_blocking=True) + if isinstance(value, paddle.Tensor): + return value.cuda() + if isinstance(value, list): + return [self._move_result_to_gpu(item) for item in value] + if isinstance(value, tuple): + return tuple(self._move_result_to_gpu(item) for item in value) + if isinstance(value, dict): + return {key: self._move_result_to_gpu(item) for key, item in value.items()} + return value + + def _try_restore_spilled_results_to_gpu(self, results): + required_bytes = sum(self._result_num_bytes(value) for value in results) + if required_bytes <= 0: + return None + try: + free_bytes, _ = torch.cuda.mem_get_info() + available_bytes = free_bytes + memory_budget = float(getattr(self.gpu_mode_config, "memory_budget", 0.0) or 0.0) + if memory_budget > 0: + reserved_bytes = torch.cuda.memory_reserved() + budget_headroom = max( + 0, + int(memory_budget * (1024**3)) - reserved_bytes, + ) + available_bytes = min(available_bytes, budget_headroom) + # Leave 20% headroom for allocator and comparison temporaries. + if required_bytes * 5 > available_bytes * 4: + return None + restored = tuple(self._move_result_to_gpu(value) for value in results) + return restored + except Exception: + return None + def _clear_runtime_inputs(self, framework): for attr_name in (f"{framework}_args", f"{framework}_kwargs"): if hasattr(self, attr_name): @@ -210,8 +258,26 @@ def test(self): gpu_mode_maybe_empty_cache( self.gpu_mode_config, "accuracy_stable_after_first_compare_spill", + force=True, ) + if self.use_gpu_mode: + restored_results = self._try_restore_spilled_results_to_gpu( + ( + torch_output_pair[0], + paddle_output_pair[0], + torch_grad_pair[0], + paddle_grad_pair[0], + ) + ) + if restored_results is not None: + ( + torch_output_pair[0], + paddle_output_pair[0], + torch_grad_pair[0], + paddle_grad_pair[0], + ) = restored_results + # ======== summary ======== self.compare(torch_output_pair[1], paddle_output_pair[1], "T2P2") self.compare(torch_grad_pair[1], paddle_grad_pair[1], "T2P2B") diff --git a/tester/api_config/config_analyzer.py b/tester/api_config/config_analyzer.py index 9e3b1dd1..add927ef 100644 --- a/tester/api_config/config_analyzer.py +++ b/tester/api_config/config_analyzer.py @@ -389,7 +389,7 @@ def _make_gpu_tensor_pair(self, api_config, dtype=None): source_dtype = "float16" if dtype in FLOAT8_DTYPES else dtype torch_source = self._make_gpu_torch_dense_tensor(source_dtype) paddle_source = paddle.utils.dlpack.from_dlpack( - torch.utils.dlpack.to_dlpack(torch_source.detach().clone()) + torch.utils.dlpack.to_dlpack(torch_source.detach()) ) if not self.is_contiguous and self.strides is not None: try: diff --git a/tester/base.py b/tester/base.py index 337ad27f..253d993c 100644 --- a/tester/base.py +++ b/tester/base.py @@ -103,12 +103,6 @@ def gpu_mode_maybe_empty_cache(gpu_config, phase="", force=False, request_spill= ) if should_cleanup: - action = "spill+empty_cache" if should_spill else "empty_cache" - print( - f"[gpu_mode_memory] {phase or 'unknown'} {action} " - f"budget={memory_budget:.1f}G reserved={torch_reserved:.1f}G allocated={torch_allocated:.1f}G", - flush=True, - ) gc.collect() torch.cuda.empty_cache() paddle.device.cuda.empty_cache() @@ -1401,7 +1395,9 @@ def error_msg(msg): def _torch_assert_accuracy_in_chunks( self, actual, expected, atol, rtol, error_msg, working_bytes ): - temp_bytes_per_element = max(32, actual.element_size() + expected.element_size() + 16) + temp_bytes_per_element = max( + 32, 4 * max(actual.element_size(), expected.element_size()) + 16 + ) chunk_numel = max(1, working_bytes // temp_bytes_per_element) actual_flat = actual.reshape(-1) expected_flat = expected.reshape(-1) @@ -1411,12 +1407,6 @@ def _torch_assert_accuracy_in_chunks( max_rel_diff = -1.0 max_rel_index = 0 exact_compare = atol == 0.0 and rtol == 0.0 - - print( - f"[chunk_compare] numel={actual_flat.numel()} chunk_numel={chunk_numel} " - f"dtype={actual.dtype}", - flush=True, - ) for start in range(0, actual_flat.numel(), chunk_numel): end = min(actual_flat.numel(), start + chunk_numel) actual_chunk = actual_flat[start:end] @@ -1425,19 +1415,19 @@ def _torch_assert_accuracy_in_chunks( equal = actual_chunk == expected_chunk if actual_chunk.is_floating_point() or actual_chunk.is_complex(): equal |= torch.isnan(actual_chunk) & torch.isnan(expected_chunk) - mismatch = ~equal + mismatch = equal.logical_not_() else: if "float8" in str(actual_chunk.dtype): actual_chunk = actual_chunk.float() if "float8" in str(expected_chunk.dtype): expected_chunk = expected_chunk.float() - mismatch = ~torch.isclose( + mismatch = torch.isclose( actual_chunk, expected_chunk, rtol=rtol, atol=atol, equal_nan=True, - ) + ).logical_not_() chunk_mismatch_count = int(mismatch.sum().item()) mismatch_count += chunk_mismatch_count @@ -1450,24 +1440,48 @@ def _torch_assert_accuracy_in_chunks( elif actual_chunk.dtype == torch.float64 or expected_chunk.dtype == torch.float64: actual_for_diff = actual_chunk.to(torch.float64) expected_for_diff = expected_chunk.to(torch.float64) - else: + elif "float8" in str(actual_chunk.dtype) or "float8" in str(expected_chunk.dtype): actual_for_diff = actual_chunk.float() expected_for_diff = expected_chunk.float() + elif not actual_chunk.dtype.is_floating_point and not actual_chunk.dtype.is_complex: + actual_for_diff = actual_chunk.to(torch.int64) + expected_for_diff = expected_chunk.to(torch.int64) + else: + actual_for_diff = actual_chunk + expected_for_diff = expected_chunk abs_diff = torch.abs(actual_for_diff - expected_for_diff) - abs_diff = abs_diff.masked_fill(~mismatch, -1.0) + matched = ~mismatch + abs_diff.masked_fill_(matched, -1.0) chunk_max_abs, chunk_abs_index = torch.max(abs_diff, dim=0) - chunk_max_abs_value = float(chunk_max_abs.item()) - if chunk_max_abs_value > max_abs_diff: - max_abs_diff = chunk_max_abs_value - max_abs_index = start + int(chunk_abs_index.item()) - rel_diff = abs_diff / torch.abs(expected_for_diff) - rel_diff = rel_diff.masked_fill(~mismatch, -1.0) + rel_diff.masked_fill_(matched, -1.0) chunk_max_rel, chunk_rel_index = torch.max(rel_diff, dim=0) - chunk_max_rel_value = float(chunk_max_rel.item()) + + # Transfer all reduction results in one synchronization. Calling + # .item() for every value dominates scans with hundreds of chunks. + ( + chunk_max_abs_value, + chunk_abs_index_value, + chunk_max_rel_value, + chunk_rel_index_value, + ) = ( + torch.stack( + ( + chunk_max_abs.to(torch.float64), + chunk_abs_index.to(torch.float64), + chunk_max_rel.to(torch.float64), + chunk_rel_index.to(torch.float64), + ) + ) + .cpu() + .tolist() + ) + if chunk_max_abs_value > max_abs_diff: + max_abs_diff = chunk_max_abs_value + max_abs_index = start + int(chunk_abs_index_value) if chunk_max_rel_value > max_rel_diff: max_rel_diff = chunk_max_rel_value - max_rel_index = start + int(chunk_rel_index.item()) + max_rel_index = start + int(chunk_rel_index_value) if mismatch_count == 0: return @@ -1561,8 +1575,6 @@ def torch_assert_accuracy( f"{expected_name} {expected_tensor.dtype}" ) - exact_compare = atol == 0.0 and rtol == 0.0 - def error_msg(msg): return ( f"Not equal to tolerance rtol={rtol}, atol={atol}\n" @@ -1575,15 +1587,35 @@ def error_msg(msg): try: working_bytes = 1024**3 - if actual_tensor.is_cuda: + estimated_temp_bytes = actual_tensor.numel() * max( + 32, + 4 * max(actual_tensor.element_size(), expected_tensor.element_size()) + 16, + ) + # Small tensors always use torch.testing.assert_close directly. Avoid + # querying the CUDA driver for a working budget on that hot path. + needs_chunk_budget = estimated_temp_bytes > working_bytes + if actual_tensor.is_cuda and needs_chunk_budget: try: free_bytes, _ = torch.cuda.mem_get_info(actual_tensor.device) - working_bytes = min(working_bytes, max(256 * 1024**2, free_bytes // 50)) + max_working_bytes = 16 * 1024**3 + min_working_bytes = 256 * 1024**2 + working_bytes = min(max_working_bytes, max(min_working_bytes, free_bytes // 5)) + + memory_budget = float( + getattr(self.gpu_mode_config, "memory_budget", 0.0) or 0.0 + ) + if memory_budget > 0: + reserved_bytes = torch.cuda.memory_reserved(actual_tensor.device) + budget_headroom = max( + 0, + int(memory_budget * (1024**3)) - reserved_bytes, + ) + working_bytes = min( + working_bytes, + max(min_working_bytes, budget_headroom // 5), + ) except Exception: pass - estimated_temp_bytes = actual_tensor.numel() * max( - 8, actual_tensor.element_size() + expected_tensor.element_size() + 4 - ) if estimated_temp_bytes > working_bytes: self._torch_assert_accuracy_in_chunks( actual_tensor, @@ -1603,12 +1635,12 @@ def error_msg(msg): ) return - # PyTorch supports direct FP8 comparison only for exact tolerances. - if not exact_compare: - if "float8" in str(actual_tensor.dtype): - actual_tensor = actual_tensor.float() - if "float8" in str(expected_tensor.dtype): - expected_tensor = expected_tensor.float() + # Keep FP8 diagnostics consistent with the chunked path by comparing + # promoted float32 values even for exact comparisons. + if "float8" in str(actual_tensor.dtype): + actual_tensor = actual_tensor.float() + if "float8" in str(expected_tensor.dtype): + expected_tensor = expected_tensor.float() torch.testing.assert_close( actual_tensor, expected_tensor,