From eb7c654275288595bebe0ec196d2ab40fad5628f Mon Sep 17 00:00:00 2001 From: lichuyang Date: Tue, 13 Oct 2026 16:03:55 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat(peft):=20=E5=90=AF=E7=94=A8=20example?= =?UTF-8?q?=20=E7=9C=8B=E6=8A=A4=E7=9A=84=E6=AF=8F=E6=97=A5=E5=AE=9A?= =?UTF-8?q?=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #201 合入时 schedule 留作注释，peft 因此一直没有定时刷新。改成每天 UTC 23:00 跑一次，即北京时间 07:00。这个整点没有被其他项目的 example 看护占用。 定时运行只在三种情况下才跑矩阵：上游出了新 release，peft 看护自己的文件变了，或者上一次失败需要重试；其余时候在 monitor 结束。合入后的第一次定时运行监控缓存为空，会跑一次全量矩阵。 --- .github/workflows/peft-examples.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/peft-examples.yml b/.github/workflows/peft-examples.yml index 91ba87928..608d0e7ff 100644 --- a/.github/workflows/peft-examples.yml +++ b/.github/workflows/peft-examples.yml @@ -23,11 +23,12 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} on: - # Bring-up phase: keep the schedule disabled until a few manual - # rounds are green (the cosdt-ci-test/workflows schedule stays the - # production guard meanwhile). Re-enable: - # schedule: - # - cron: '30 */6 * * *' + # Daily at 07:00 Beijing time (UTC 23:00). A scheduled run only runs + # the matrix when the monitor sees a reason: a new upstream release, + # a changed file of this project's guard, or a failed previous run to + # retry. Otherwise it ends after the monitor job. + schedule: + - cron: '0 23 * * *' workflow_dispatch: inputs: target_ref: From 48b513c871621cc255fa4172e27a76a37edef97c Mon Sep 17 00:00:00 2001 From: lichuyang Date: Tue, 13 Oct 2026 16:04:05 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix(peft):=20=E5=8E=BB=E6=8E=89=E8=84=9A?= =?UTF-8?q?=E6=9C=AC=E9=87=8C=E5=A4=9A=E4=BD=99=E7=9A=84=20HF=5FENDPOINT?= =?UTF-8?q?=20=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 引擎在 eff1749 之后不再设置 HF_ENDPOINT，默认就直连 huggingface.co。setup_example.sh 和 run_example.sh 里的 export 与注释还在说引擎默认走 hf-mirror，已经和事实不符。删掉后行为不变，下载端点交给引擎默认值。 --- examples-guards/peft/scripts/run_example.sh | 4 ---- examples-guards/peft/scripts/setup_example.sh | 4 ---- 2 files changed, 8 deletions(-) diff --git a/examples-guards/peft/scripts/run_example.sh b/examples-guards/peft/scripts/run_example.sh index fbe5a8587..244dfd8cf 100755 --- a/examples-guards/peft/scripts/run_example.sh +++ b/examples-guards/peft/scripts/run_example.sh @@ -44,10 +44,6 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh # （73s/step → 3s/step）；其余条目不依赖 dynamo 编译路径，全局关闭 # 无副作用。 export TORCHDYNAMO_DISABLE=1 -# Hub ids the examples hardcode download on a cache miss; same endpoint -# as setup_example.sh (the engine's hf-mirror default cannot serve -# namespace-less ids such as t5-base). -export HF_ENDPOINT=https://huggingface.co python -c "import torch, torch_npu; print('NPU available:', torch.npu.is_available(), 'devices:', torch.npu.device_count())" expand_overlay() { diff --git a/examples-guards/peft/scripts/setup_example.sh b/examples-guards/peft/scripts/setup_example.sh index ae5222821..187494872 100755 --- a/examples-guards/peft/scripts/setup_example.sh +++ b/examples-guards/peft/scripts/setup_example.sh @@ -312,10 +312,6 @@ GITHUB_ENV="${GITHUB_ENV:?GITHUB_ENV is required}" HERE=$(cd "$(dirname "$0")" && pwd) export PIP_CONSTRAINT="$(cd "$HERE/.." && pwd)/constraints-npu.txt" -# This runner pool reaches huggingface.co directly. The engine's -# hf-mirror default 308-redirects namespace-less ids such as roberta-base -# to huggingface.co, a redirect huggingface_hub does not follow. -export HF_ENDPOINT=https://huggingface.co source /usr/local/Ascend/ascend-toolkit/set_env.sh From 83a33cbc2371193a77ac79700ad5f12d02d94de7 Mon Sep 17 00:00:00 2001 From: lichuyang Date: Tue, 13 Oct 2026 16:04:05 +0800 Subject: [PATCH 3/3] =?UTF-8?q?refactor(peft):=20=E6=8A=8A=20run=5Fexample?= =?UTF-8?q?.sh=20=E5=86=85=E5=B5=8C=E7=9A=84=20Python=20=E6=8F=90=E5=8F=96?= =?UTF-8?q?=E6=88=90=E7=8B=AC=E7=AB=8B=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按 example 看护迁移约定，项目脚本里不写 heredoc。run_example.sh 的三处内嵌 Python 提取到 scripts/ 下，逻辑逐字搬出，不改行为：expand_overlay.py 展开 overlay 参数，ensure_passthrough.py 给 shell 示例补 "$@"，ci_sitecustomize.py 是启动时注入的 CI 补丁。 原来 shim 开头的 transfer_to_npu 导入是由 shell 按环境变量拼进去的，现在改成 shim 自己读 SKIP_TRANSFER_TO_NPU 判断，语义相同。 --- .../peft/scripts/ci_sitecustomize.py | 96 +++++++++++ .../peft/scripts/ensure_passthrough.py | 36 +++++ .../peft/scripts/expand_overlay.py | 37 +++++ examples-guards/peft/scripts/run_example.sh | 150 ++---------------- 4 files changed, 179 insertions(+), 140 deletions(-) create mode 100644 examples-guards/peft/scripts/ci_sitecustomize.py create mode 100644 examples-guards/peft/scripts/ensure_passthrough.py create mode 100644 examples-guards/peft/scripts/expand_overlay.py diff --git a/examples-guards/peft/scripts/ci_sitecustomize.py b/examples-guards/peft/scripts/ci_sitecustomize.py new file mode 100644 index 000000000..d9ceec470 --- /dev/null +++ b/examples-guards/peft/scripts/ci_sitecustomize.py @@ -0,0 +1,96 @@ +"""CI shim for the peft examples; run_example.sh installs it as sitecustomize.py. + +One sitecustomize.py, four patches, applied at interpreter startup: +1. CUDA->NPU (transfer_to_npu): most peft examples hardcode + device="cuda"; torch_npu's transfer_to_npu maps torch.cuda onto npu + (also covers `torch.device("cuda" if torch.cuda.is_available() else + "cpu")` device-branch examples such as lora_ga). NOT applied to the + bitsandbytes examples: run_example.sh sets SKIP_TRANSFER_TO_NPU for + them. +2. Dataset: the sft example calls load_dataset(path) with a local + .jsonl fixture and reads BOTH "train" and "test" splits + (splits="train,test"); datasets 3.x cannot infer a builder from a + single file path - rewrite local data files to + load_dataset("", data_files={"train": path, "test": path}). +3. fp16 neutralizer: several finetuning scripts hardcode + TrainingArguments(fp16=True, ...). accelerate 1.15's fp16 chain + (optimizer.step(grad_scaler=scaler)) is incompatible with torch_npu + amp.GradScaler -> TypeError "Adam.step got unexpected keyword + argument grad_scaler". No supported entry relies on fp16=True, so + forcing fp16=False globally is a no-op for every other entry. +4. wandb neutralizer: boft_dreambooth hardcodes wandb_init + mode="online"; force wandb.init mode="disabled" so it becomes + wandb's own no-op run without an API key. +""" +import os + +# Skipped for the bitsandbytes examples: bnb 0.50.2's 4-bit/8-bit work +# via the default CPU backend, but transfer_to_npu makes +# torch.cuda.is_available()=True which routes bnb to its CUDA backend +# and crashes on torch._C._cuda_getCurrentRawStream (torch is +cpu-built). +if not os.environ.get("SKIP_TRANSFER_TO_NPU"): + from torch_npu.contrib import transfer_to_npu # noqa: F401 + +import datasets as _datasets + +_original_load_dataset = _datasets.load_dataset + +_BUILDERS = { + ".json": "json", + ".jsonl": "json", + ".json.gz": "json", + ".csv": "csv", + ".tsv": "csv", + ".parquet": "parquet", + ".txt": "text", +} + + +def _patched_load_dataset(path, *args, **kwargs): + if isinstance(path, str) and os.path.isfile(path): + ext = os.path.splitext(path)[1].lower() + if ext in _BUILDERS and "data_files" not in kwargs: + name = kwargs.pop("name", None) + if args and name is None: + name = args[0] + args = args[1:] + if name is not None: + kwargs["name"] = name + # Map the single file onto both splits: the sft example + # loads split="train" and split="test" from the same + # dataset_name. + kwargs["data_files"] = {"train": path, "test": path} + return _original_load_dataset(_BUILDERS[ext], *args, **kwargs) + return _original_load_dataset(path, *args, **kwargs) + + +_datasets.load_dataset = _patched_load_dataset + +import transformers as _tf + +_orig_training_args_init = _tf.TrainingArguments.__init__ + + +def _fp16_neutral_init(self, *args, **kwargs): + if kwargs.get("fp16"): + kwargs["fp16"] = False + _orig_training_args_init(self, *args, **kwargs) + + +_tf.TrainingArguments.__init__ = _fp16_neutral_init + +# wandb neutralizer: boft_dreambooth hardcodes wandb_init +# mode="online" and --report_to wandb; force mode="disabled" so +# wandb.init becomes wandb's own no-op run without an API key. +try: + import wandb as _wandb + + _real_wandb_init = _wandb.init + + def _init_disabled(*args, **kwargs): + kwargs["mode"] = "disabled" + return _real_wandb_init(*args, **kwargs) + + _wandb.init = _init_disabled +except Exception: + pass diff --git a/examples-guards/peft/scripts/ensure_passthrough.py b/examples-guards/peft/scripts/ensure_passthrough.py new file mode 100644 index 000000000..19f1000df --- /dev/null +++ b/examples-guards/peft/scripts/ensure_passthrough.py @@ -0,0 +1,36 @@ +"""Attach "$@" to the last command line of a shell example. + +Usage: ensure_passthrough.py + +Shell examples that do not forward "$@" cannot take the overlay args. +This appends ' "$@"' to the last non-comment, non-blank line (the tail of +the example's main command) in the CI working copy only. Never commit it +back to the target repo. +""" +from pathlib import Path +import sys + + +def main(): + path = Path(sys.argv[1]) + lines = path.read_text(encoding='utf-8').splitlines(keepends=True) + for i in range(len(lines) - 1, -1, -1): + stripped = lines[i].strip() + if stripped and not stripped.startswith('#'): + raw = lines[i] + newline = '' + if raw.endswith('\r\n'): + newline = '\r\n' + raw = raw[:-2] + elif raw.endswith('\n'): + newline = '\n' + raw = raw[:-1] + lines[i] = raw.rstrip() + ' "$@"' + newline + path.write_text(''.join(lines), encoding='utf-8') + print(f'patched {path} to pass "$@" on last command line') + raise SystemExit(0) + raise SystemExit(f'{path}: cannot find a command line to attach "$@"') + + +if __name__ == '__main__': + main() diff --git a/examples-guards/peft/scripts/expand_overlay.py b/examples-guards/peft/scripts/expand_overlay.py new file mode 100644 index 000000000..ed2084368 --- /dev/null +++ b/examples-guards/peft/scripts/expand_overlay.py @@ -0,0 +1,37 @@ +"""Expand the manifest's overlay_args into one shell-quoted line. + +The workflow serializes manifest.overlay_args as JSON into OVERLAY_ARGS. +Each item is shell-split with its quoting intact, then CI paths such as +${CI_OUTPUT_DIR} / ${SFT_MODEL_PATH} resolve only in this job's +environment. The tokens are printed space-joined, each shell-quoted, so +the caller can eval them into a bash array. Nothing is printed when +there are no overlay args. +""" +import json +import os +import shlex + + +def main(): + raw = os.environ.get('OVERLAY_ARGS', '').strip() + if not raw or raw in ('null', '""'): + raise SystemExit(0) + try: + items = json.loads(raw) + except json.JSONDecodeError as exc: + raise SystemExit(f'OVERLAY_ARGS is not valid JSON: {exc}') from exc + if items in (None, ''): + raise SystemExit(0) + if not isinstance(items, list): + raise SystemExit( + f'OVERLAY_ARGS must be a JSON array, got {type(items).__name__}') + tokens = [] + for item in items: + if not isinstance(item, str) or not item.strip(): + raise SystemExit('OVERLAY_ARGS items must be non-empty strings') + tokens.extend(shlex.split(os.path.expandvars(item), posix=True)) + print(' '.join(shlex.quote(token) for token in tokens)) + + +if __name__ == '__main__': + main() diff --git a/examples-guards/peft/scripts/run_example.sh b/examples-guards/peft/scripts/run_example.sh index 244dfd8cf..941cb65a0 100755 --- a/examples-guards/peft/scripts/run_example.sh +++ b/examples-guards/peft/scripts/run_example.sh @@ -16,6 +16,8 @@ EXAMPLE_REL="$1" TARGET_ROOT="${TARGET_ROOT:?TARGET_ROOT is required}" CI_OUTPUT_DIR="${CI_OUTPUT_DIR:?CI_OUTPUT_DIR is required}" +HERE=$(cd "$(dirname "$0")" && pwd) + EXAMPLE_PATH="$TARGET_ROOT/$EXAMPLE_REL" [[ -e "$EXAMPLE_PATH" ]] || { echo "example not found: $EXAMPLE_PATH" >&2; exit 1; } @@ -51,30 +53,7 @@ expand_overlay() { # item with shell quoting intact, then allow CI paths such as # ${CI_OUTPUT_DIR} / ${SFT_MODEL_PATH} to resolve only in this job's # environment. - "$PYTHON" - <<'PY' -import json -import os -import shlex - -raw = os.environ.get('OVERLAY_ARGS', '').strip() -if not raw or raw in ('null', '""'): - raise SystemExit(0) -try: - items = json.loads(raw) -except json.JSONDecodeError as exc: - raise SystemExit(f'OVERLAY_ARGS is not valid JSON: {exc}') from exc -if items in (None, ''): - raise SystemExit(0) -if not isinstance(items, list): - raise SystemExit( - f'OVERLAY_ARGS must be a JSON array, got {type(items).__name__}') -tokens = [] -for item in items: - if not isinstance(item, str) or not item.strip(): - raise SystemExit('OVERLAY_ARGS items must be non-empty strings') - tokens.extend(shlex.split(os.path.expandvars(item), posix=True)) -print(' '.join(shlex.quote(token) for token in tokens)) -PY + "$PYTHON" "$HERE/expand_overlay.py" } eval "EXTRA_ARGS=( $(expand_overlay) )" @@ -95,130 +74,21 @@ ensure_passthrough() { echo "example already has \"\$@\"; skipping patch" return fi - "$PYTHON" - "$script" <<'PY' -from pathlib import Path -import sys - -path = Path(sys.argv[1]) -lines = path.read_text(encoding='utf-8').splitlines(keepends=True) -for i in range(len(lines) - 1, -1, -1): - stripped = lines[i].strip() - if stripped and not stripped.startswith('#'): - raw = lines[i] - newline = '' - if raw.endswith('\r\n'): - newline = '\r\n' - raw = raw[:-2] - elif raw.endswith('\n'): - newline = '\n' - raw = raw[:-1] - lines[i] = raw.rstrip() + ' "$@"' + newline - path.write_text(''.join(lines), encoding='utf-8') - print(f'patched {path} to pass "$@" on last command line') - raise SystemExit(0) -raise SystemExit(f'{path}: cannot find a command line to attach "$@"') -PY + "$PYTHON" "$HERE/ensure_passthrough.py" "$script" } ensure_passthrough "$LAUNCH_PATH" -# One sitecustomize.py, four patches, injected at interpreter startup: -# 1. CUDA->NPU (transfer_to_npu): most peft examples hardcode -# device="cuda"; torch_npu's transfer_to_npu maps torch.cuda onto npu -# (also covers `torch.device("cuda" if torch.cuda.is_available() else -# "cpu")` device-branch examples such as lora_ga). NOT applied to the -# bitsandbytes examples (see SKIP_TRANSFER_TO_NPU below). -# 2. Dataset: the sft example calls load_dataset(path) with a local -# .jsonl fixture and reads BOTH "train" and "test" splits -# (splits="train,test"); datasets 3.x cannot infer a builder from a -# single file path - rewrite local data files to -# load_dataset("", data_files={"train": path, "test": path}). -# 3. fp16 neutralizer: several finetuning scripts hardcode -# TrainingArguments(fp16=True, ...). accelerate 1.15's fp16 chain -# (optimizer.step(grad_scaler=scaler)) is incompatible with torch_npu -# amp.GradScaler -> TypeError "Adam.step got unexpected keyword -# argument grad_scaler". No supported entry relies on fp16=True, so -# forcing fp16=False globally is a no-op for every other entry. -# 4. wandb neutralizer: boft_dreambooth hardcodes wandb_init -# mode="online"; force wandb.init mode="disabled" so it becomes -# wandb's own no-op run without an API key. +# Install ci_sitecustomize.py as sitecustomize.py so every Python +# process the example starts picks up the four CI patches (CUDA->NPU, +# local-file load_dataset, fp16 neutralizer, wandb neutralizer; see the +# docstring of ci_sitecustomize.py). SKIP_TRANSFER_TO_NPU, set below for +# the bitsandbytes examples, turns off the first one. prepare_shims() { local shim_dir="$GITHUB_WORKSPACE/ci_patch" mkdir -p "$shim_dir" - local transfer_line="from torch_npu.contrib import transfer_to_npu # noqa: F401" - if [[ -n "${SKIP_TRANSFER_TO_NPU:-}" ]]; then - transfer_line="# transfer_to_npu skipped for bitsandbytes example" - fi - cat > "$shim_dir/sitecustomize.py" <