diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..877f319f7 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,6 @@ +self-hosted-runner: + labels: + - linux-aarch64-a2-1 + - linux-aarch64-a2-2 + - linux-aarch64-a2-4 + - linux-aarch64-a2-8 diff --git a/.github/workflows/examples-template.yml b/.github/workflows/examples-template.yml index ae97c4343..a5f86ff03 100644 --- a/.github/workflows/examples-template.yml +++ b/.github/workflows/examples-template.yml @@ -1,5 +1,11 @@ # Examples guard engine - the common, reusable part of the examples # guarding pipeline. +# +# Events: schedule runs the monitor loop (release + workflow-files +# signals, monitor-state cache); workflow_dispatch and pull_request +# are active-trigger runs that always test the full supported matrix +# and never read or write the monitor state, so a guard PR shows its +# matrix to the reviewer before merge. name: examples-template @@ -50,6 +56,10 @@ jobs: workflow_files_hash: ${{ steps.monitor.outputs.workflow_files_hash }} env: UPSTREAM_REPO: ${{ inputs.upstream_repo }} + # Unauthenticated api.github.com calls share the hosted runner's + # per-IP quota of 60/h and intermittently get 403; the job token + # (read-only on fork PRs) raises it to the per-repo quota. + GH_TOKEN: ${{ github.token }} steps: - name: Restore monitor state id: cache-restore @@ -100,6 +110,7 @@ jobs: # and reason=release (workflow-files only fires when release does not). RESP_FILE=$(mktemp) HTTP_CODE=$(curl -sSL --retry 3 -o "$RESP_FILE" -w '%{http_code}' \ + -H "Authorization: Bearer $GH_TOKEN" \ "https://api.github.com/repos/$UPSTREAM_REPO/releases/latest" || true) TAG="" if [ "$HTTP_CODE" = "200" ]; then @@ -117,6 +128,7 @@ jobs: # release_outcome retry bookkeeping is shared. HEAD_RESP=$(mktemp) HEAD_CODE=$(curl -sSL --retry 3 -o "$HEAD_RESP" -w '%{http_code}' \ + -H "Authorization: Bearer $GH_TOKEN" \ "https://api.github.com/repos/$UPSTREAM_REPO/commits/$DEFAULT_BRANCH" || true) HEAD_SHA="" if [ "$HEAD_CODE" = "200" ]; then @@ -165,6 +177,7 @@ jobs: # GitHub blob/tree SHAs from /git/trees/main?recursive=1; sorted # path+SHA pairs then sha256sum. Fires only when release did not. TREE_RESP=$(curl -fsSL --retry 3 \ + -H "Authorization: Bearer $GH_TOKEN" \ "https://api.github.com/repos/$WORKFLOW_REPO/git/trees/main?recursive=1" \ 2>/dev/null || echo "") TREE_JSON="" @@ -202,8 +215,8 @@ jobs: echo "workflow_files_hash=${WORKFLOW_FILES_HASH}" } >> "$GITHUB_OUTPUT" - # Schedule picks the ref from the release signal; manual runs - # resolve an empty target_ref to the latest upstream release + # Schedule picks the ref from the release signal; manual and PR + # runs resolve an empty target_ref to the latest upstream release # (fallback chain: explicit input -> latest release -> default_branch). # Without this, manual dispatch always tests upstream `main`, # which can pick up in-progress pyproject changes that the @@ -220,7 +233,11 @@ jobs: UPSTREAM_REPO: ${{ inputs.upstream_repo }} run: | set -euo pipefail - if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + # The monitor steps above are schedule-gated: a PR run must + # resolve its target here, or every downstream job skips and + # the PR goes green untested. PR runs have no inputs, so + # they always take the release lookup below. + if [ "$EVENT_NAME" = "workflow_dispatch" ] || [ "$EVENT_NAME" = "pull_request" ]; then REF="" REASON="" if [ -n "$INPUT_REF" ]; then @@ -233,6 +250,7 @@ jobs: # main if the API is unreachable or no release exists. RESP_FILE=$(mktemp) HTTP_CODE=$(curl -sSL --retry 3 -o "$RESP_FILE" -w '%{http_code}' \ + -H "Authorization: Bearer $GH_TOKEN" \ "https://api.github.com/repos/$UPSTREAM_REPO/releases/latest" || true) TAG="" if [ "$HTTP_CODE" = "200" ]; then @@ -251,11 +269,11 @@ jobs: REPO="${INPUT_REPO:-$UPSTREAM_REPO}" { echo "need_to_run=true" - echo "trigger=workflow_dispatch" + echo "trigger=$EVENT_NAME" echo "target_repo=$REPO" echo "target_ref=$REF" } >> "$GITHUB_OUTPUT" - echo "resolved repo=$REPO ref=$REF reason=$REASON (manual)" + echo "resolved repo=$REPO ref=$REF reason=$REASON ($EVENT_NAME)" exit 0 fi NEED="${{ steps.monitor.outputs.need_to_run }}" diff --git a/.github/workflows/peft-examples.yml b/.github/workflows/peft-examples.yml new file mode 100644 index 000000000..91ba87928 --- /dev/null +++ b/.github/workflows/peft-examples.yml @@ -0,0 +1,71 @@ +# peft examples guard - project thin trigger. +# +# Calls the common engine .github/workflows/examples-template.yml; +# this file only declares what varies per project: schedule, +# concurrency, upstream repo, dispatch input forwarding, and matrix +# parallelism. The guard loop (release + workflow-files monitor / +# manifest-check / run-example matrix / validate-results / +# save-monitor-state) lives in the engine. See examples-guards/README.md +# for the design. + +name: peft-examples + +concurrency: + # Three groups: schedule runs serialize (a run cancelled mid-way + # loses its outcome writeback, and the outcome drives the next + # cycle's retry); each PR serializes per number so a new push + # cancels the stale run instead of queueing another matrix on the + # NPU runners; each manual run stands alone. Group names are + # repository-wide, so the PR group carries the workflow name: a bare + # pr- would let two projects' runs on one PR cancel each + # other. + group: ${{ github.event_name == 'schedule' && 'peft-examples-schedule' || (github.event_name == 'pull_request' && format('peft-examples-pr-{0}', github.event.pull_request.number)) || format('manual-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +on: + # Bring-up phase: keep the schedule disabled until a few manual + # rounds are green (the cosdt-ci-test/workflows schedule stays the + # production guard meanwhile). Re-enable: + # schedule: + # - cron: '30 */6 * * *' + workflow_dispatch: + inputs: + target_ref: + description: Branch, tag, or SHA to test (empty = latest upstream release; falls back to main on API failure) + required: false + default: '' + type: string + # PR runs need no merge: GitHub executes the workflow files from the + # PR merge ref, so a newly added trigger and engine run on the PR + # that adds them. Whether a fork PR waits for a maintainer's approval + # first is the repository's fork-PR policy. `pull_request`, not + # `pull_request_target`: the declared read permissions are all the + # engine needs. paths lists only this project's files, as the + # quick-start triggers do: a PR that touches only the shared engine, + # scripts or schema starts no project's matrix. + pull_request: + branches: [main] + types: [opened, synchronize, reopened] + paths: + - 'examples-guards/peft/**' + - '.github/workflows/peft-examples.yml' + +# Engine jobs query the GitHub Job API (validate-results); called +# workflows run with the caller's GITHUB_TOKEN permissions. +permissions: + actions: read + contents: read + +jobs: + peft-examples: + uses: ./.github/workflows/examples-template.yml + with: + # Namespaces cache keys (examples-monitor-state-peft_*), artifact + # names and the scheduling paths under examples-guards/peft. + project: peft + upstream_repo: huggingface/peft + target_ref: ${{ inputs.target_ref }} + # 48 supported entries, 43 of them single-card: max 8 keeps the + # long single-card legs flowing while the 2- and 8-card legs + # queue. + max_parallel: 8 diff --git a/.github/workflows/schemas/result.schema.json b/.github/workflows/schemas/result.schema.json index 81bac5f09..d794d1b1b 100644 --- a/.github/workflows/schemas/result.schema.json +++ b/.github/workflows/schemas/result.schema.json @@ -15,7 +15,7 @@ "properties": { "trigger": { "type": "string", - "description": "How this run was started. Typical values: workflow_dispatch, schedule." + "description": "How this run was started. Typical values: workflow_dispatch, schedule, pull_request." }, "target_repo": { "type": "string", diff --git a/conf.py b/conf.py index 018f185d0..472737444 100644 --- a/conf.py +++ b/conf.py @@ -70,7 +70,7 @@ # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '.venv', 'README.md', - '.github', 'tests', + '.github', 'tests', 'examples-guards', # Guard / CI source of truth; Sphinx includes each into # the project index.rst so the sidebar landing page # is the tutorial, not a nested「快速开始」child page. diff --git a/examples-guards/README.md b/examples-guards/README.md new file mode 100644 index 000000000..c42fd13ed --- /dev/null +++ b/examples-guards/README.md @@ -0,0 +1,57 @@ +# examples-guards 目录：example 看护的项目数据 + +本目录存放 example 看护线的项目配套数据，每个子目录对应一个上游项目。看护流水线在 `.github/workflows/-examples.yml`，公共回路在 `.github/workflows/examples-template.yml`。Quick Start 看护是另一条线，测试在 `tests//`，文档在 `sources//`，与本目录无关。 + +## 分工 + +- 引擎 `examples-template.yml`：监控、调度、结果发布、状态回写的全回路，不含任何项目名。 +- 薄触发器 `-examples.yml`：只声明项目变量，包括 cron、并发组、PR 触发的 `paths`、`project`、`upstream_repo`、dispatch 输入转发和矩阵并行度。 +- 项目数据 `examples-guards//`：`examples_manifest.yaml` 声明 supported 条目即调度参数，`scripts/setup_example.sh` 与 `scripts/run_example.sh` 实现环境准备与运行，`fixtures/` 提供数据，`constraints-npu.txt` 钉住依赖。 +- 共享脚本 `scripts/check_supported_entries.py`、`scripts/write_example_result.py` 与结果 schema `.github/workflows/schemas/result.schema.json` 由全部项目共用，修改前先确认对每个接入项目都成立。 + +加删一条 supported example 只改清单；example 环境变化只改项目脚本；看护回路演进只改引擎。 + +## 看护回路 + +schedule 定时轮询两个信号：上游最新 release，以及本项目看护文件的变化。信号触发时，把清单 supported 条目在最新 release tag 上逐条跑矩阵；每条结果写入 result.json 并作为 artifact 上传；矩阵结论回写监控缓存，失败驱动下一轮重试。上游从未发布 release 时，改测默认分支的最新提交。查询 GitHub API 失败时本轮不跑，监控状态不动，下一轮再查。 + +## 模型与数据集缓存 + +每个 GitHub 组织的 self-hosted runner 池各挂一块持久缓存盘，同池 runner 共用，跨次运行保留；别的组织投递过的资产在本仓看不见。本仓 runner 池能直连 huggingface.co，项目的 setup 与 run 脚本把 `HF_ENDPOINT` 设为 huggingface.co，不用引擎默认的 hf-mirror：后者把 `t5-base` 这类无命名空间的 id 308 跳转到 huggingface.co，huggingface_hub 不跟随这种跳转。peft 是按这套写法接入的范例。 + +- 例程硬编码的模型与数据集由 `from_pretrained`、`load_dataset` 自行下载进缓存，下次运行直接命中。 +- 以本地路径传给 overlay_args 的模型由项目的 `scripts/hub_cache.py` 解析：缓存里的快照必需文件齐全、且每个 safetensors 文件头完好，就直接用，不联网；否则先从 HuggingFace 下载，失败再从 ModelScope 下载。 +- 缓存盘跨 pod 共享，huggingface_hub 自带的 `.locks` 没拦住并发：run 37764199490 里两条 dreambooth 腿同时下 SD v1.5，追加写到同一个文件上，落得比完整文件还大。所以 `hub_cache.py` 不直接下载进缓存，先下到本 job 在同一块盘上的私有暂存目录，校验通过后再逐个文件改名进快照，完好的旧文件保持不动。同一模型另有一个 `mkdir` 锁目录让其余 job 等待，锁只省带宽，不承担正确性。 +- 损坏的分片连同它指向的 blob 必须先删掉再重下：`snapshot_download` 会把缓存里已有的 blob 直接复制给下载目标且不重新校验，不删就会把坏字节一路带下去。 +- 没有单独的预投递流水线，也不把模型或数据集提交进仓库。 + +## 结果契约 + +result.json 必填六个字段：trigger、target_repo、target_ref、path、image、job_status。job_status 只取 success、failure、cancelled。artifact 名为 `-examples--`，内容只有 result.json。上传永远在 GitHub 托管 runner 上完成，NPU job 只跑 example 并报告自身状态。 + +## PR 运行 + +每个项目的 example 看护 PR 不需要合入即可运行看护。原理：pull_request 事件执行的是 PR merge ref 上的 workflow 文件，PR 里新增的薄触发器和引擎当时即可运行，reviewer 在 checks 页直接看到矩阵结论。 + +PR 运行语义： + +- 只看本项目自己的文件：PR 改动 `examples-guards//**` 或本项目触发器时才运行。只改共用引擎、共享脚本或 schema 的 PR 不启动任何项目的矩阵，与 Quick Start 触发器一致；这类改动合入后由维护者手动运行受影响的项目。 +- 必然执行全量 supported 矩阵。被测 ref 与不填 target_ref 的手动运行相同：取上游最新 release tag，查不到时回落到默认分支。 +- 不读不写监控缓存，不影响定时看护的基线状态。 +- 同一 PR 新的提交自动取消未完成的旧运行，避免占用 NPU。并发组名必须带项目前缀，即 `-examples-pr-`：组名在整个仓库内共享，不带前缀时，多个项目在同一 PR 上的运行会互相取消。 +- result.json 的 trigger 字段记为 pull_request，artifact 命名与定时运行一致。汇总定时看护结论的外部系统按 trigger 字段过滤掉 PR 运行。 +- 来自 fork 的 PR 是否需要维护者先批准才能运行，取决于仓库或组织的 fork PR 审批策略，GitHub 默认要求首次贡献者批准一次；同仓分支的 PR 不受此限制。 + +## 迁移一个新项目 + +从 cosdt-ci-test/workflows 迁移共用模板套项目： + +1. 把源仓 `projects//` 下的 example 线文件拷到本仓 `examples-guards//`：清单、脚本、fixtures、constraints。Quick Start 线的 docs 与 tests 不拷贝，它们已在本仓单独迁移。源仓 setup 里依赖 cache-seed 预投递的资产，改成上文「模型与数据集缓存」一节的写法。 +2. 新建薄触发器 `.github/workflows/-examples.yml`，参照 `peft-examples.yml`：schedule 保持注释；声明 pull_request 触发，`paths` 只列 `examples-guards//**` 和触发器自身；PR 并发组名用 `-examples-pr-`；填入 `project` 与 `upstream_repo`。 +3. 用 actionlint 检查 workflow。 +4. 提 PR 后确认矩阵在 PR 上真实运行且结论符合预期。诚实红条目原样保留，跑红的不能改成 unsupported 换绿灯。 +5. PR 合入后手动 dispatch 跑绿几轮。是否打开 cron 由维护者另行决定；打开时槽位错开已有排布，并同时关闭旧仓同项目的 schedule，避免双看护占用 NPU。 + +## 诚实红约定 + +清单允许诚实红：在昇腾上真实失败的条目保持 supported 并如实标红。看护噪音例如镜像拉取失败、下载限流、标签排队单独归类，不算被测失败，也不因此放宽清单。 diff --git a/examples-guards/peft/constraints-npu.txt b/examples-guards/peft/constraints-npu.txt new file mode 100644 index 000000000..19be1da62 --- /dev/null +++ b/examples-guards/peft/constraints-npu.txt @@ -0,0 +1,44 @@ +# CUDA exclusion constraints for peft example CI on Ascend NPU. +# The runners are aarch64 without CUDA; pip resolvers on the default +# index would happily pull nvidia-* / cuda-* metapackages (and +# flashinfer) as transitive deps, which either fail to install or are +# dead weight. Same list as the quick-start test class +# projects/peft/tests/test_quick_start_ascend.py. +cuda-toolkit<0 +cuda-python<0 +cuda-bindings<0 +cuda-core<0 +cuda-pathfinder<0 +flashinfer-python<0 +nvidia-cublas<0 +nvidia-cuda-runtime<0 +nvidia-cuda-nvrtc<0 +nvidia-cuda-cupti<0 +nvidia-cudnn<0 +nvidia-cudnn-frontend<0 +nvidia-cufft<0 +nvidia-curand<0 +nvidia-cusolver<0 +nvidia-cusparse<0 +nvidia-cutlass-dsl<0 +nvidia-cutlass-dsl-libs-base<0 +nvidia-cutlass-dsl-libs-core<0 +nvidia-cutlass-dsl-libs-cu12<0 +nvidia-ml-py<0 +nvidia-nccl<0 +nvidia-nvjitlink<0 +nvidia-nvtx<0 +nvidia-cublas-cu12<0 +nvidia-cuda-nvdisasm<0 +nvidia-cuda-runtime-cu12<0 +nvidia-cuda-nvrtc-cu12<0 +nvidia-cuda-cupti-cu12<0 +nvidia-cudnn-cu12<0 +nvidia-cufft-cu12<0 +nvidia-curand-cu12<0 +nvidia-cusolver-cu12<0 +nvidia-cusparse-cu12<0 +nvidia-cusparselt-cu12<0 +nvidia-nccl-cu12<0 +nvidia-nvjitlink-cu12<0 +nvidia-nvtx-cu12<0 diff --git a/examples-guards/peft/examples_manifest.yaml b/examples-guards/peft/examples_manifest.yaml new file mode 100644 index 000000000..662c0707e --- /dev/null +++ b/examples-guards/peft/examples_manifest.yaml @@ -0,0 +1,1256 @@ +version: 1 +scan: + root: examples + include_extensions: ['.sh', '.py'] + # 仅 .sh/.py；peft/examples 下的所有 .yaml 都是 accelerate/deepspeed/fsdp 运行配置，非 training recipe，不纳入审计 + # files-only 模型：扫描范围是 examples/ 下的 .sh/.py + +supported: + # ===== 排序约定：按 run 35575049729 实际耗时降序 ===== + # 同耗时下保留原顺序（stable sort）。adalora_seq2seq 65 min 领跑、 + # lora_ga/mica/adamss_image/psoft/kasa/shadow/dora + # 几条长腿填前 8 slot（max_parallel=8），让首批并发跑长腿、短腿填后 + # 续空闲 slot，整轮 wall-clock 被最长腿（adalora）单一决定；调换 + # 之前 lora_ga/mica/adamss_image 在第 2-5 批串行排队 ≈ 5–25 min。 + # 再次实测后若耗时分布再变，照实测重排（不要按 timeout 倒序： + # run_peft.sh timeout 90 实际只跑 4.9 min；mica timeout 30 实际 29.3 min）。 + # AdaLoRA seq2seq：零 CLI 入口（模块常量 facebook/bart-base + 8 epochs + + # bs 8），npu-5 原样实跑 exit 0（8 epochs 训完 + accuracy 打印 + adapter + # 落盘 + 末段 generate 复现）。全量 ≈105-110 min（1074 步/epoch @~2.6 it/s + # + 每 epoch eval），是全部条目中最长的腿，timeout 120。模型 facebook/bart-base + # 与数据集 zeroshot/twitter-financial-news-sentiment 在运行时从 + # huggingface.co 下载进共享缓存。 + # pin_memory=True 纯 DataLoader（无 accelerator.prepare）实测不挂。 + - path: examples/conditional_generation/peft_adalora_seq2seq.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + timeout_minutes: 120 + + # MonteCLoRA 序列分类：原判 evaluate.load("glue","mrpc") import 时挂 + # （:33）——那是 npu-3 裸跑离线环境；CI 环境下 evaluate metric 走 + # hf-mirror 小文件正常（与 adamss_image 同路径）。npu-1 实跑 1 epoch + # 173s exit 0，eval_accuracy 0.74 / eval_f1 0.83（glue 真 metric）。 + # glue/mrpc 数据与 roberta-base 均在 seed 缓存 + + # LoRA-GA：0.5B + wikitext-2（默认 gpt2 换 seed 的 0.5B，target_modules + # 换 Qwen 系；c_attn 是 gpt2 专用）。梯度估计（device 分支）+ 4590 步 + # 全量 1 epoch 训练 + adapter 落盘，npu-5 实跑 30.7min exit 0 + # （train_loss 4.43）。eval/save_steps 拉满跳过中途评估与存点（只留最终 + # save_pretrained）；num_epochs 默认 3 压到 1。 + - path: examples/lora_ga_finetuning/lora_ga_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model ${SFT_MODEL_PATH} + - --target_modules q_proj k_proj v_proj o_proj + - --grad_estimate_iters 8 + - --num_epochs 1 + - --eval_steps 1000000 + - --save_steps 1000000 + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 60 + + # AdaLoRA seq2seq：零 CLI 入口（模块常量 facebook/bart-base + 8 epochs + + # bs 8），npu-5 原样实跑 exit 0（8 epochs 训完 + accuracy 打印 + adapter + # 落盘 + 末段 generate 复现）。全量 ≈105-110 min（1074 步/epoch @~2.6 it/s + # + 每 epoch eval），是全部条目中最长的腿，timeout 120。模型 facebook/bart-base + # 与数据集 zeroshot/twitter-financial-news-sentiment 在运行时从 + # huggingface.co 下载进共享缓存。 + # pin_memory=True 纯 DataLoader（无 accelerator.prepare）实测不挂。 + + # MiCA：同上配方（Qwen2.5-0.5B + imdb 1% + nll） + - path: examples/mica_finetuning/mica_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --base_model_name_or_path ${SFT_MODEL_PATH} # HfArgumentParser 必填：注入 0.5B + - --loss_type nll # 同 miss：mica_finetuning.py:52 同样 device_map="auto" 触发 accelerate add_hook_to_module；chunked_nll patch 在 functools.partial 上 inspect.signature(__func__) 崩 + timeout_minutes: 30 + + # PVeRa：dinov2-base + beans，原样跑通 + + # AdaMSS image：vit-base + cifar10；coder npu-3 2026-09-17 实跑 e2e 通过 + # train 625 steps @1.34 it/s ≈7.5min + best eval exit (eval_acc=0.0 因 _StubAccuracy 总是返回 0); + # 3 patch：(1) `model = model.to("npu")` 在 AdaMSSConfig 构造之前（AdaMSS layer.py device=weight.device， + # 不 patch 永远 CPU SVD）；(2) stub evaluate.load("accuracy") 同 [[peft-monteclora-evaluate-load-hangs-offline]]； + # (3) PYTHONPATH=/home/coder/work/peft-b-venv/stubs 提供 torchvision.transforms stub（pytorch.org/whl/cpu + # 的 torchvision 0.22-0.29 与 torch 2.9.0+cpu ABI 不匹配，见 [[torchvision-v29-stable-abi]]）。 + # 以上 3 patch 均为 coder 验证机本地补丁，CI 不需要：(1) 脚本不 .to()，get_peft_model + # 时权重在 CPU 上做 SVD（等价），Trainer 再搬 NPU；(2) evaluate.load 走 hf-mirror + # 小文件（glue_adamss_asa 同用法已过）；(3) CI 装 aarch64 torchvision 0.27.0 与 + # torch 2.12.0 ABI 匹配（2026-09-20 随整栈升 2.12 同步，见 setup_example.sh）。 + # CI 资产（run 35222723526 302 cas-bridge 超时根因）：脚本 DATASET_CONFIGS["cifar10"] + # 硬编码 Multimodal-Fatima/CIFAR10_train + CIFAR10_test（不是 uoft-cs/cifar10）， + # 模型默认 google/vit-base-patch16-224-in21k —— ModelScope 均无镜像，运行时 + # 从 huggingface.co 下载进共享缓存。 + # test eval 全量 10000 samples/1250 steps 长跑 ≈15min 是脚本默认行为（无 max_eval_samples 参数）； + - path: examples/adamss_finetuning/image_classification_adamss_asa.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --max_train_samples 200 # 默认整 cifar10 train 50k：压到 200 = 25 step 即可验证 + - --num_epochs 1 # 默认 10：压训练 + - --batch_size 8 # 默认 32：910B4 单卡 32GB 够 + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 30 + + # PiSSA：Qwen2.5-0.5B + ci_alpaca_10 fixture + trl 1.12 NLL loss， + # 1 epoch 5.17s exit 0（loss 5.33→1.59）。 + # 关键 overlay：--loss_type nll 绕开 [[trl-chunked-nll-functools-partial-bug]] + # （pissa 用 device_map="auto" 触发 accelerate.add_hook_to_module 把 forward + # 包成 functools.partial，trl chunked_nll inspect.signature(__func__) 崩）； + # --bits bf16（脚本默认 fp32，省内存） + # data_path 复用 waveft 同款 ci_alpaca_10（目录 + train.jsonl，字段 + # instruction/input/output——alpaca 标准格式）；run 35303810367 曾指向 + # 不存在的 ci_alpaca_10_dir（fixture 从未提交），字段名本身没错。 + + # PSoft：原判"trl 1.10 可跑、1.12 backward 崩"版本敏感——1.12 崩点实为 + # chunked_nll 与 device_map="auto" 的组合（同 miss/mica），--loss_type nll + # 后通过。npu-1 5 步 exit 0（loss 3.31） + - path: examples/psoft_finetuning/psoft_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model_name_or_path ${SFT_MODEL_PATH} + - --loss_type nll + - --max_steps 5 + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 30 + + # Shadow：原判"8B fp32 主机 RAM OOM"——换 0.5B 即解；:59 设备分支 + # torch.device("cuda" if ...) 在 shim 下直通 NPU。内置 3 条 toy texts + # 无数据依赖，默认 5 步。npu-1 exit 0（loss 10.3→7.3，adapter 落盘， + # 末段 reloaded_base 复现加载） + + # KaSA（SVD 正则 LoRA）：原"backward ERR00100 归因待复跑"是 npu-3 裸跑 + # 环境假象。与 miss/mica 同构（SFTConfig 子类 + device_map="auto" + + # imdb 1%），同配方 --loss_type nll 绕 chunked_nll；a2-1 单卡下 + # device_map 全落 npu:0。npu-1 实跑 5 步 exit 0（loss 6.83 含 kasa 正则项） + + # KaSA（SVD 正则 LoRA）：原"backward ERR00100 归因待复跑"是 npu-3 裸跑 + # 环境假象。与 miss/mica 同构（SFTConfig 子类 + device_map="auto" + + # imdb 1%），同配方 --loss_type nll 绕 chunked_nll；a2-1 单卡下 + # device_map 全落 npu:0。npu-1 实跑 5 步 exit 0（loss 6.83 含 kasa 正则项） + - path: examples/kasa_finetuning/kasa_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model_name_or_path ${SFT_MODEL_PATH} + - --loss_type nll # device_map="auto" 触发 functools.partial forward，trl 1.12 chunked_nll 崩（同 miss/mica） + - --max_steps 5 + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 30 + + # UniLoRA：同 kasa 配方（原 backward 崩同为环境假象），npu-1 5 步 exit 0 + + # Shadow：原判"8B fp32 主机 RAM OOM"——换 0.5B 即解；:59 设备分支 + # torch.device("cuda" if ...) 在 shim 下直通 NPU。内置 3 条 toy texts + # 无数据依赖，默认 5 步。npu-1 exit 0（loss 10.3→7.3，adapter 落盘， + # 末段 reloaded_base 复现加载） + - path: examples/shadow_finetuning/shadow_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model_name_or_path ${SFT_MODEL_PATH} # 默认 Qwen/Qwen3-8B（fp32 32GB 必 OOM）：换 0.5B + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 30 + + # LoRA-FA：原判":52-60 设备分支无 npu → device_map='cpu' 主机内存"—— + # shim 下 torch.cuda.is_available()→True → cuda 分支 → device_map="cuda" + # 映射 NPU，bf16 探测同路径通过。数据 query/response 字段走新 fixture + # ci_query_response_8（单文件，shim 映射 train/test，:118 自切 val）。 + # npu-1 1 epoch exit 0（loss 2.68） + + # DoRA（训练脚本，区别于已支持的 dora-caching.py）：同 gralora 配方 + # （npu-5 exit 0，loss 2.37） + - path: examples/dora_finetuning/dora_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model ${SFT_MODEL_PATH} + - --data_path ${TARGET_ROOT}/fixtures/ci_text_8_hf + - --num_epochs 1 + - --batch_size 1 + - --cutoff_len 256 + - --val_set_size 1 + - --learning_rate 1e-4 + - --output_dir ${CI_OUTPUT_DIR} + - --device npu:0 + - --eval_step 100 + - --save_step 100 + timeout_minutes: 30 + + # RoAD：同 dora 配方，**必须 a2-1 单卡**——:76 device_map="auto" 在 2 卡 + # 会把模型切开（mat1 npu:1 vs npu:0 崩），单卡全部落 npu:0 + + # SD dreambooth + stable_diffusion full LoRA：脚本用 argparse subparsers， + # overlay 关键：父 parser flag 全部在前，"lora" 子命令放最后 + # （父 args 在子 parser 之前生效，lora 子 parser 仅接收 --unet_r/--unet_alpha 等） + # 10 steps exit 0（loss 0.0247，17s） + - path: examples/stable_diffusion/train_dreambooth.py + profile: peft_dreambooth + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --pretrained_model_name_or_path ${SD_MODEL_PATH} + - --instance_data_dir ${TARGET_ROOT}/fixtures/ci_dummy_image + - --instance_prompt 'a photo of sks dog' + - --max_train_steps 10 + - --train_batch_size 1 + - --report_to tensorboard + - --output_dir ${CI_OUTPUT_DIR} + - --mixed_precision no + - --resolution 256 + - --center_crop + - lora + timeout_minutes: 30 + + # SD dreambooth + HRA：10 steps exit 0（loss 0.00388，1m10s，npu-5 复验）。 + # v0.21.0 脚本两个坑（均 overlay/setup 绕过，0 patch）： + # (1) :89 Accelerator(project_dir=ProjectConfiguration(...)) 传错 kwarg + # （应为 project_config=）→ accelerate 1.15 init_trackers 在 + # os.path.join(ProjectConfiguration, run_name) 上 TypeError。 + # 绕过：--report_to none → 脚本 :91 log_with=None → trackers 为空， + # init_trackers 变 no-op（wandb 分支同崩溃路径，无关 tensorboard）。 + # (2) :316-319 cwd/data/dreambooth 不存在时无条件 git clone + # github.com/google/dreambooth（runner 网络不通；该路径在脚本里只用于 + # clone 自身，纯死代码）。绕过：setup_peft_dreambooth 预 mkdir。 + # 注意：args_loader 的开关类参数（--push_to_hub/--center_crop 等）是 + # store_true 无值开关，overlay 不能带值（run 35482204039 曾因 + # `--push_to_hub False` 报 unrecognized arguments: False）。 + + # BOFT dreambooth：SD v1.5 + ci_dummy_image + BOFTConfig。与 hra_dreambooth + # 同构（utils/args_loader.py + Accelerator），但 v0.21.0 脚本比 hra 少两处 + # 修复，**零源码 patch 绕过**（2026-09-21 npu-2 实测定论）： + # - :391 init_trackers(...init_kwargs=wandb_init) 无 guard、wandb_init 只在 + # report_to=="wandb" 分支定义 → 走 --report_to wandb 让 wandb_init 有值； + # - wandb_init 硬编码 mode="online" 会要求登录 → run_example.sh 的 + # sitecustomize wandb neutralizer 把 wandb.init 强制 disabled（no-op run）。 + # overlay 关键：--use_boft 是 store_true 必须显式开；--wandb_run_name 必传 + # （adapter_name=None 会让 get_peft_model 崩 TypeError）；--validation_prompt + # 必传（脚本 :88 无条件 args.validation_prompt[0]）。 + # 2026-09-21 npu-2 单卡 2.12 零 patch 实跑 10 步 exit 0（loss 0.0376）。 + - path: examples/boft_dreambooth/train_dreambooth.py + profile: peft_dreambooth + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --pretrained_model_name_or_path ${SD_MODEL_PATH} + - --instance_data_dir ${TARGET_ROOT}/fixtures/ci_dummy_image + - --instance_prompt 'a photo of sks dog' + - --validation_prompt 'a photo of sks dog' # :88 无条件访问 + - --wandb_run_name boft-run # adapter_name=None 崩 + - --wandb_project_name boft-proj + - --use_boft # store_true：不开就是裸 UNet 全参微调 + - --max_train_steps 10 + - --train_batch_size 1 + - --report_to wandb # 零 patch 关键：wandb 分支定义 wandb_init，shim 中性化 login + - --output_dir ${CI_OUTPUT_DIR} + - --mixed_precision no + - --resolution 256 + - --center_crop + - --checkpointing_steps 1000000 + timeout_minutes: 30 + + # t5-base seq2seq LoRA（Accelerator 裸跑，FSDP 分支需 launcher 才激活）： + # 零 CLI 硬编码 t5-base + 本地 FinancialPhraseBank 两个 jsonl + # （temp/data/FinancialPhraseBank-v1.0/，setup_peft 从仓内 fixture 放置）。 + # t5-base 在运行时从 huggingface.co 下载进共享缓存，裸 id 走 hub cache + # 解析。 + + # PiSSA：Qwen2.5-0.5B + ci_alpaca_10 fixture + trl 1.12 NLL loss， + # 1 epoch 5.17s exit 0（loss 5.33→1.59）。 + # 关键 overlay：--loss_type nll 绕开 [[trl-chunked-nll-functools-partial-bug]] + # （pissa 用 device_map="auto" 触发 accelerate.add_hook_to_module 把 forward + # 包成 functools.partial，trl chunked_nll inspect.signature(__func__) 崩）； + # --bits bf16（脚本默认 fp32，省内存） + # data_path 复用 waveft 同款 ci_alpaca_10（目录 + train.jsonl，字段 + # instruction/input/output——alpaca 标准格式）；run 35303810367 曾指向 + # 不存在的 ci_alpaca_10_dir（fixture 从未提交），字段名本身没错。 + - path: examples/pissa_finetuning/pissa_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model_name_or_path ${SFT_MODEL_PATH} + - --bits bf16 + - --data_path ${TARGET_ROOT}/fixtures/ci_alpaca_10 + - --dataset_field instruction output + - --dataset_split 'train[:100%]' + - --max_steps 5 + - --num_train_epochs 1 + - --per_device_train_batch_size 1 + - --gradient_accumulation_steps 1 + - --learning_rate 1e-4 + - --max_length 512 + - --logging_steps 1 + - --save_strategy no + - --report_to none + - --loss_type nll + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 30 + + # DeFT：Qwen2.5-0.5B + ci_text_8_hf fixture + npu:0 1 epoch 5s 内 exit 0 + # （loss 1.86）。 + # 注：原 C-7B-FA-family"族推断"理论**实测不成立**——脚本默认模型即 Qwen2.5-0.5B + # 而非 7B；910B4 单卡 32G 完全够；未触发 aclnn 207001（脚本未硬编码 flash_attention_2） + + # FP4（bnb 4-bit NF4 QLoRA）：零 CLI 模块级脚本，硬编码 facebook/opt-350m + # + Abirate/english_quotes 数据 + fp16=True + OPT 专属 target_modules。 + # 无需 overlay/patch——模型 facebook/opt-350m 与 2508 行的小数据集都在 + # 运行时从 huggingface.co 下载进共享缓存。 + # 关键：本条目是唯一走 bnb 的 supported 例（profile peft_fp4 专装 bnb）， + # run_example.sh 的 SKIP_TRANSFER_TO_NPU 白名单（fp4_finetuning/ 目录） + # 对它跳过 transfer_to_npu——bnb 0.50.2 默认 CPU 后端在 NPU 上跑 4-bit + # NF4，挂 transfer_to_npu 反而让 bnb 误走 cuda backend 崩。fp16=True 由 + # fp16 neutralizer 中性化。bnb 不装全局 profile：diffusers 0.39 硬 + # import bnb，dreambooth 条目挂 transfer_to_npu 时会连带崩。 + - path: examples/fp4_finetuning/finetune_fp4_opt_bnb_peft.py + profile: peft_fp4 + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + timeout_minutes: 30 + + # SD dreambooth + OFT：同上配方，10 steps exit 0（loss 0.00172） + - path: examples/oft_dreambooth/train_dreambooth.py + profile: peft_dreambooth + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --pretrained_model_name_or_path ${SD_MODEL_PATH} + - --instance_data_dir ${TARGET_ROOT}/fixtures/ci_dummy_image + - --instance_prompt 'a photo of sks dog' + - --max_train_steps 10 + - --train_batch_size 1 + - --report_to tensorboard + - --output_dir ${CI_OUTPUT_DIR} + - --mixed_precision no + - --resolution 256 + - --center_crop + timeout_minutes: 30 + + # SD dreambooth + DeFT：同上配方，10 steps exit 0（loss 0.00452） + + # GraLoRA：0.5B + ci_text_8_hf，npu-5 实跑 1 epoch exit 0（loss 2.37） + - path: examples/gralora_finetuning/gralora_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model ${SFT_MODEL_PATH} + - --data_path ${TARGET_ROOT}/fixtures/ci_text_8_hf + - --num_epochs 1 + - --batch_size 1 + - --cutoff_len 256 + - --val_set_size 1 + - --learning_rate 1e-4 + - --output_dir ${CI_OUTPUT_DIR} + - --device npu:0 + - --eval_step 100 + - --save_step 100 + timeout_minutes: 30 + + # LiLY：同 gralora 配方（npu-5 exit 0，loss 2.37） + + # AdaMSS manual：手动 API 演示，roberta-base 单 epoch + - path: examples/adamss_finetuning/glue_adamss_asa_manual_example.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --num_epochs 1 # 默认 100：压到单 epoch + timeout_minutes: 45 + + # Supertuning：Qwen2.5-0.5B + guanaco 完整 1 epoch（308 步 exit 0） + + # ALoRA：0.5B + ci_alpaca_train_test（input/output 字段）；npu-5 exit 0 + # （loss 7.83，alora invocation tokens 路径） + - path: examples/alora_finetuning/alora_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model ${SFT_MODEL_PATH} + - --data_path ${TARGET_ROOT}/fixtures/ci_alpaca_train_test + - --num_epochs 1 + - --batch_size 1 + - --cutoff_len 256 + - --val_set_size 2 + - --learning_rate 1e-4 + - --output_dir ${CI_OUTPUT_DIR} + - --device npu:0 + - --eval_step 100 + - --save_step 100 + timeout_minutes: 30 + + # DoRA（训练脚本，区别于已支持的 dora-caching.py）：同 gralora 配方 + # （npu-5 exit 0，loss 2.37） + + # Peanut：同 gralora 配方（npu-5 exit 0，loss 2.37） + - path: examples/peanut_finetuning/peanut_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model ${SFT_MODEL_PATH} + - --data_path ${TARGET_ROOT}/fixtures/ci_text_8_hf + - --num_epochs 1 + - --batch_size 1 + - --cutoff_len 256 + - --val_set_size 1 + - --learning_rate 1e-4 + - --output_dir ${CI_OUTPUT_DIR} + - --device npu:0 + - --eval_step 100 + - --save_step 100 + timeout_minutes: 30 + + # ALoRA：0.5B + ci_alpaca_train_test（input/output 字段）；npu-5 exit 0 + # （loss 7.83，alora invocation tokens 路径） + + # SD dreambooth + DeFT：同上配方，10 steps exit 0（loss 0.00452） + - path: examples/deft_dreambooth/train_dreambooth.py + profile: peft_dreambooth + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --pretrained_model_name_or_path ${SD_MODEL_PATH} + - --instance_data_dir ${TARGET_ROOT}/fixtures/ci_dummy_image + - --instance_prompt 'a photo of sks dog' + - --max_train_steps 10 + - --train_batch_size 1 + - --report_to tensorboard + - --output_dir ${CI_OUTPUT_DIR} + - --mixed_precision no + - --resolution 256 + - --center_crop + timeout_minutes: 30 + + # SD dreambooth + stable_diffusion full LoRA：脚本用 argparse subparsers， + # overlay 关键：父 parser flag 全部在前，"lora" 子命令放最后 + # （父 args 在子 parser 之前生效，lora 子 parser 仅接收 --unet_r/--unet_alpha 等） + # 10 steps exit 0（loss 0.0247，17s） + + # AdaMSS GLUE：roberta-base + mrpc 单 epoch + - path: examples/adamss_finetuning/glue_adamss_asa_example.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --model_name_or_path ${ROBERTA_BASE_PATH} # argparse 收：setup 预下，避开 hf-mirror xet + - --dataset_name mrpc # 默认 cola；mrpc 数据量小跑得快（docstring 推荐用法） + - --num_epochs 1 # 默认 100：压到单 epoch（mrpc 3.7k 样本几分钟） + timeout_minutes: 45 + + # AdaMSS manual：手动 API 演示，roberta-base 单 epoch + + # MonteCLoRA 序列分类：原判 evaluate.load("glue","mrpc") import 时挂 + # （:33）——那是 npu-3 裸跑离线环境；CI 环境下 evaluate metric 走 + # hf-mirror 小文件正常（与 adamss_image 同路径）。npu-1 实跑 1 epoch + # 173s exit 0，eval_accuracy 0.74 / eval_f1 0.83（glue 真 metric）。 + # glue/mrpc 数据与 roberta-base 均在 seed 缓存 + - path: examples/monteclora_finetuning/sequence_classification_finetune_monteclora.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model ${ROBERTA_BASE_PATH} # 默认 roberta-base：seed 已种，显式注入本地路径 + - --num_epochs 1 + - --batch_size 16 + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 30 + + # BEFT：mt0-small + financial_phrasebank，原样跑通（两套环境均 exit 0） + - path: examples/beft_finetuning/beft_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + timeout_minutes: 30 + + # MiSS：Qwen2.5-0.5B + imdb 1% + + # UniLoRA：同 kasa 配方（原 backward 崩同为环境假象），npu-1 5 步 exit 0 + - path: examples/unilora_finetuning/unilora_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model_name_or_path ${SFT_MODEL_PATH} + - --loss_type nll + - --max_steps 5 + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 30 + + # PSoft：原判"trl 1.10 可跑、1.12 backward 崩"版本敏感——1.12 崩点实为 + # chunked_nll 与 device_map="auto" 的组合（同 miss/mica），--loss_type nll + # 后通过。npu-1 5 步 exit 0（loss 3.31） + + # SFT + torchrun DDP（Mistral-7B 换 0.5B）：首行 --nproc_per_node 8 + # 原样跑 a2-8；npu-5 2 rank hccl exit 0（56s） + # timeout 60：a2-8 pod 排队实测可达 24 分钟（run 37877060110 本条 30.4 + # 分钟被杀，其中干活仅约 6 分钟），30 分钟会把排队误判成被测失败。 + - path: examples/sft/run_peft_multigpu.sh + profile: peft + runner: linux-aarch64-a2-8 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --model_name_or_path ${SFT_MODEL_PATH} # 默认 mistralai/Mistral-7B-v0.1：换 0.5B + - --dataset_name ${TARGET_ROOT}/fixtures/ci_sft_8.jsonl + - --use_4bit_quantization False # .sh 默认 True：bnb 无 NPU 后端 + - --use_nested_quant False # 同上（bnb_4bit_* 残留参数 4bit 关时不被消费） + - --use_flash_attn False + - --gradient_checkpointing False + - --use_reentrant False + - --push_to_hub False + - --hub_private_repo False + - --packing False + - --per_device_train_batch_size 2 + - --per_device_eval_batch_size 2 + - --gradient_accumulation_steps 1 + - --max_steps 1 + - --logging_steps 1 + - --report_to none + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 60 + + # SFT + QLoRA-FSDP（Llama-2-70b 换 0.5B，bnb 关掉退化成 LoRA+FSDP1）： + # fsdp_config_qlora.yaml 本就 num_processes: 2，只需 a2-2（不必占 a2-8）； + # npu-5 2×910B4 2 rank 实跑 1m18s exit 0 + # profile peft_29：同 run_peft_fsdp.sh，torch 2.12 存点路径崩，锁 2.9.0 + + # LoRA-FA：原判":52-60 设备分支无 npu → device_map='cpu' 主机内存"—— + # shim 下 torch.cuda.is_available()→True → cuda 分支 → device_map="cuda" + # 映射 NPU，bf16 探测同路径通过。数据 query/response 字段走新 fixture + # ci_query_response_8（单文件，shim 映射 train/test，:118 自切 val）。 + # npu-1 1 epoch exit 0（loss 2.68） + - path: examples/lorafa_finetune/lorafa_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model_name_or_path ${SFT_MODEL_PATH} # 默认 gated Meta-Llama-3-8B：换 0.5B + - --dataset_name_or_path ${TARGET_ROOT}/fixtures/ci_query_response_8.jsonl # 默认 meta-math/MetaMathQA-40K：换 fixture + - --num_epochs 1 + - --batch_size 1 + - --cutoff_len 256 + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 30 + + # BEFT：mt0-small + financial_phrasebank，原样跑通（两套环境均 exit 0） + + # SD dreambooth + LoRA：stable-diffusion-v1-5 + dummy image + 10 steps + # exit 0（loss 0.00695，17s）。overlay 关键： + # --report_to tensorboard（accelerate 拒 "none"） + # --center_crop 是 store_true flag，不传 =True + # --mixed_precision no（peft SD 例无 grad_scaler，NPU bf16 需 explicit） + # CI 资产（run 35303810367 缺 diffusers 崩 import）：profile peft_dreambooth + # 在 setup_peft 之上装 diffusers + tensorboard 并从共享缓存解析 + # ${SD_MODEL_PATH}：stable-diffusion-v1-5/stable-diffusion-v1-5，缓存未命中 + # 时由 scripts/hub_cache.py 下载；ci_dummy_image fixture = + # 2 张 256×256 PNG。 + - path: examples/lora_dreambooth/train_dreambooth.py + profile: peft_dreambooth + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --pretrained_model_name_or_path ${SD_MODEL_PATH} + - --instance_data_dir ${TARGET_ROOT}/fixtures/ci_dummy_image + - --instance_prompt 'a photo of sks dog' + - --max_train_steps 10 + - --train_batch_size 1 + - --report_to tensorboard + - --output_dir ${CI_OUTPUT_DIR} + - --mixed_precision no + - --resolution 256 + - --center_crop + timeout_minutes: 30 + + # SD dreambooth + OFT：同上配方，10 steps exit 0（loss 0.00172） + + # 序列分类（非 LoRA 方法）：bert-base + GLUE + - path: examples/sequence_classification/peft_no_lora_accelerate.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --model_name_or_path ${BERT_BASE_UNCASED_PATH} # argparse 必填：setup 预下，避开 hf-mirror xet + - --output_dir ${CI_OUTPUT_DIR} # argparse 断言必填（脚本硬校验） + timeout_minutes: 45 + + # AdaMSS GLUE：roberta-base + mrpc 单 epoch + + # MiSS：Qwen2.5-0.5B + imdb 1% + - path: examples/miss_finetuning/miss_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --base_model_name_or_path ${SFT_MODEL_PATH} # HfArgumentParser 必填：注入 setup 预下载的 0.5B + - --dataset_field text label # upstream bug：ScriptArguments(SFTConfig) dataclass 默认 None，但 miss_finetuning.py:89 直接 dataset_field[0]/[1] 取下标，CUDA 也一样 TypeError（纯 Python 逻辑与硬件无关）。text=imdb 评论, label=0/1 情感 + - --loss_type nll # 绕开 trl 1.12+ 默认 chunked_nll：miss_finetuning.py:61 用 device_map="auto" 加载模型，accelerate.add_hook_to_module 把 model.forward 替换成 functools.partial；trl 1.12 sft_trainer.py:383 (1.13 在 386) 的 _patch_chunked_ce_lm_head 走 inspect.signature(original_forward.__func__) 假设 forward 是普通 method，partial 没 __func__ → AttributeError（纯 PyTorch + accelerate，硬件无关，CUDA 同问题）。nll 走标准 cross-entropy 跳过整个 chunked patch。注：sft_trainer.py:1331 isinstance(BaseTunerLayer) guard 在这两例不会触发——target_modules=["q_proj","o_proj","k_proj","v_proj","gate_proj","up_proj","down_proj"] 不含 lm_head，model.get_output_embeddings() 还是普通 Linear，guard 放过后 patch 内部才崩 + timeout_minutes: 30 + + # MiCA：同上配方（Qwen2.5-0.5B + imdb 1% + nll） + + # PVeRa：dinov2-base + beans，原样跑通 + - path: examples/pvera/confidence_interval_generation.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + timeout_minutes: 30 + + # 序列分类（非 LoRA 方法）：bert-base + GLUE + + # SD dreambooth + HRA：10 steps exit 0（loss 0.00388，1m10s，npu-5 复验）。 + # v0.21.0 脚本两个坑（均 overlay/setup 绕过，0 patch）： + # (1) :89 Accelerator(project_dir=ProjectConfiguration(...)) 传错 kwarg + # （应为 project_config=）→ accelerate 1.15 init_trackers 在 + # os.path.join(ProjectConfiguration, run_name) 上 TypeError。 + # 绕过：--report_to none → 脚本 :91 log_with=None → trackers 为空， + # init_trackers 变 no-op（wandb 分支同崩溃路径，无关 tensorboard）。 + # (2) :316-319 cwd/data/dreambooth 不存在时无条件 git clone + # github.com/google/dreambooth（runner 网络不通；该路径在脚本里只用于 + # clone 自身，纯死代码）。绕过：setup_peft_dreambooth 预 mkdir。 + # 注意：args_loader 的开关类参数（--push_to_hub/--center_crop 等）是 + # store_true 无值开关，overlay 不能带值（run 35482204039 曾因 + # `--push_to_hub False` 报 unrecognized arguments: False）。 + - path: examples/hra_dreambooth/train_dreambooth.py + profile: peft_dreambooth + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --pretrained_model_name_or_path ${SD_MODEL_PATH} + - --instance_data_dir ${TARGET_ROOT}/fixtures/ci_dummy_image + - --instance_prompt 'a photo of sks dog' + - --validation_prompt 'a photo of sks dog' # 脚本 :83 无条件访问 + - --max_train_steps 10 + - --train_batch_size 1 + - --report_to none # 见上 (1)：躲 ProjectConfiguration init_trackers 崩 + - --output_dir ${CI_OUTPUT_DIR} + - --mixed_precision no + - --resolution 256 + - --center_crop + - --checkpointing_steps 1000000 + timeout_minutes: 30 + + # SFT + QLoRA-DeepSpeed-Z3（Llama-2-70b 换 0.5B，bnb 关掉退化 ZeRO-3+ + # LoRA）：deepspeed_config_z3_qlora.yaml 本就 num_processes: 2 且无 + # gradient_accumulation_steps 键（transformers 不做硬校验），只需 a2-2 + # （不必占 a2-8）；npu-5 2 rank 实跑 1m38s exit 0 + # torch 栈锁 2.9.0 对：同 run_peft_deepspeed.sh（存点 gather_object + # 路径 + ZeRO broadcast，run 35502546409） + # timeout 45：a2-2 pod 排队比 a2-8 短得多（run 37877060110 同轮仅 5.7 + # 分钟），但仍留出排队余量。 + - path: examples/sft/run_peft_qlora_deepspeed_stage3.sh + profile: peft_ds + runner: linux-aarch64-a2-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --model_name_or_path ${SFT_MODEL_PATH} + - --dataset_name ${TARGET_ROOT}/fixtures/ci_sft_8.jsonl + - --use_4bit_quantization False + - --use_nested_quant False + - --use_flash_attn False + - --gradient_checkpointing False + - --use_reentrant False + - --push_to_hub False + - --hub_private_repo False + - --packing False + - --per_device_train_batch_size 2 + - --per_device_eval_batch_size 2 + - --gradient_accumulation_steps 1 + - --max_steps 1 + - --logging_steps 1 + - --report_to none + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 45 + + # GraLoRA：0.5B + ci_text_8_hf，npu-5 实跑 1 epoch exit 0（loss 2.37） + + # Glora / GraLoRA 变体：Qwen2.5-0.5B + 10 行 alpaca-style fixture + 小 val_size + # 1 epoch；coder npu-3 2026-09-17 实跑 4 步 5.68s exit 0 train_loss 3.26， + # adapter 落盘。无任何源码 patch——用 Trainer 默认 fp16=False，load_dataset + # + train_test_split（无 max_steps 硬编码），兼容 fixtures/ci_alpaca_10 目录 + - path: examples/glora_finetuning/glora_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --base_model ${SFT_MODEL_PATH} + - --data_path ${TARGET_ROOT}/fixtures/ci_alpaca_10 # 默认 yahma/alpaca-cleaned 52k：换 fixture + - --num_epochs 1 + - --batch_size 2 + - --val_set_size 2 + - --eval_step 100 + - --save_step 100 + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 30 + + # HiRA（GraLoRA 的 rank-α 变体）：同 glora 配方 + 同 fixture；coder npu-3 + # 2026-09-17 实跑 4 步 4.39s exit 0 train_loss 3.26，0 patch + + # SFT + DeepSpeed ZeRO-3（Llama-2-70b 换 0.5B）：deepspeed_config.yaml + # num_processes: 8 + gradient_accumulation_steps: 4 原样跑 a2-8，overlay + # grad_accum 必须跟 4（transformers 硬校验）；profile peft_ds 装 + # deepspeed 0.19.7（npu accelerator 自动识别，DS_BUILD_OPS=0 免编译）； + # npu-5 1m40s exit 0 + # torch 栈锁 2.9.0 对：torch 2.12 broadcast sm90 检查 + torch_npu + # get_device_capability→None 在 ZeRO zero-init 的 param broadcast 崩 + # （run 35502546409） + - path: examples/sft/run_peft_deepspeed.sh + profile: peft_ds + runner: linux-aarch64-a2-8 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --model_name_or_path ${SFT_MODEL_PATH} + - --dataset_name ${TARGET_ROOT}/fixtures/ci_sft_8.jsonl + - --use_flash_attn False + - --gradient_checkpointing False + - --use_reentrant False + - --push_to_hub False + - --hub_private_repo False + - --packing False + - --per_device_train_batch_size 2 + - --per_device_eval_batch_size 2 + - --gradient_accumulation_steps 4 # 必须与 deepspeed_config.yaml 硬编码的 4 一致（transformers 硬校验） + - --max_steps 1 + - --logging_steps 1 + - --report_to none + - --output_dir ${CI_OUTPUT_DIR} + # timeout 60：同 run_peft_multigpu.sh 的 a2-8 排队风险，本条在 + # run 37877060110 跑了 26.9 分钟，距 30 分钟上限只剩 3 分钟。 + timeout_minutes: 60 + + # SFT + QLoRA-DeepSpeed-Z3（Llama-2-70b 换 0.5B，bnb 关掉退化 ZeRO-3+ + # LoRA）：deepspeed_config_z3_qlora.yaml 本就 num_processes: 2 且无 + # gradient_accumulation_steps 键（transformers 不做硬校验），只需 a2-2 + # （不必占 a2-8）；npu-5 2 rank 实跑 1m38s exit 0 + # torch 栈锁 2.9.0 对：同 run_peft_deepspeed.sh（存点 gather_object + # 路径 + ZeRO broadcast，run 35502546409） + + # t5-base seq2seq LoRA（Accelerator 裸跑，FSDP 分支需 launcher 才激活）： + # 零 CLI 硬编码 t5-base + 本地 FinancialPhraseBank 两个 jsonl + # （temp/data/FinancialPhraseBank-v1.0/，setup_peft 从仓内 fixture 放置）。 + # t5-base 在运行时从 huggingface.co 下载进共享缓存，裸 id 走 hub cache + # 解析。 + - path: examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + timeout_minutes: 30 + + # FP4（bnb 4-bit NF4 QLoRA）：零 CLI 模块级脚本，硬编码 facebook/opt-350m + # + Abirate/english_quotes 数据 + fp16=True + OPT 专属 target_modules。 + # 无需 overlay/patch——模型 facebook/opt-350m 与 2508 行的小数据集都在 + # 运行时从 huggingface.co 下载进共享缓存。 + # 关键：本条目是唯一走 bnb 的 supported 例（profile peft_fp4 专装 bnb）， + # run_example.sh 的 SKIP_TRANSFER_TO_NPU 白名单（fp4_finetuning/ 目录） + # 对它跳过 transfer_to_npu——bnb 0.50.2 默认 CPU 后端在 NPU 上跑 4-bit + # NF4，挂 transfer_to_npu 反而让 bnb 误走 cuda backend 崩。fp16=True 由 + # fp16 neutralizer 中性化。bnb 不装全局 profile：diffusers 0.39 硬 + # import bnb，dreambooth 条目挂 transfer_to_npu 时会连带崩。 + + # RoAD：同 dora 配方，**必须 a2-1 单卡**——:76 device_map="auto" 在 2 卡 + # 会把模型切开（mat1 npu:1 vs npu:0 崩），单卡全部落 npu:0 + - path: examples/road_finetuning/road_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model ${SFT_MODEL_PATH} + - --data_path ${TARGET_ROOT}/fixtures/ci_text_8_hf + - --num_epochs 1 + - --batch_size 1 + - --cutoff_len 256 + - --val_set_size 1 + - --learning_rate 1e-4 + - --output_dir ${CI_OUTPUT_DIR} + - --device npu:0 + - --eval_step 100 + - --save_step 100 + timeout_minutes: 30 + + # LoRA-GA：0.5B + wikitext-2（默认 gpt2 换 seed 的 0.5B，target_modules + # 换 Qwen 系；c_attn 是 gpt2 专用）。梯度估计（device 分支）+ 4590 步 + # 全量 1 epoch 训练 + adapter 落盘，npu-5 实跑 30.7min exit 0 + # （train_loss 4.43）。eval/save_steps 拉满跳过中途评估与存点（只留最终 + # save_pretrained）；num_epochs 默认 3 压到 1。 + + # WaveFT（FFT 频域初始化 + LoRA）：同 glora 配方 + 同 fixture；coder npu-3 + # 2026-09-17 实跑 4 步 4.22s exit 0 train_loss 3.23，0 patch + - path: examples/waveft_finetuning/waveft_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --base_model ${SFT_MODEL_PATH} + - --data_path ${TARGET_ROOT}/fixtures/ci_alpaca_10 + - --num_epochs 1 + - --batch_size 2 + - --val_set_size 2 + - --eval_step 100 + - --save_step 100 + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 30 + + # AdaMSS image：vit-base + cifar10；coder npu-3 2026-09-17 实跑 e2e 通过 + # train 625 steps @1.34 it/s ≈7.5min + best eval exit (eval_acc=0.0 因 _StubAccuracy 总是返回 0); + # 3 patch：(1) `model = model.to("npu")` 在 AdaMSSConfig 构造之前（AdaMSS layer.py device=weight.device， + # 不 patch 永远 CPU SVD）；(2) stub evaluate.load("accuracy") 同 [[peft-monteclora-evaluate-load-hangs-offline]]； + # (3) PYTHONPATH=/home/coder/work/peft-b-venv/stubs 提供 torchvision.transforms stub（pytorch.org/whl/cpu + # 的 torchvision 0.22-0.29 与 torch 2.9.0+cpu ABI 不匹配，见 [[torchvision-v29-stable-abi]]）。 + # 以上 3 patch 均为 coder 验证机本地补丁，CI 不需要：(1) 脚本不 .to()，get_peft_model + # 时权重在 CPU 上做 SVD（等价），Trainer 再搬 NPU；(2) evaluate.load 走 hf-mirror + # 小文件（glue_adamss_asa 同用法已过）；(3) CI 装 aarch64 torchvision 0.27.0 与 + # torch 2.12.0 ABI 匹配（2026-09-20 随整栈升 2.12 同步，见 setup_example.sh）。 + # CI 资产（run 35222723526 302 cas-bridge 超时根因）：脚本 DATASET_CONFIGS["cifar10"] + # 硬编码 Multimodal-Fatima/CIFAR10_train + CIFAR10_test（不是 uoft-cs/cifar10）， + # 模型默认 google/vit-base-patch16-224-in21k —— ModelScope 均无镜像，运行时 + # 从 huggingface.co 下载进共享缓存。 + # test eval 全量 10000 samples/1250 steps 长跑 ≈15min 是脚本默认行为（无 max_eval_samples 参数）； + + # TRL SFTTrainer LoRA，1 卡 1 step + - path: examples/sft/run_peft.sh + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --model_name_or_path ${SFT_MODEL_PATH} # 脚本默认 Mistral-7B：换 setup 预下载的 0.5B（CI 规模 + 避 HF 下载） + - --dataset_name ${TARGET_ROOT}/fixtures/ci_sft_8.jsonl # 默认拉 HF hub：换仓内 8 行 fixture + - --use_4bit_quantization False # bnb 4bit 无 NPU 后端 + - --use_flash_attn False # flash-attn 仅 CUDA + - --gradient_checkpointing False # 0.5B 无需省显存；关掉避免 NPU reentrant 兼容问题 + - --use_reentrant False # 同上（checkpointing 已关，双保险） + - --push_to_hub False # 默认 True：CI 不推模型仓 + - --packing False # 8 行样本无需打包，行为更可预期 + - --max_length 512 # 默认 2048：压序列长度 + - --per_device_train_batch_size 1 # 默认 8：压批 + - --per_device_eval_batch_size 1 # 同上 + - --gradient_accumulation_steps 1 # 默认 8：1 step 即可验证 + - --max_steps 1 # 默认整 epoch：1 步压到 CI 规模 + - --logging_steps 1 # 默认 5：确保 1 步内也有日志 + - --dataset_num_proc 1 # 避免 datasets 多进程在 CI 容器的不稳定 + - --report_to none # 关闭 wandb 等外部上报（未装会报错） + - --output_dir ${CI_OUTPUT_DIR} # 默认仓内相对路径：输出必须进 CI 输出目录 + timeout_minutes: 90 + + # SFT + FSDP1（Llama-2-70b 换 0.5B）：fsdp_config.yaml num_processes: 8 + # 原样跑 a2-8；npu-5 2 rank 实跑 1m07s exit 0 + # profile peft_29：torch 栈锁 2.9.0 对——torch 2.12 c10d broadcast 的 + # sm90 检查 + torch_npu get_device_capability 返回 None → None[0] + # TypeError（FSDP 存点 dist_cp gather_object 路径，run 35502546409） + + # HiRA（GraLoRA 的 rank-α 变体）：同 glora 配方 + 同 fixture；coder npu-3 + # 2026-09-17 实跑 4 步 4.39s exit 0 train_loss 3.26，0 patch + - path: examples/hira_finetuning/hira_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --base_model ${SFT_MODEL_PATH} + - --data_path ${TARGET_ROOT}/fixtures/ci_alpaca_10 + - --num_epochs 1 + - --batch_size 2 + - --val_set_size 2 + - --eval_step 100 + - --save_step 100 + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 30 + + # OLoRA（正交 LoRA）：同 glora 配方 + 同 fixture；coder npu-3 2026-09-17 + # 实跑 4 步 3.33s exit 0 train_loss 3.25，0 patch（quantize 默认关时无 bnb 依赖） + + # OLoRA（正交 LoRA）：同 glora 配方 + 同 fixture；coder npu-3 2026-09-17 + # 实跑 4 步 3.33s exit 0 train_loss 3.25，0 patch（quantize 默认关时无 bnb 依赖） + - path: examples/olora_finetuning/olora_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --base_model ${SFT_MODEL_PATH} + - --data_path ${TARGET_ROOT}/fixtures/ci_alpaca_10 + - --num_epochs 1 + - --batch_size 2 + - --val_set_size 2 + - --eval_step 100 + - --save_step 100 + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 30 + + # WaveFT（FFT 频域初始化 + LoRA）：同 glora 配方 + 同 fixture；coder npu-3 + # 2026-09-17 实跑 4 步 4.22s exit 0 train_loss 3.23，0 patch + + # ShiRA（稀疏 mask LoRA）已撤至 unsupported：torch_npu 2.12 sparse + # backward 缺陷 flaky（bf16 也不可靠），见 unsupported 段。 + + # BOFT dreambooth：SD v1.5 + ci_dummy_image + BOFTConfig。与 hra_dreambooth + # 同构（utils/args_loader.py + Accelerator），但 v0.21.0 脚本比 hra 少两处 + # 修复，**零源码 patch 绕过**（2026-09-21 npu-2 实测定论）： + # - :391 init_trackers(...init_kwargs=wandb_init) 无 guard、wandb_init 只在 + # report_to=="wandb" 分支定义 → 走 --report_to wandb 让 wandb_init 有值； + # - wandb_init 硬编码 mode="online" 会要求登录 → run_example.sh 的 + # sitecustomize wandb neutralizer 把 wandb.init 强制 disabled（no-op run）。 + # overlay 关键：--use_boft 是 store_true 必须显式开；--wandb_run_name 必传 + # （adapter_name=None 会让 get_peft_model 崩 TypeError）；--validation_prompt + # 必传（脚本 :88 无条件 args.validation_prompt[0]）。 + # 2026-09-21 npu-2 单卡 2.12 零 patch 实跑 10 步 exit 0（loss 0.0376）。 + + # Supertuning：Qwen2.5-0.5B + guanaco 完整 1 epoch（308 步 exit 0） + - path: examples/supertuning_finetuning/supertuning_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --base_model ${SFT_MODEL_PATH} # 默认 meta-llama/Llama-3.2-1B（gated + 大）：换 0.5B + - --data_path ${TARGET_ROOT}/fixtures/ci_supertuning_8.jsonl # 默认 timdettmers/openassistant-guanaco：该仓文件已转 xet 存储，hf-mirror 的 resolve 对 xet 文件 302 直指 cas-bridge.xethub.hf.co（runner 不可达，HF_HUB_DISABLE_XET 也救不了 plain 路径；ModelScope 无此数据集）。换 8 行真实样本 fixture（{"text": "### Human:...### Assistant:..."} 同上游 schema），run_example.sh 的 load_dataset 垫片会映射 train/test 双 split + - --batch_size 2 # 默认 1：控时长 + timeout_minutes: 45 + + # CorDA：Qwen2.5-0.5B + 走 --corda_mode False + --lora_r 8 退化普通 LoRA（避开 + # preprocess.py SVD 产物 + 默认 MetaMathQA 10 万条） + + # CorDA：Qwen2.5-0.5B + 走 --corda_mode False + --lora_r 8 退化普通 LoRA（避开 + # preprocess.py SVD 产物 + 默认 MetaMathQA 10 万条） + - path: examples/corda_finetuning/corda_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - --model_name_or_path ${SFT_MODEL_PATH} # 默认 facebook/opt-125m：换 setup 预下载的 0.5B + - --corda_mode False # 默认 True：走 preprocess.py SVD 产物，NPU 上没 preprocess 跑不通 + - --lora_r 8 # corda_mode=False 时必传（:212 elif 分支判 lora_r），否则落 full finetuning + - --data_path ${TARGET_ROOT}/fixtures/ci_corda_8 # load_dataset 须**目录**（含 train.jsonl + test.jsonl），单文件路径报 FileNotFoundError + - --dataset_field instruction response # 默认 None：TrainingArguments:69 必填（preprocess 用作 query/response 取字段名） + - --dataset_split train # 默认 train[:100000]：fixture 只有 8 行 + - --dataloader_num_proc 1 # 默认 16：CI 容器 datasets 多进程不稳 + - --max_steps 5 # 1 epoch 跑通即可 + - --per_device_train_batch_size 2 # 默认 8 + Qwen2.5-0.5B fp16 ≈ 10GB HBM，batch 2 安全 + - --report_to none # 默认 wandb：未装会报错 + - --output_dir ${CI_OUTPUT_DIR} # 默认仓内相对路径：输出必须进 CI 输出目录 + timeout_minutes: 30 + + # Glora / GraLoRA 变体：Qwen2.5-0.5B + 10 行 alpaca-style fixture + 小 val_size + # 1 epoch；coder npu-3 2026-09-17 实跑 4 步 5.68s exit 0 train_loss 3.26， + # adapter 落盘。无任何源码 patch——用 Trainer 默认 fp16=False，load_dataset + # + train_test_split（无 max_steps 硬编码），兼容 fixtures/ci_alpaca_10 目录 + + # SFT + FSDP1（Llama-2-70b 换 0.5B）：fsdp_config.yaml num_processes: 8 + # 原样跑 a2-8；npu-5 2 rank 实跑 1m07s exit 0 + # profile peft_29：torch 栈锁 2.9.0 对——torch 2.12 c10d broadcast 的 + # sm90 检查 + torch_npu get_device_capability 返回 None → None[0] + # TypeError（FSDP 存点 dist_cp gather_object 路径，run 35502546409） + - path: examples/sft/run_peft_fsdp.sh + profile: peft_29 + runner: linux-aarch64-a2-8 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --model_name_or_path ${SFT_MODEL_PATH} # 默认 gated meta-llama/Llama-2-70b-hf：换 seed 的 0.5B + - --dataset_name ${TARGET_ROOT}/fixtures/ci_sft_8.jsonl # 默认 smangrul/ultrachat-10k-chatml：换仓内 fixture（messages 字段，chatml 模板兼容） + - --use_flash_attn False # .sh 默认 True，flash-attn 仅 CUDA + - --gradient_checkpointing False # 0.5B 无需省显存；避开 NPU reentrant 兼容问题 + - --use_reentrant False # 双保险 + - --push_to_hub False # 默认 True：CI 不推模型仓 + - --hub_private_repo False # 同上 + - --packing False # 8 行样本无需打包 + - --per_device_train_batch_size 2 # 8 行 / bs2 / 2 rank = 各 2 步，DDP/FSDP 均衡 + - --per_device_eval_batch_size 2 + - --gradient_accumulation_steps 1 # 1 step 即可验证 + - --max_steps 1 + - --logging_steps 1 + - --report_to none + - --output_dir ${CI_OUTPUT_DIR} + # timeout 60：同 run_peft_multigpu.sh 的 a2-8 排队风险。 + timeout_minutes: 60 + + # SFT + torchrun DDP（Mistral-7B 换 0.5B）：首行 --nproc_per_node 8 + # 原样跑 a2-8；npu-5 2 rank hccl exit 0（56s） + + # DeFT：Qwen2.5-0.5B + ci_text_8_hf fixture + npu:0 1 epoch 5s 内 exit 0 + # （loss 1.86）。 + # 注：原 C-7B-FA-family"族推断"理论**实测不成立**——脚本默认模型即 Qwen2.5-0.5B + # 而非 7B；910B4 单卡 32G 完全够；未触发 aclnn 207001（脚本未硬编码 flash_attention_2） + - path: examples/deft_finetuning/deft_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model ${SFT_MODEL_PATH} + - --data_path ${TARGET_ROOT}/fixtures/ci_text_8_hf + - --num_epochs 1 + - --batch_size 1 + - --cutoff_len 256 + - --val_set_size 1 + - --learning_rate 1e-4 + - --output_dir ${CI_OUTPUT_DIR} + - --device npu:0 + - --eval_step 100 + - --save_step 100 + timeout_minutes: 30 + + # DeLoRA：同 DeFT 配方，exit 0 loss 1.86 + + # RandLoRA：同 DeFT 配方 + --use_lora flag，exit 0 loss 1.86 + - path: examples/randlora_finetuning/randlora_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model ${SFT_MODEL_PATH} + - --data_path ${TARGET_ROOT}/fixtures/ci_text_8_hf + - --num_epochs 1 + - --batch_size 1 + - --cutoff_len 256 + - --val_set_size 1 + - --learning_rate 1e-4 + - --output_dir ${CI_OUTPUT_DIR} + - --device npu:0 + - --eval_step 100 + - --save_step 100 + - --use_lora + timeout_minutes: 30 + + # DoRA caching：Qwen2.5-0.5B + npu + memory profiling，exit 0 + # （多次 forward/backward 展示 NPU HBM 占用） + + # SFT + QLoRA-FSDP（Llama-2-70b 换 0.5B，bnb 关掉退化成 LoRA+FSDP1）： + # fsdp_config_qlora.yaml 本就 num_processes: 2，只需 a2-2（不必占 a2-8）； + # npu-5 2×910B4 2 rank 实跑 1m18s exit 0 + # profile peft_29：同 run_peft_fsdp.sh，torch 2.12 存点路径崩，锁 2.9.0 + # timeout 45：同 run_peft_qlora_deepspeed_stage3.sh 的 a2-2 排队余量。 + - path: examples/sft/run_peft_qlora_fsdp.sh + profile: peft_29 + runner: linux-aarch64-a2-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --model_name_or_path ${SFT_MODEL_PATH} + - --dataset_name ${TARGET_ROOT}/fixtures/ci_sft_8.jsonl + - --use_4bit_quantization False # .sh 默认 True：bnb 无 NPU 后端，关掉退化 LoRA+FSDP + - --use_nested_quant False + - --use_flash_attn False + - --gradient_checkpointing False + - --use_reentrant False + - --push_to_hub False + - --hub_private_repo False + - --packing False + - --per_device_train_batch_size 2 + - --per_device_eval_batch_size 2 + - --gradient_accumulation_steps 1 + - --max_steps 1 + - --logging_steps 1 + - --report_to none + - --output_dir ${CI_OUTPUT_DIR} + timeout_minutes: 45 + + # SFT + DeepSpeed ZeRO-3（Llama-2-70b 换 0.5B）：deepspeed_config.yaml + # num_processes: 8 + gradient_accumulation_steps: 4 原样跑 a2-8，overlay + # grad_accum 必须跟 4（transformers 硬校验）；profile peft_ds 装 + # deepspeed 0.19.7（npu accelerator 自动识别，DS_BUILD_OPS=0 免编译）； + # npu-5 1m40s exit 0 + # torch 栈锁 2.9.0 对：torch 2.12 broadcast sm90 检查 + torch_npu + # get_device_capability→None 在 ZeRO zero-init 的 param broadcast 崩 + # （run 35502546409） + + # DoRA caching：Qwen2.5-0.5B + npu + memory profiling，exit 0 + # （多次 forward/backward 展示 NPU HBM 占用） + - path: examples/dora_finetuning/dora-caching.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --model_id ${SFT_MODEL_PATH} + - --num_runs 3 + timeout_minutes: 30 + + # SD dreambooth + LoRA：stable-diffusion-v1-5 + dummy image + 10 steps + # exit 0（loss 0.00695，17s）。overlay 关键： + # --report_to tensorboard（accelerate 拒 "none"） + # --center_crop 是 store_true flag，不传 =True + # --mixed_precision no（peft SD 例无 grad_scaler，NPU bf16 需 explicit） + # CI 资产（run 35303810367 缺 diffusers 崩 import）：profile peft_dreambooth + # 在 setup_peft 之上装 diffusers + tensorboard 并从共享缓存解析 + # ${SD_MODEL_PATH}：stable-diffusion-v1-5/stable-diffusion-v1-5，缓存未命中 + # 时由 scripts/hub_cache.py 下载；ci_dummy_image fixture = + # 2 张 256×256 PNG。 + + # LiLY：同 gralora 配方（npu-5 exit 0，loss 2.37） + - path: examples/lily_finetuning/lily_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model ${SFT_MODEL_PATH} + - --data_path ${TARGET_ROOT}/fixtures/ci_text_8_hf + - --num_epochs 1 + - --batch_size 1 + - --cutoff_len 256 + - --val_set_size 1 + - --learning_rate 1e-4 + - --output_dir ${CI_OUTPUT_DIR} + - --device npu:0 + - --eval_step 100 + - --save_step 100 + timeout_minutes: 30 + + # Peanut：同 gralora 配方（npu-5 exit 0，loss 2.37） + + # DeLoRA：同 DeFT 配方，exit 0 loss 1.86 + - path: examples/delora_finetuning/delora_finetuning.py + profile: peft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.10 + overlay_args: + - --base_model ${SFT_MODEL_PATH} + - --data_path ${TARGET_ROOT}/fixtures/ci_text_8_hf + - --num_epochs 1 + - --batch_size 1 + - --cutoff_len 256 + - --val_set_size 1 + - --learning_rate 1e-4 + - --output_dir ${CI_OUTPUT_DIR} + - --device npu:0 + - --eval_step 100 + - --save_step 100 + timeout_minutes: 30 + + # RandLoRA：同 DeFT 配方 + --use_lora flag，exit 0 loss 1.86 + +unsupported: + - examples/causal_language_modeling/peft_lora_clm_accelerate_ds_zero3_offload.py + - examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py + - examples/orthogonal_subspace_learning/osf_continual_learning.py + + # 语义检索 LoRA：sanity_test 调用 load_dataset("smangrul/amazon_esci", split="train[:1024]")。 + # Hub 数据集卡片记录的 train / validation 行数是 parquet 实际行数的两倍 + # （train 839306 vs 419653，validation 363402 vs 181701），datasets 5.0.1 + # 在校验阶段抛 NonMatchingSplitsSizesError。空缓存可复现，发生在训练开始之前， + # 与 NPU 无关。run 38015073648 在模型与 NPU 就绪后失败。 + - examples/feature_extraction/peft_lora_embedding_semantic_search.py + + # ===== bnb（bitsandbytes）量化 + # bnb 0.50.2 的 4-bit NF4 与 8-bit int8 在 NPU 上**功能可用**：走默认 + # CPU 后端（权重 CPU 内核量化 + npu 矩阵乘），不挂 transfer_to_npu 时 + # QLoRA 4bit/8bit 训练、Linear8bitLt、int8_vectorwise_quant/dequant + # 全通（npu-1/npu-2 双机复现）。为此 run_example.sh 加了 + # SKIP_TRANSFER_TO_NPU 白名单（fp4_finetuning/int8_training/ + # loftq_finetuning/KappaTune/arrow_multitask 目录跳过 transfer_to_npu， + # 保留 fp16 neutralizer + dataset shim），fp4 由此翻案成功。 + # 下方 8 条仍未翻，各自有一票否决的独立死因（非 bnb 后端问题）： + # - LoftQ 3 条（int8_correction/quantize_save_load/train_gsm8k）：即便 + # 绕过 shim，peft 源码 loftq_utils.py:70 compute_device="cuda"（无 npu + # 分支）在 torch +cpu 下 AssertionError，需源码 patch。 + # - KappaTune：gated Mixtral-8x7B + 零 CLI 双实验。 + # - arrow_phi3：纯推理/评估脚本，非训练 example。 + # - blip2：零 CLI + 硬编码 50 epoch（无 max_steps）+ 2.7b VLM 未 seed。 + # - whisper：数据 mozilla/common_voice_11_0 在 hf-mirror 不可达。 + - examples/KappaTune/experiments_kappatune_peft.py # gated mistralai/Mixtral-8x7B（46GB，需 token）+ 零 CLI 双实验 + - examples/arrow_multitask/arrow_phi3_mini.py # 纯推理/评估脚本（evaluate_on_multi_choice_batched + no_grad，无 trainer.train），非训练 example + - examples/int8_training/fine_tune_blip2_int8.py # 零 CLI + 硬编码 50 epoch 手动循环（无 max_steps），blip2-2.7b VLM 未 seed + - examples/int8_training/peft_adalora_whisper_large_training.py # 8bit+AdaLoRA 模型构造已实测可通，但默认数据 mozilla/common_voice_11_0 在 hf-mirror 不可达（DatasetNotFoundError） + - examples/int8_training/run_adalora_whisper_int8.sh + - examples/loftq_finetuning/int8_correction.py + - examples/loftq_finetuning/quantize_save_load.py + - examples/loftq_finetuning/train_gsm8k_llama.py + - examples/boft_controlnet/train_controlnet.py + - examples/boft_controlnet/train_controlnet.sh + - examples/boft_dreambooth/train_dreambooth.sh + - examples/hra_dreambooth/train_dreambooth.sh + - examples/sft/run_peft_fsdp_gptq.sh + - examples/frod_finetuning/frod_image_classification.py + - examples/frod_finetuning/frod_text_classification.py + # ShiRA（稀疏 mask LoRA）：sparse 算子族在 torch_npu 上无稳定路径。 + # torch_npu 2.9 在 torch.sparse_coo_tensor 构造处 INTERNAL ASSERT 崩； + # 2.12 能跑但 sparse backward kernel 丢梯度 value： + # SparseCooTensorWithDimsAndTensorsBackward0 返回的梯度元素数少于期望 + # （float16 实测 57327≠57344 = q/o_proj 896×896×r32；bf16 实测 + # 32767≠32768 = k/v_proj 896×128×r32），丢失数量随数据/层变化， + # flaky 不可靠——run 35575049729（2026-09-21）bf16 overlay 下第 9/9 + # 步 backward 仍崩（前 8 步正常），2026-09-20 npu-1 的 bf16 exit 0 + # 只是没踩中。曾以 2.12+bf16 overlay 上过 supported，现撤回；等 + # torch_npu 修 sparse backward 再翻案。 + - examples/shira_finetuning/shira_finetuning.py + - examples/qalora_finetuning/qalora_gptq_finetuning.py + - examples/eva_finetuning/eva_finetuning.py + - examples/eva_finetuning/eva_finetuning_multi_accelerator.py + - examples/boft_controlnet/eval.py + - examples/boft_controlnet/eval.sh + - examples/stable_diffusion/inc_flux_lora_hpu.py + - examples/lora_finetuning_transformer_engine/lora_finetuning_te.py + - examples/sft/run_unsloth_peft.sh + # xlora/xlora_inference_mistralrs.py：upstream README 标题 "X-LoRA examples" + 章节 + # `## xlora_inference_mistralrs.py` 把它文档化为 X-LoRA 的官方 example（不是 stage 子脚本）。 + # 实际是 mistralrs（Rust/CUDA 推理引擎）调用代码：model_id / xlora_model_id / order 三个 + # 关键参数硬编 `...` Ellipsis 占位，文件本身不能直接跑（README 也注明 "Runnable example" + # 链接指 mistralrs 自己仓 https://github.com/EricLBuehler/mistral.rs/blob/master/examples/python/xlora_zephyr.py）。 + # 无 NPU 后端，NPU 上无解。 + - examples/xlora/xlora_inference_mistralrs.py + # cartridge_self_study：train_distill.py 是 README 文档化的"Train cartridge"主入口，arxiv_train.py 是它的 arxiv-default 平行入口（from train_distill import ...），两者 NPU 锁死同根因（device choices=[cpu,mps,cuda,xpu] 无 npu）。synthesize.py / arxiv_synthesize.py 是 stage-1 数据合成（vLLM），无 trainer.train，归非 example 配套物（见下方"非 example 配套物"段） + - examples/cartridge_self_study/train_distill.py + - examples/cartridge_self_study/arxiv_train.py + + # ===== 非 example 配套物===== + # 以下是被训练脚本 import 的库模块 / 自测 / 预处理 / 推理·转换·合成工具，无 trainer.train() 训练闭环。 + + # --- sft/train.py：run_peft*.sh 共同调用的实现（hfArgumentParser + SFTTrainer），入口是 .sh --- + - examples/sft/train.py + + # --- utils 子包：被对应 train_*.py import 的库代码，无 __main__ 入口，不可独立执行 --- + - examples/boft_controlnet/utils/__init__.py + - examples/boft_controlnet/utils/args_loader.py + - examples/boft_controlnet/utils/dataset.py + - examples/boft_controlnet/utils/light_controlnet.py + - examples/boft_controlnet/utils/pipeline_controlnet.py + - examples/boft_controlnet/utils/tracemalloc.py + - examples/boft_controlnet/utils/unet_2d_condition.py + - examples/boft_dreambooth/utils/__init__.py + - examples/boft_dreambooth/utils/args_loader.py + - examples/boft_dreambooth/utils/dataset.py + - examples/boft_dreambooth/utils/tracemalloc.py + - examples/hra_dreambooth/utils/__init__.py + - examples/hra_dreambooth/utils/args_loader.py + - examples/hra_dreambooth/utils/dataset.py + - examples/hra_dreambooth/utils/tracemalloc.py + + # --- __init__.py 包标记（boft_* 子包 init），空模块 --- + - examples/boft_controlnet/__init__.py + - examples/boft_dreambooth/__init__.py + + # --- 顶层 helper 模块：训练脚本 import 的数据/工具库，无入口 --- + - examples/corda_finetuning/datautils.py + - examples/eva_finetuning/utils.py + - examples/orthogonal_subspace_learning/utils.py + - examples/sft/utils.py + + # --- 上游自测脚本：非训练入口。 + # test_adamss_quick.py 是 image_classification_adamss_asa.py 的 100 样本快速版（有 Trainer.train()， + # 但 :8 硬编码坏 sys.path、load_dataset("cifar10") 与已支持条目资产不一致，且与已支持条目重复，不翻案）； + # test_controlnet.{py,sh} 是 BOFT ControlNet 推理自测，依赖训练产物 checkpoint（model.safetensors/unet）+ SD2.1 --- + - examples/adamss_finetuning/test_adamss_quick.py + - examples/boft_controlnet/test_controlnet.py + - examples/boft_controlnet/test_controlnet.sh + + # --- 数据预处理脚本：CorDA/PiSSA 的 SVD 分解初始化（preprocess_corda / pissa init）， + # 无 trainer.train()，且需 device_map="auto" 加载 7B + 下载校准数据集； + # 对应训练脚本 corda/pissa_finetuning.py 已以 --corda_mode False / 退化 LoRA 绕开此步 --- + - examples/corda_finetuning/preprocess.py + - examples/pissa_finetuning/preprocess.py + + # --- 推理/转换/合成工具脚本：均非训练 recipe --- + # chat.py：打 localhost:8000 vLLM 服务的纯 HTTP 客户端（不加载模型）； + # synthesize.py / arxiv_synthesize.py：cartridge stage-1 数据合成（vLLM 分支无 NPU 后端，HF 分支非训练）； + # load_with_dora.py：硬编码 device_map={"":"cpu"} + 256GiB 内存的 CPU offload 演示（7B 模型）； + # convert_*.py ×3：SD LoRA 格式转换（kohya_ss↔peft、lycoris→peft，CPU 纯工具，required 输入 checkpoint/adapter） + - examples/bdlora_finetuning/chat.py + - examples/cartridge_self_study/arxiv_synthesize.py + - examples/cartridge_self_study/synthesize.py + - examples/ephemeral_gpu_offloading/load_with_dora.py + - examples/lora_dreambooth/convert_kohya_ss_sd_lora_to_peft.py + - examples/lora_dreambooth/convert_peft_sd_lora_to_kohya_ss.py + - examples/stable_diffusion/convert_sd_adapter_to_peft.py diff --git a/examples-guards/peft/fixtures/ci_alpaca_10/train.jsonl b/examples-guards/peft/fixtures/ci_alpaca_10/train.jsonl new file mode 100644 index 000000000..d6ac65a8d --- /dev/null +++ b/examples-guards/peft/fixtures/ci_alpaca_10/train.jsonl @@ -0,0 +1,10 @@ +{"instruction": "What is 2+2?", "input": "", "output": "4"} +{"instruction": "Capital of France?", "input": "", "output": "Paris"} +{"instruction": "Largest planet?", "input": "", "output": "Jupiter"} +{"instruction": "Boiling point of water?", "input": "", "output": "100C"} +{"instruction": "Author of Hamlet?", "input": "", "output": "Shakespeare"} +{"instruction": "Year of moon landing?", "input": "", "output": "1969"} +{"instruction": "Speed of light?", "input": "", "output": "3e8 m/s"} +{"instruction": "Heart chambers?", "input": "", "output": "Four"} +{"instruction": "Largest ocean?", "input": "", "output": "Pacific"} +{"instruction": "Smallest prime?", "input": "", "output": "2"} diff --git a/examples-guards/peft/fixtures/ci_alpaca_train_test/test.jsonl b/examples-guards/peft/fixtures/ci_alpaca_train_test/test.jsonl new file mode 100644 index 000000000..934f65e0e --- /dev/null +++ b/examples-guards/peft/fixtures/ci_alpaca_train_test/test.jsonl @@ -0,0 +1,2 @@ +{"instruction": "Largest ocean?", "input": "", "output": "Pacific"} +{"instruction": "Smallest prime?", "input": "", "output": "2"} diff --git a/examples-guards/peft/fixtures/ci_alpaca_train_test/train.jsonl b/examples-guards/peft/fixtures/ci_alpaca_train_test/train.jsonl new file mode 100644 index 000000000..f8af1bf02 --- /dev/null +++ b/examples-guards/peft/fixtures/ci_alpaca_train_test/train.jsonl @@ -0,0 +1,8 @@ +{"instruction": "What is 2+2?", "input": "", "output": "4"} +{"instruction": "Capital of France?", "input": "", "output": "Paris"} +{"instruction": "Largest planet?", "input": "", "output": "Jupiter"} +{"instruction": "Boiling point of water?", "input": "", "output": "100C"} +{"instruction": "Author of Hamlet?", "input": "", "output": "Shakespeare"} +{"instruction": "Year of moon landing?", "input": "", "output": "1969"} +{"instruction": "Speed of light?", "input": "", "output": "3e8 m/s"} +{"instruction": "Heart chambers?", "input": "", "output": "Four"} diff --git a/examples-guards/peft/fixtures/ci_corda_8/test.jsonl b/examples-guards/peft/fixtures/ci_corda_8/test.jsonl new file mode 100644 index 000000000..94117d75c --- /dev/null +++ b/examples-guards/peft/fixtures/ci_corda_8/test.jsonl @@ -0,0 +1,2 @@ +{"instruction": "What is 1+1?", "response": "2"} +{"instruction": "Color of grass?", "response": "Green"} diff --git a/examples-guards/peft/fixtures/ci_corda_8/train.jsonl b/examples-guards/peft/fixtures/ci_corda_8/train.jsonl new file mode 100644 index 000000000..e6f17593c --- /dev/null +++ b/examples-guards/peft/fixtures/ci_corda_8/train.jsonl @@ -0,0 +1,8 @@ +{"instruction": "What is 2+2?", "response": "4"} +{"instruction": "Capital of France?", "response": "Paris"} +{"instruction": "Largest planet?", "response": "Jupiter"} +{"instruction": "Boiling point of water?", "response": "100C"} +{"instruction": "Author of Hamlet?", "response": "Shakespeare"} +{"instruction": "Year of moon landing?", "response": "1969"} +{"instruction": "Speed of light?", "response": "3e8 m/s"} +{"instruction": "Heart chambers?", "response": "Four"} diff --git a/examples-guards/peft/fixtures/ci_dummy_image/dog_01.png b/examples-guards/peft/fixtures/ci_dummy_image/dog_01.png new file mode 100644 index 000000000..15a6f6311 Binary files /dev/null and b/examples-guards/peft/fixtures/ci_dummy_image/dog_01.png differ diff --git a/examples-guards/peft/fixtures/ci_dummy_image/dog_02.png b/examples-guards/peft/fixtures/ci_dummy_image/dog_02.png new file mode 100644 index 000000000..fd72184e6 Binary files /dev/null and b/examples-guards/peft/fixtures/ci_dummy_image/dog_02.png differ diff --git a/examples-guards/peft/fixtures/ci_query_response_8.jsonl b/examples-guards/peft/fixtures/ci_query_response_8.jsonl new file mode 100644 index 000000000..4bf09fdea --- /dev/null +++ b/examples-guards/peft/fixtures/ci_query_response_8.jsonl @@ -0,0 +1,8 @@ +{"query": "What is 2+2?", "response": "2 plus 2 equals 4."} +{"query": "What is the capital of France?", "response": "The capital of France is Paris."} +{"query": "Name the color of a clear daytime sky.", "response": "The sky is blue on a clear day."} +{"query": "How many legs does a spider have?", "response": "A spider has eight legs."} +{"query": "What is the largest planet?", "response": "Jupiter is the largest planet."} +{"query": "What do bees make?", "response": "Bees make honey."} +{"query": "Which animal is known as the king of the jungle?", "response": "The lion is known as the king of the jungle."} +{"query": "What is the boiling point of water?", "response": "Water boils at 100 degrees Celsius."} diff --git a/examples-guards/peft/fixtures/ci_sft_8.jsonl b/examples-guards/peft/fixtures/ci_sft_8.jsonl new file mode 100644 index 000000000..97197d7ac --- /dev/null +++ b/examples-guards/peft/fixtures/ci_sft_8.jsonl @@ -0,0 +1,8 @@ +{"messages": [{"role": "user", "content": "What is 1 plus 1?"}, {"role": "assistant", "content": "1 plus 1 equals 2."}]} +{"messages": [{"role": "user", "content": "What is the capital of France?"}, {"role": "assistant", "content": "The capital of France is Paris."}]} +{"messages": [{"role": "user", "content": "Name the color of a clear daytime sky."}, {"role": "assistant", "content": "The sky is blue on a clear day."}]} +{"messages": [{"role": "user", "content": "How many days are there in a week?"}, {"role": "assistant", "content": "There are seven days in a week."}]} +{"messages": [{"role": "user", "content": "What is 2 times 3?"}, {"role": "assistant", "content": "2 times 3 equals 6."}]} +{"messages": [{"role": "user", "content": "Which planet is closest to the Sun?"}, {"role": "assistant", "content": "Mercury is the planet closest to the Sun."}]} +{"messages": [{"role": "user", "content": "What sound does a cat make?"}, {"role": "assistant", "content": "A cat meows."}]} +{"messages": [{"role": "user", "content": "How many legs does a spider have?"}, {"role": "assistant", "content": "A spider has eight legs."}]} diff --git a/examples-guards/peft/fixtures/ci_supertuning_8.jsonl b/examples-guards/peft/fixtures/ci_supertuning_8.jsonl new file mode 100644 index 000000000..df6728192 --- /dev/null +++ b/examples-guards/peft/fixtures/ci_supertuning_8.jsonl @@ -0,0 +1,8 @@ +{"text": "### Human: Напиши функцию на языке swift, которая сортирует массив целых чисел, а затем выводит его на экран### Assistant: Вот функция, которая сортирует массив целых чисел и выводит его на экран:\n\n```swift\nfunc sortAndPrintArray(_ array: [Int]) {\n // Создаем копию массива, чтобы не изменять исходный\n var sortedArray = array\n // Сортируем массив по возрастанию\n sortedArray.sort()\n // Выводим отсортированный массив на экран\n print(sortedArray)\n}\n```\n\n\nТы можешь проверить работу функции, вызвав ее с любым массивом целых чисел, например:\n\n```swift\nsortAndPrintArray([5, 2, 7, 9, -3])\n```\n\n"} +{"text": "### Human: Inventa un monstruo altamente complejo, descríbelo y enumera sus ventajas y desventajas ante el ambiente en el que se supone que se debe de encontrar.### Assistant: Aquí te presento mi creación de un monstruo altamente complejo:\n\nNombre: Xalakthrax\n\n"} +{"text": "### Human: Escribe un codigo para ESP32 que indique por el monitor serial la direccion de los sensores de temperatura dallas 18b20 junto con su temperatura.### Assistant: Aquí te proporciono un código de ejemplo para ESP32 que busca todos los dispositivos de temperatura Dallas 18B20 en el bus OneWire y muestra su dirección y temperatura en el monitor serial.\n\nscss\n\n#include \n#include \n\n// Define el pin al que está conectado el bus OneWire\n#define ONE_WIRE_BUS 5\n\n// Instancia OneWire\nOneWire oneWire(ONE_WIRE_BUS);\n\n// Instancia DallasTemperature\n"} +{"text": "### Human: What do you think about ChatGPT?### Assistant: As an open source alternative to ChatGPT, I do not have personal opinions. However, I can provide objective information about ChatGPT's capabilities and limitations based on its architecture and training data.\n\n"} +{"text": "### Human: Can you please provide me the names of the two players in the atomic bomb game (in go)? \n\nIf you can get me the referee's name as well, that's even better!### Assistant: Yes. Given that you're requesting information on the game of Go played in 1945, the two players were Hashimoto Utaro and Iwamoto Kaoru, who was vying for the title. The referee for that game was Segoe Kensaku."} +{"text": "### Human: Implementa el algoritmo `bubble sort` en C.### Assistant: Aquí te presento una implementación del algoritmo \"bubble sort\" en C:\n#include \n\nvoid bubbleSort(int arr[], int n) {\n int i, j;\n for (i = 0; i < n-1; i++) {\n for (j = 0; j < n-i-1; j++) {\n if (arr[j] > arr[j+1]) {\n int temp = arr[j];\n arr[j] = arr[j+1];\n arr[j+1] = temp;\n }\n }\n }\n}\n\nint main() {\n int arr[] = {64, 34, 25, 12, 22, 11, 90};\n int n = sizeof(arr)/sizeof(arr[0]);\n printf(\"Array original: \\n\");\n for (int i=0; i < n; i++)\n printf(\"%d \", arr[i]);\n printf(\"\\n\");\n"} +{"text": "### Human: How would the Future of AI in 10 Years look?### Assistant: Predicting the future is always a challenging task, but here are some possible ways that AI could evolve over the next 10 years:\n\nContinued advancements in deep learning: Deep learning has been one of the main drivers of recent AI breakthroughs, and we can expect continued advancements in this area. This may include improvements to existing algorithms, as well as the development of new architectures that are better suited to specific types of data and tasks.\n\n"} +{"text": "### Human: Traducime la letra de \"The Real Slim Shady\" del inglés al español y después tranformalo en un poema escrito en verso### Assistant: Aquí está la letra traducida de \"The Real Slim Shady\" de Eminem al español:\n\n¿Me puedes sentir cuando te muevo?\nMe dijo el doctor que tengo TDAH\nTengo problemas en mi cerebro\nA veces me siento como si fuera un loco\nMi única carrera es correr a la mierda\nSiempre estoy haciendo travesuras\n¿Cómo puede ser esto?\nEl policía no me detiene\nConoce el sexo oral así que yo lo hago\nMezclo el valium con la Fanta y agito\nMi vida amorosa es un gran círculo\n"} diff --git a/examples-guards/peft/fixtures/ci_text_8_hf/test.jsonl b/examples-guards/peft/fixtures/ci_text_8_hf/test.jsonl new file mode 100644 index 000000000..4f5b4407f --- /dev/null +++ b/examples-guards/peft/fixtures/ci_text_8_hf/test.jsonl @@ -0,0 +1,2 @@ +{"text": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nGive one tip for staying focused while studying.\n\n### Response:\nWork in short timed sessions with brief breaks, such as 25 minutes of study followed by a 5-minute rest."} +{"text": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nWhat causes seasons on Earth?\n\n### Response:\nEarth's tilted axis changes how directly sunlight hits each hemisphere as it orbits the Sun."} diff --git a/examples-guards/peft/fixtures/ci_text_8_hf/train.jsonl b/examples-guards/peft/fixtures/ci_text_8_hf/train.jsonl new file mode 100644 index 000000000..621a4bf02 --- /dev/null +++ b/examples-guards/peft/fixtures/ci_text_8_hf/train.jsonl @@ -0,0 +1,8 @@ +{"text": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nSummarize the water cycle in two sentences.\n\n### Response:\nWater evaporates from oceans and lakes, rises, cools, and condenses into clouds. It then falls back to earth as precipitation and returns to bodies of water."} +{"text": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nName three primary colors and give an example object for each.\n\n### Response:\nRed, like a fire truck; blue, like the sky on a clear day; yellow, like a sunflower."} +{"text": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nExplain why the sky appears blue during the day.\n\n### Response:\nAir molecules scatter shorter wavelengths of sunlight more strongly than longer ones, and the scattered blue light reaches our eyes from all directions."} +{"text": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nConvert 3 kilometers to meters.\n\n### Response:\n3 kilometers equals 3000 meters, since one kilometer is 1000 meters."} +{"text": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nList two health benefits of drinking enough water.\n\n### Response:\nStaying hydrated helps regulate body temperature and supports nutrient transport to cells."} +{"text": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nWrite a haiku about autumn leaves.\n\n### Response:\nRed leaves drift and fall\nWhispering summer goodbye\nCrisp air waits beneath"} +{"text": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nWhat is the difference between a CPU and a GPU?\n\n### Response:\nA CPU has few cores optimized for sequential general-purpose tasks, while a GPU has thousands of cores designed for parallel workloads."} +{"text": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\nSuggest a simple way to remember the order of the planets.\n\n### Response:\nUse the mnemonic: My Very Educated Mother Just Served Us Noodles, for Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune."} diff --git a/examples-guards/peft/fixtures/financial_phrasebank/financial_phrase_bank_train.jsonl b/examples-guards/peft/fixtures/financial_phrasebank/financial_phrase_bank_train.jsonl new file mode 100644 index 000000000..37e55aba5 --- /dev/null +++ b/examples-guards/peft/fixtures/financial_phrasebank/financial_phrase_bank_train.jsonl @@ -0,0 +1,8 @@ +{"sentence": "The company posted record quarterly profits, beating expectations.", "label": "positive"} +{"sentence": "Revenue declined for the third straight quarter amid weak demand.", "label": "negative"} +{"sentence": "Management announced a new share buyback program worth two billion.", "label": "positive"} +{"sentence": "The firm faces a lawsuit over alleged accounting irregularities.", "label": "negative"} +{"sentence": "Sales in the European market grew strongly despite currency headwinds.", "label": "positive"} +{"sentence": "The startup announced layoffs affecting twenty percent of staff.", "label": "negative"} +{"sentence": "Investors cheered the dividend increase and upgraded guidance.", "label": "positive"} +{"sentence": "The merger cleared regulatory approval, removing a key uncertainty.", "label": "positive"} diff --git a/examples-guards/peft/fixtures/financial_phrasebank/financial_phrase_bank_val.jsonl b/examples-guards/peft/fixtures/financial_phrasebank/financial_phrase_bank_val.jsonl new file mode 100644 index 000000000..b61505af1 --- /dev/null +++ b/examples-guards/peft/fixtures/financial_phrasebank/financial_phrase_bank_val.jsonl @@ -0,0 +1,4 @@ +{"sentence": "Supply chain disruptions continued to weigh on margins this quarter.", "label": "negative"} +{"sentence": "The new product launch received strong pre-orders from customers.", "label": "positive"} +{"sentence": "Analysts downgraded the stock citing slowing growth momentum.", "label": "negative"} +{"sentence": "Cash reserves remain healthy despite the recent acquisition spree.", "label": "positive"} diff --git a/examples-guards/peft/scripts/hub_cache.py b/examples-guards/peft/scripts/hub_cache.py new file mode 100644 index 000000000..4e7f0d531 --- /dev/null +++ b/examples-guards/peft/scripts/hub_cache.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 +"""Resolve peft's overlay model paths through the shared HF hub cache. + +setup_example.sh passes the env var names its profile needs, for example +`hub_cache.py SFT_MODEL_PATH ROBERTA_BASE_PATH`. For each one this +appends VAR= to $GITHUB_ENV for the manifest's +overlay_args: + +- a cached snapshot that holds every required file and no corrupt + safetensors is used as is, without network access; +- otherwise the asset is downloaded, HuggingFace first and ModelScope + as the fallback, and published into the cache, so the next run hits + the cache. + +The cache root is HF_HOME (default ~/.cache/huggingface), which on the +self-hosted runners lives on the runner pool's persistent NFS volume, +shared by every concurrent matrix leg on every pod. The hub's own +.locks do not exclude across pods there: in run 37764199490 two legs +fetching SD v1.5 appended to one blob and left it larger than the whole +file. So nothing is downloaded into the cache in place. A job +downloads into its own staging dir on the same volume, checks it, and +renames the files into the snapshot, which readers then see whole or +not at all. A mkdir lock per asset lets one leg download while the +others wait for it; correctness does not depend on the lock. +""" +from __future__ import annotations + +import os +import shutil +import sys +import tempfile +import threading +import time +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path + +from huggingface_hub import HfApi, snapshot_download +from modelscope import snapshot_download as ms_snapshot_download +from safetensors import SafetensorError, safe_open + +HUB_ROOT = Path(os.environ.get( + 'HF_HOME', os.path.expanduser('~/.cache/huggingface'))) / 'hub' +# Siblings of HUB_ROOT on the same volume, so a rename from staging into +# the snapshot is atomic, kept out of the hub's own cache layout. +STAGING_DIR = 'guard-staging' +LOCKS_DIR = 'guard-locks' +# ModelScope's own snapshot bookkeeping, not repository content. +MODELSCOPE_METADATA = {'.mdl', '.msc', '.mv'} +HEARTBEAT_SECONDS = 30 +# A lock this long without a heartbeat belongs to a killed job. +LOCK_STALE_SECONDS = 300 +LOCK_POLL_SECONDS = 10 +# Past this, download alongside the holder rather than spend the job's +# timeout waiting. +LOCK_WAIT_SECONDS = 900 +# Longer than any job lives: the job that made it was killed. +STAGING_STALE_SECONDS = 24 * 3600 + + +@dataclass(frozen=True) +class Asset: + hf_id: str + ms_id: str + # Files to fetch; skips the duplicate bin / h5 / onnx formats. + patterns: tuple[str, ...] + # Globs that must each match a snapshot file for a cache hit, so an + # interrupted download is refetched instead of used. + required: tuple[str, ...] + + +TEXT_MODEL = ('*.json', '*.txt', '*.safetensors') +TEXT_REQUIRED = ('config.json', '*.safetensors') + +ASSETS = { + 'SFT_MODEL_PATH': Asset( + 'Qwen/Qwen2.5-0.5B', 'Qwen/Qwen2.5-0.5B', TEXT_MODEL, TEXT_REQUIRED), + 'ROBERTA_BASE_PATH': Asset( + 'roberta-base', 'AI-ModelScope/roberta-base', TEXT_MODEL, TEXT_REQUIRED), + 'BERT_BASE_UNCASED_PATH': Asset( + 'bert-base-uncased', 'AI-ModelScope/bert-base-uncased', TEXT_MODEL, + TEXT_REQUIRED), + 'SD_MODEL_PATH': Asset( + 'stable-diffusion-v1-5/stable-diffusion-v1-5', + 'AI-ModelScope/stable-diffusion-v1-5', + ('model_index.json', '*/*.json', '*/model.safetensors', + '*/diffusion_pytorch_model.safetensors', 'tokenizer/*', + 'feature_extractor/*'), + ('model_index.json', 'unet/diffusion_pytorch_model.safetensors', + 'vae/diffusion_pytorch_model.safetensors', + 'text_encoder/model.safetensors')), +} + + +def repo_dir(asset: Asset) -> Path: + return HUB_ROOT / f"models--{asset.hf_id.replace('/', '--')}" + + +def safetensors_header_ok(path: Path) -> bool | None: + """True if the header parses, False if the file is readable but + malformed, None if it cannot be read at all. + + A read failure on the shared NFS volume is not evidence of + corruption, so None must not get a shard deleted. + """ + try: + with safe_open(str(path), framework='numpy') as f: + list(f.keys()) + except SafetensorError: + return False + except Exception: # noqa: BLE001 - OSError and the like: cannot judge + return None + return True + + +def corrupt_shards(root: Path) -> list[Path]: + """Safetensors under root whose header or size is inconsistent.""" + return [p for p in sorted(root.rglob('*.safetensors')) + if safetensors_header_ok(p) is False] + + +def cached_snapshot(asset: Asset) -> Path | None: + """The snapshot refs/main points at, if it holds every required file + and none of its safetensors is corrupt.""" + refs = repo_dir(asset) / 'refs' / 'main' + if not refs.is_file(): + return None + snap = repo_dir(asset) / 'snapshots' / refs.read_text().strip() + if not snap.is_dir(): + return None + if not all(any(p.is_file() for p in snap.glob(g)) for g in asset.required): + return None + corrupt = corrupt_shards(snap) + if corrupt: + print(f'[corrupt] {asset.hf_id}: ' + f'{[p.relative_to(snap).as_posix() for p in corrupt]}', flush=True) + return None + return snap + + +def purge_corrupt(asset: Asset) -> None: + """Delete corrupt shards and the blobs behind them. + + snapshot_download copies a blob the cache already holds into the + download target without rechecking it, so a damaged blob is handed + forward on every retry until it is gone. + """ + snapshots = repo_dir(asset) / 'snapshots' + if not snapshots.is_dir(): + return + for snap in sorted(snapshots.iterdir()): + for shard in corrupt_shards(snap): + blob = Path(os.path.realpath(shard)) if shard.is_symlink() else None + print(f'[purge] {asset.hf_id}: {shard.relative_to(snap).as_posix()}', + flush=True) + shard.unlink(missing_ok=True) + if blob is not None: + blob.unlink(missing_ok=True) + + +def revision(asset: Asset) -> str: + """The hub commit to file the snapshot under.""" + try: + return HfApi().model_info(asset.hf_id).sha + except Exception: # noqa: BLE001 - HuggingFace may be the reason we fall back + refs = repo_dir(asset) / 'refs' / 'main' + return refs.read_text().strip() if refs.is_file() else 'modelscope' + + +def from_huggingface(asset: Asset, dest: Path) -> Path: + snapshot_download( + repo_id=asset.hf_id, allow_patterns=list(asset.patterns), + local_dir=str(dest)) + return dest + + +def from_modelscope(asset: Asset, dest: Path) -> Path: + # snapshot_download does not create its cache root: it then fails + # mid-transfer. + dest.mkdir(parents=True, exist_ok=True) + return Path(ms_snapshot_download( + asset.ms_id, cache_dir=str(dest), + allow_patterns=list(asset.patterns))) + + +def checked(src: Path) -> Path: + corrupt = corrupt_shards(src) + if corrupt: + raise RuntimeError('corrupt safetensors downloaded: ' + f'{[p.relative_to(src).as_posix() for p in corrupt]}') + return src + + +def intact(dest: Path, staged: Path) -> bool: + """Whether dest already holds staged's content, by size and header.""" + try: + if dest.stat().st_size != staged.stat().st_size: + return False + except FileNotFoundError: + return False + return dest.suffix != '.safetensors' or safetensors_header_ok(dest) is not False + + +def publish(asset: Asset, src: Path, sha: str) -> Path: + """Rename src's files into snapshots/, then point refs/main at it. + + Intact files are left in place: another job may be reading them. + """ + snap = repo_dir(asset) / 'snapshots' / sha + for item in sorted(src.rglob('*')): + rel = item.relative_to(src) + # .cache is huggingface_hub's local_dir download bookkeeping. + if (not item.is_file() or rel.parts[0] == '.cache' + or rel.as_posix() in MODELSCOPE_METADATA): + continue + dest = snap / rel + if intact(dest, item): + continue + dest.parent.mkdir(parents=True, exist_ok=True) + os.replace(item, dest) + refs = repo_dir(asset) / 'refs' + refs.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + 'w', dir=refs, prefix='.main-', delete=False) as fh: + # No trailing newline: the hub compares this string with the + # snapshot folder name without stripping it. + fh.write(sha) + os.replace(fh.name, refs / 'main') + return snap + + +def fetch(asset: Asset) -> Path: + """Download into a staging dir of this job, check it, publish it.""" + sha = revision(asset) + staging_root = HUB_ROOT.parent / STAGING_DIR + staging_root.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp( + prefix=f'{repo_dir(asset).name}-', dir=staging_root)) + try: + print(f'[download] {asset.hf_id} <- HuggingFace', flush=True) + try: + src = checked(from_huggingface(asset, staging / 'hf')) + except Exception as exc: # noqa: BLE001 - falls back below + print(f'[fallback] {asset.hf_id}: HuggingFace failed ' + f'({type(exc).__name__}: {exc}); trying ModelScope {asset.ms_id}', + flush=True) + src = checked(from_modelscope(asset, staging / 'ms')) + snap = publish(asset, src, sha) + finally: + shutil.rmtree(staging, ignore_errors=True) + if cached_snapshot(asset) != snap: + raise RuntimeError( + f'{asset.hf_id}: downloaded snapshot {snap} lacks {asset.required}') + print(f'[cached] {asset.hf_id} -> {snap}', flush=True) + return snap + + +def heartbeat(lock: Path, stop: threading.Event) -> None: + while not stop.wait(HEARTBEAT_SECONDS): + try: + os.utime(lock) + except OSError: + return + + +def break_if_stale(lock: Path) -> None: + try: + idle = time.time() - lock.stat().st_mtime + except FileNotFoundError: + return + if idle < LOCK_STALE_SECONDS: + return + print(f'[lock] {lock.name}: no heartbeat for {idle:.0f}s, breaking it', + flush=True) + # Rename first: of several waiters breaking it, only one succeeds. + stale = lock.with_name(f'{lock.name}.stale-{uuid.uuid4().hex}') + try: + os.rename(lock, stale) + except FileNotFoundError: + return + shutil.rmtree(stale, ignore_errors=True) + + +@contextmanager +def download_lock(asset: Asset) -> Iterator[None]: + """Hold the asset's lock for the body; after LOCK_WAIT_SECONDS of + waiting, run the body without it.""" + locks = HUB_ROOT.parent / LOCKS_DIR + locks.mkdir(parents=True, exist_ok=True) + lock = locks / repo_dir(asset).name + deadline = time.monotonic() + LOCK_WAIT_SECONDS + waiting = False + while True: + try: + lock.mkdir() + break + except FileExistsError: + pass + if time.monotonic() >= deadline: + print(f'[lock] {asset.hf_id}: still held after {LOCK_WAIT_SECONDS}s, ' + 'downloading without it', flush=True) + yield + return + if not waiting: + print(f'[lock] {asset.hf_id}: another job is downloading it, waiting', + flush=True) + waiting = True + break_if_stale(lock) + time.sleep(LOCK_POLL_SECONDS) + stop = threading.Event() + beat = threading.Thread(target=heartbeat, args=(lock, stop), daemon=True) + beat.start() + try: + yield + finally: + stop.set() + beat.join() + shutil.rmtree(lock, ignore_errors=True) + + +def resolve(var: str) -> Path: + asset = ASSETS[var] + snap = cached_snapshot(asset) + if snap is None: + with download_lock(asset): + # The previous holder may have published it meanwhile. + snap = cached_snapshot(asset) + if snap is None: + purge_corrupt(asset) + return fetch(asset) + print(f'[cache] {asset.hf_id} -> {snap}', flush=True) + return snap + + +def sweep_staging() -> None: + """Remove staging dirs that killed jobs left behind.""" + staging_root = HUB_ROOT.parent / STAGING_DIR + if not staging_root.is_dir(): + return + cutoff = time.time() - STAGING_STALE_SECONDS + for entry in staging_root.iterdir(): + try: + if entry.stat().st_mtime < cutoff: + shutil.rmtree(entry, ignore_errors=True) + except OSError: + continue + + +def main(argv: list[str]) -> int: + unknown = [var for var in argv if var not in ASSETS] + if not argv or unknown: + print(f'usage: hub_cache.py VAR... (known: {sorted(ASSETS)}; ' + f'unknown: {unknown})', file=sys.stderr) + return 2 + sweep_staging() + for var in argv: + try: + snap = resolve(var) + except Exception as exc: # noqa: BLE001 - reported as guard noise + print(f'{ASSETS[var].hf_id}: not in the shared cache and neither ' + f'HuggingFace nor ModelScope delivered it ({exc}) - guard ' + 'noise, not an example failure', file=sys.stderr) + return 1 + with open(os.environ['GITHUB_ENV'], 'a', encoding='utf-8') as fh: + fh.write(f'{var}={snap}\n') + print(f'{var}={snap}', flush=True) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main(sys.argv[1:])) diff --git a/examples-guards/peft/scripts/run_example.sh b/examples-guards/peft/scripts/run_example.sh new file mode 100755 index 000000000..fbe5a8587 --- /dev/null +++ b/examples-guards/peft/scripts/run_example.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash +# Run one peft example from a CI working copy of the target tree. +# $1 is the manifest entry path. EXEC, when set, names the launchable +# file relative to the target root; otherwise path itself must be a +# launchable file. Overlay CLI args come from OVERLAY_ARGS (JSON array, +# possibly []). Shell examples that do not pass "$@" get it attached in +# this working copy only (last command line). Never git add/commit/push. +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +EXAMPLE_REL="$1" +TARGET_ROOT="${TARGET_ROOT:?TARGET_ROOT is required}" +CI_OUTPUT_DIR="${CI_OUTPUT_DIR:?CI_OUTPUT_DIR is required}" + +EXAMPLE_PATH="$TARGET_ROOT/$EXAMPLE_REL" +[[ -e "$EXAMPLE_PATH" ]] || { echo "example not found: $EXAMPLE_PATH" >&2; exit 1; } + +# Resolve the launchable file: EXEC (relative to the target root) when +# set, otherwise path itself. +if [[ -n "${EXEC:-}" ]]; then + LAUNCH_PATH="$TARGET_ROOT/$EXEC" +else + LAUNCH_PATH="$EXAMPLE_PATH" +fi +if [[ ! -f "$LAUNCH_PATH" ]]; then + echo "launchable file not found: $LAUNCH_PATH (directory examples need an exec field)" >&2 + exit 1 +fi + +mkdir -p "$CI_OUTPUT_DIR" + +if command -v python3 >/dev/null 2>&1; then + PYTHON=python3 +else + PYTHON=python +fi + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +# bnb 0.50.2 的 dequantize 函数会触发 torch 2.12 dynamo 反复重编译 +# （73s/step → 3s/step）；其余条目不依赖 dynamo 编译路径，全局关闭 +# 无副作用。 +export TORCHDYNAMO_DISABLE=1 +# Hub ids the examples hardcode download on a cache miss; same endpoint +# as setup_example.sh (the engine's hf-mirror default cannot serve +# namespace-less ids such as t5-base). +export HF_ENDPOINT=https://huggingface.co +python -c "import torch, torch_npu; print('NPU available:', torch.npu.is_available(), 'devices:', torch.npu.device_count())" + +expand_overlay() { + # The workflow serializes manifest.overlay_args as JSON. Expand each + # item with shell quoting intact, then allow CI paths such as + # ${CI_OUTPUT_DIR} / ${SFT_MODEL_PATH} to resolve only in this job's + # environment. + "$PYTHON" - <<'PY' +import json +import os +import shlex + +raw = os.environ.get('OVERLAY_ARGS', '').strip() +if not raw or raw in ('null', '""'): + raise SystemExit(0) +try: + items = json.loads(raw) +except json.JSONDecodeError as exc: + raise SystemExit(f'OVERLAY_ARGS is not valid JSON: {exc}') from exc +if items in (None, ''): + raise SystemExit(0) +if not isinstance(items, list): + raise SystemExit( + f'OVERLAY_ARGS must be a JSON array, got {type(items).__name__}') +tokens = [] +for item in items: + if not isinstance(item, str) or not item.strip(): + raise SystemExit('OVERLAY_ARGS items must be non-empty strings') + tokens.extend(shlex.split(os.path.expandvars(item), posix=True)) +print(' '.join(shlex.quote(token) for token in tokens)) +PY +} + +eval "EXTRA_ARGS=( $(expand_overlay) )" + +echo "running $LAUNCH_PATH with ${#EXTRA_ARGS[@]} overlay args" +if ((${#EXTRA_ARGS[@]})); then + printf 'overlay arg: %q\n' "${EXTRA_ARGS[@]}" +fi + +ensure_passthrough() { + # Shell examples that already forward "$@" need no patch; otherwise + # attach it in this CI working copy only, on the last non-comment + # line (the tail of the example's main command). Python entry points + # take the overlay args directly on their own command line. + local script="$1" + [[ "$script" == *.sh ]] || return 0 + if grep -qE '"\$@"' "$script"; then + echo "example already has \"\$@\"; skipping patch" + return + fi + "$PYTHON" - "$script" <<'PY' +from pathlib import Path +import sys + +path = Path(sys.argv[1]) +lines = path.read_text(encoding='utf-8').splitlines(keepends=True) +for i in range(len(lines) - 1, -1, -1): + stripped = lines[i].strip() + if stripped and not stripped.startswith('#'): + raw = lines[i] + newline = '' + if raw.endswith('\r\n'): + newline = '\r\n' + raw = raw[:-2] + elif raw.endswith('\n'): + newline = '\n' + raw = raw[:-1] + lines[i] = raw.rstrip() + ' "$@"' + newline + path.write_text(''.join(lines), encoding='utf-8') + print(f'patched {path} to pass "$@" on last command line') + raise SystemExit(0) +raise SystemExit(f'{path}: cannot find a command line to attach "$@"') +PY +} + +ensure_passthrough "$LAUNCH_PATH" + + +# One sitecustomize.py, four patches, injected at interpreter startup: +# 1. CUDA->NPU (transfer_to_npu): most peft examples hardcode +# device="cuda"; torch_npu's transfer_to_npu maps torch.cuda onto npu +# (also covers `torch.device("cuda" if torch.cuda.is_available() else +# "cpu")` device-branch examples such as lora_ga). NOT applied to the +# bitsandbytes examples (see SKIP_TRANSFER_TO_NPU below). +# 2. Dataset: the sft example calls load_dataset(path) with a local +# .jsonl fixture and reads BOTH "train" and "test" splits +# (splits="train,test"); datasets 3.x cannot infer a builder from a +# single file path - rewrite local data files to +# load_dataset("", data_files={"train": path, "test": path}). +# 3. fp16 neutralizer: several finetuning scripts hardcode +# TrainingArguments(fp16=True, ...). accelerate 1.15's fp16 chain +# (optimizer.step(grad_scaler=scaler)) is incompatible with torch_npu +# amp.GradScaler -> TypeError "Adam.step got unexpected keyword +# argument grad_scaler". No supported entry relies on fp16=True, so +# forcing fp16=False globally is a no-op for every other entry. +# 4. wandb neutralizer: boft_dreambooth hardcodes wandb_init +# mode="online"; force wandb.init mode="disabled" so it becomes +# wandb's own no-op run without an API key. +prepare_shims() { + local shim_dir="$GITHUB_WORKSPACE/ci_patch" + mkdir -p "$shim_dir" + local transfer_line="from torch_npu.contrib import transfer_to_npu # noqa: F401" + if [[ -n "${SKIP_TRANSFER_TO_NPU:-}" ]]; then + transfer_line="# transfer_to_npu skipped for bitsandbytes example" + fi + cat > "$shim_dir/sitecustomize.py" <" >&2 + exit 2 +fi + +PROFILE="$1" + +CLUSTER_PIP_HOST=cache-service.nginx-pypi-cache.svc.cluster.local +export CLUSTER_PIP_INDEX="http://${CLUSTER_PIP_HOST}/pypi/simple" +ASCEND_PIP_INDEX=https://repo.huaweicloud.com/ascend/repos/pypi +ALIYUN_PIP_INDEX=https://mirrors.aliyun.com/pypi/simple/ + +pip_ascend() { + python -m pip install --extra-index-url "$ASCEND_PIP_INDEX" "$@" +} + +select_pip_index() { + # Runners live in mainland China: prefer the cluster pip cache, fall + # back to the Aliyun mirror. The ascend index stays available via + # PIP_EXTRA_INDEX_URL (set by the engine) for torch_npu wheels. + if python -c " +import os +import urllib.error +import urllib.request +try: + urllib.request.urlopen(os.environ['CLUSTER_PIP_INDEX'], timeout=3) +except urllib.error.HTTPError: + pass +" 2>/dev/null; then + export PIP_INDEX_URL="$CLUSTER_PIP_INDEX" + export PIP_TRUSTED_HOST="$CLUSTER_PIP_HOST" + else + export PIP_INDEX_URL="$ALIYUN_PIP_INDEX" + unset PIP_TRUSTED_HOST + fi + echo "pip index: $PIP_INDEX_URL" +} + +torch_stack_for_profile() { + # Two coexisting torch stacks (2026-09-21), selected per profile: + # - default (peft / peft_dreambooth): torch 2.12.0 + torch_npu 2.12.0, + # the post-2026-09-20 verified line for single-card entries. The + # 2.12 upgrade was originally made for the sparse-COO tuner shira + # (torch_npu 2.9 crashes at torch.sparse_coo_tensor construction + # plus dense+=sparse); shira was pulled back to unsupported on + # 2026-09-21 (torch_npu 2.12's sparse backward still loses gradient + # values under bf16, run 35575049729 — see examples_manifest.yaml), + # but the 2.12 line stays as default for the remaining entries. + # - peft_29 / peft_ds (the 4 multi-card sft entries): torch 2.9.0 + + # torch_npu 2.9.0.post2, the pre-2026-09-20 verified stack. torch + # 2.12's c10d broadcast() computes sm90_or_more via + # torch.cuda.get_device_capability(tensor.device)[0], and torch_npu + # patches Tensor.is_cuda = Tensor.is_npu while its + # get_device_capability shim returns None -> None[0] TypeError. + # That kills FSDP checkpoint save (dist_cp gather_object -> + # broadcast_object_list) and DeepSpeed ZeRO zero-init param + # broadcast (run 35502546409). Upstream torch_npu bug; pin these + # profiles back to 2.9 until fixed. + case "$1" in + peft_29|peft_ds) printf '2.9.0 2.9.0.post2' ;; + *) printf '2.12.0 2.12.0' ;; + esac +} + +ensure_torch_stack() { + local torch_ver npu_ver + read -r torch_ver npu_ver <<< "$(torch_stack_for_profile "$PROFILE")" + if python -c " +import torch, torch_npu +raise SystemExit( + 0 if torch.__version__.startswith('$torch_ver') + and torch_npu.__version__.startswith('$npu_ver') else 1) +"; then + echo "reusing image torch stack ($(python -c 'import torch; print(torch.__version__)'))" + return + fi + if [[ "$torch_ver" == "2.9.0" ]]; then + echo "installing torch==$torch_ver torch_npu==$npu_ver (pre-2.12 verified stack)" + pip_ascend "torch==${torch_ver}" "torch_npu==${npu_ver}" + return + fi + # torch 2.12.0 + torch_npu 2.12.0 + CANN 9.1.0 (upgraded 2026-09-20 + # from 2.9.0 to unblock the sparse-COO tuners shira, see above). + # + # torch==2.12.0 is NOT installed via `pip_ascend`: aliyun (and the + # cluster pip cache, both PyPI mirrors) only host the CUDA torch + # wheel, whose METADATA declares `Requires-Dist: cuda-toolkit`, which + # conflicts with constraints-npu.txt `cuda-toolkit<0`. The CPU + # variant is `torch==2.12.0+cpu` (PEP 440 local label) published ONLY + # at https://download.pytorch.org/whl/cpu/ - so fetch it by direct + # URL. We compute the cp tag at runtime because the manifest mixes + # py3.10 (30 entries) and py3.12 (15 entries) images. + echo "installing torch==2.12.0+cpu (direct URL) + torch_npu==2.12.0" + CP_ABI=$(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')") + python -m pip install --no-deps \ + "https://download.pytorch.org/whl/cpu/torch-2.12.0%2Bcpu-${CP_ABI}-${CP_ABI}-manylinux_2_28_aarch64.whl" + # torch's pure-Python deps (filelock / typing-extensions / sympy / + # networkx / jinja2 / fsspec) from aliyun so `import torch` succeeds + # (the +cpu wheel does not pull them; none declare cuda-toolkit). + python -m pip install -i "$ALIYUN_PIP_INDEX" \ + 'filelock' 'typing-extensions>=4.10.0' 'setuptools<82' \ + 'sympy>=1.13.3' 'networkx>=2.5.1' 'jinja2' 'fsspec>=0.8.5' + pip_ascend torch_npu==2.12.0 +} + +install_cpu_torchvision() { + # torchvision matching torch 2.12 is 0.27.0; like torch it ships as a + # +cpu direct-download wheel only (PyPI linux wheels link libcudart.so + # and are cuda-toolkit-gated). Only the adamss image example needs it. + # pillow is torchvision's image-codec dep (numpy already comes from + # scikit-learn/evaluate); install it separately so its deps resolve. + local cp_abi + cp_abi=$(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')") + python -m pip install --no-deps \ + "https://download.pytorch.org/whl/cpu/torchvision-0.27.0%2Bcpu-${cp_abi}-${cp_abi}-manylinux_2_28_aarch64.whl" + python -m pip install pillow +} + +resolve_hub_paths() { + # $@ = env var names from scripts/hub_cache.py ASSETS. Each resolves to + # a snapshot in the shared HF hub cache, downloaded on a miss, and is + # appended to GITHUB_ENV for manifest overlay_args. + python "$HERE/hub_cache.py" "$@" +} + +# Copy CI fixture data into the target root so that example scripts can +# load them via a local path under $TARGET_ROOT/fixtures/ (same +# decoupling from the workflows-checkout subtree as trl). +# +# Both flavors are supported, all top-level entries in $FIXTURE_DIR: +# *.jsonl → single-file fixtures (e.g. ci_sft_8.jsonl) +# */ → directory fixtures (e.g. ci_alpaca_10/ with train.jsonl, +# ci_corda_8/ with train.jsonl + test.jsonl) +# Examples that need a directory data_path (corda, glora, hira, olora, +# waveft) fail with FileNotFoundError if directory fixtures are not +# copied here — root cause of CI failures in run 35222723526. +prepare_fixtures() { + local src="${FIXTURE_DIR:?FIXTURE_DIR is required}" + local dst="$TARGET_ROOT/fixtures" + echo "preparing fixtures from $src to $dst" + if [[ ! -d "$src" ]]; then + echo "FATAL: fixture source dir not found: $src" >&2 + exit 1 + fi + shopt -s nullglob + # Top-level *.jsonl files + local n_files=0 + for f in "$src"/*.jsonl; do + mkdir -p "$dst" + cp "$f" "$dst/" + n_files=$((n_files + 1)) + done + # Top-level subdirectories (e.g. ci_alpaca_10/, ci_corda_8/) + local n_dirs=0 + for d in "$src"/*/; do + [[ -d "$d" ]] || continue + mkdir -p "$dst" + cp -r "$d" "$dst/" + n_dirs=$((n_dirs + 1)) + done + shopt -u nullglob + if (( n_files == 0 && n_dirs == 0 )); then + echo "FATAL: no fixture files (*.jsonl) or subdirs found in $src" >&2 + exit 1 + fi + echo "copied $n_files fixture file(s) and $n_dirs fixture dir(s) to $dst" +} + +setup_peft() { + # peft from the guarded release checkout, plus the verified dependency + # line (2026-09-15, coder npu-1 端到端验证结论): + # transformers 4.57.1 + datasets>=4.7.0,<6 + hub<1.0 + trl 1.12.0 + # - transformers 4.57.1: 5.x 移除 send_example_telemetry 等旧 API + # - datasets>=4.7.0,<6: trl 1.12+ 在 wheel metadata 声明 datasets>=4.7.0 + # (pyproject.toml 自 v1.0.0 起 commit ac5421b4 引入，datasets<4 会 + # ResolutionImpossible)，<6 留出口避开未来 6.x breaking + # - hub<1.0: hub 1.x 拒绝 imdb 等无命名空间数据集 + # - trl 1.12.0: trl ≥ 1.12 默认 chunked_nll 走 _patch_chunked_ce_lm_head + # (sft_trainer.py:233)。该 patch 第 383 行 (1.13 在 386) 走 + # inspect.signature(original_forward.__func__)，假设 forward 是普通 + # method；但 miss/mica 用 device_map="auto" 加载，accelerate + # .add_hook_to_module 把 forward 包成 functools.partial，没 __func__ + # → AttributeError（纯 PyTorch + accelerate，硬件无关，CUDA 同问题）。 + # overlay 在 examples_manifest.yaml 的 miss/mica 例里显式 --loss_type nll + # 跳过 chunked patch 走标准 cross-entropy。 + # 注：sft_trainer.py:1331 isinstance(BaseTunerLayer) guard 在这两例不 + # 会触发（target_modules 不含 lm_head），guard 放过后 patch 内部才崩。 + # - scikit-learn: adamss 的 ASA 回调（peft.tuners.adamss）硬性 import + # sklearn；evaluate.load("glue") 的 metric 模块同样要 sklearn.metrics。 + # coder 验证机里碰巧预装，CANN 裸镜像没有（run 35045940066 实测缺失）。 + # PIP_CONSTRAINT keeps CUDA metapackages out. + echo "installing peft from $TARGET_ROOT" + python -m pip install -e "$TARGET_ROOT" + # modelscope: hub_cache.py's fallback when HuggingFace fails; 1.40.1 + # fails to import next to hub<1.0. + python -m pip install "transformers==4.57.1" "datasets>=4.7.0,<6" \ + "huggingface_hub<1.0" "trl==1.12.0" evaluate scikit-learn \ + "modelscope==1.37.0" + # torchvision only pairs with the 2.12 stack (adamss image example + # imports it); the 2.9 profiles are sft-only and don't import + # torchvision (its 0.27 wheel links the torch 2.12 ABI). + local _torch_ver _npu_ver + read -r _torch_ver _npu_ver <<< "$(torch_stack_for_profile "$PROFILE")" + if [[ "$_torch_ver" == "2.12.0" ]]; then + install_cpu_torchvision + fi + # t5-base seq2seq FSDP 例（peft_lora_seq2seq_accelerate_fsdp.py）零 CLI， + # 数据路径硬编码 cwd 相对 temp/data/FinancialPhraseBank-v1.0/ 下两个 + # jsonl——从仓内 fixture 放置（run 阶段 cwd=$TARGET_ROOT 直接命中）。 + mkdir -p "$TARGET_ROOT/temp/data/FinancialPhraseBank-v1.0" + cp "$TARGET_ROOT/fixtures/financial_phrasebank/financial_phrase_bank_train.jsonl" \ + "$TARGET_ROOT/temp/data/FinancialPhraseBank-v1.0/" + cp "$TARGET_ROOT/fixtures/financial_phrasebank/financial_phrase_bank_val.jsonl" \ + "$TARGET_ROOT/temp/data/FinancialPhraseBank-v1.0/" + python -c "import peft, trl, transformers, datasets, accelerate; print('peft', peft.__version__, '/ trl', trl.__version__, '/ transformers', transformers.__version__)" + + # Local snapshot paths for overlay_args: cache hit as is, a miss is + # downloaded into the shared cache once. Hub ids the examples hardcode + # (mt0-small, dinov2-base, glue, t5-base, opt-350m, imdb ...) are + # fetched by from_pretrained / load_dataset into the same cache. + resolve_hub_paths SFT_MODEL_PATH ROBERTA_BASE_PATH BERT_BASE_UNCASED_PATH +} + +setup_peft_dreambooth() { + # SD dreambooth 例（lora/oft/deft/hra/stable_diffusion ×5）：SFT 栈之外 + # 还要 diffusers + tensorboard。run 35303810367 实测缺 diffusers 直接 + # import 崩（train_dreambooth.py:14）→ 初版修复"diffusers tensorboard" + # 不 pin，以为 pip 不会动已装版本——run 35316518546（2026-09-18）5 条 + # dreambooth 全挂打脸：最新 diffusers 0.40 要求 huggingface-hub>=1.23， + # pip 把 hub 从 0.36.2 升到 1.31+（对已装的 transformers 只给 warning + # 不回退），transformers 4.57.1 的 dependency_versions_check 硬校验 + # hub<1.0 → `import transformers` 直接 ImportError。 + # 修复（coder npu-5 2026-09-18 实测 5 条 exit 0）：hub<1.0 + + # diffusers==0.39.0 双 pin（0.39 与 hub<1.0 兼容，0.40 起不兼容）。 + # hra 例外：v0.21.0 脚本 :319 在 cwd/data/dreambooth 不存在时无条件 + # git clone github.com/google/dreambooth（runner 网络不通 + 纯死代码， + # 该路径只用于 clone 自身）；预先 mkdir 空目录即可绕过，无需 patch。 + setup_peft + echo "installing dreambooth stack (diffusers==0.39.0 + tensorboard + wandb, hub<1.0)" + # wandb：boft_dreambooth 走 --report_to wandb（脚本 `import wandb` 硬性， + # wandb_init 只在 wandb 分支定义）；sitecustomize 的 wandb neutralizer + # 把 wandb.init 强制 disabled，无需 API key。 + python -m pip install "huggingface_hub<1.0" "diffusers==0.39.0" tensorboard wandb + python -c "import diffusers, tensorboard, wandb; print('diffusers', diffusers.__version__)" + mkdir -p "$TARGET_ROOT/data/dreambooth" + + # SD v1.5 只有本 profile 需要，只取配置与 safetensors；其它例不解析它。 + resolve_hub_paths SD_MODEL_PATH +} + +setup_peft_29() { + # Multi-card fsdp sft entries (run_peft_fsdp.sh / + # run_peft_qlora_fsdp.sh): identical deps to setup_peft, only the + # torch stack differs (2.9.0 pair, see torch_stack_for_profile). + setup_peft +} + +setup_peft_fp4() { + # fp4_finetuning/finetune_fp4_opt_bnb_peft.py：唯一走 bnb 的条目。 + # bnb 0.50.2 走默认 CPU 后端在 NPU 上跑 4-bit NF4（无需 NPU 专用 + # kernel）；run_example.sh 对该目录跳过 transfer_to_npu（见 + # SKIP_TRANSFER_TO_NPU）。bnb **不能**进全局 setup_peft：diffusers + # 0.39 的 quantizers/auto.py 硬 import bnb，而 dreambooth 条目挂 + # transfer_to_npu 时 bnb 的 cuda backend 崩 torch._C. + # _cuda_getCurrentRawStream——装了 bnb 会把 6 条 dreambooth 全带崩。 + setup_peft + python -m pip install --index-url "$ALIYUN_PIP_INDEX" bitsandbytes +} + +setup_peft_ds() { + # deepspeed 多卡 sft 例（run_peft_deepspeed.sh / + # run_peft_qlora_deepspeed_stage3.sh）：setup_peft 之上装 deepspeed。 + # torch 栈固定 2.9.0 对（torch_stack_for_profile）：torch 2.12 的 + # c10d broadcast sm90 检查 + torch_npu get_device_capability 返回 + # None 会在 ZeRO zero-init 的 dist.broadcast(param.data) 直接崩 + # （run 35502546409）。 + # 0.19.7 的 npu accelerator 自动识别 Ascend + HCCL，coder npu-5 + # 2026-09-18 实跑 ZeRO-3 2 卡 exit 0（loss 4.64）。DS_BUILD_OPS=0 + # 跳过 CPU op 内核编译：ZeRO-3 bf16 训练路径不需要编译 op，且 + # 编译耗时 + 裸镜像缺编译链，CI 不划算。 + setup_peft + echo "installing deepspeed for ZeRO-3 sft entries" + DS_BUILD_OPS=0 python -m pip install deepspeed==0.19.7 + python -c "import deepspeed; from deepspeed.accelerator import get_accelerator; print('deepspeed', deepspeed.__version__, 'accelerator', get_accelerator()._name)" +} + +supported_profiles() { + declare -F | awk '/^declare -f setup_/ { sub(/^declare -f setup_/, ""); print }' | paste -sd' ' - +} + +if ! declare -F "setup_${PROFILE}" >/dev/null 2>&1; then + echo "unknown profile: ${PROFILE} (supported: $(supported_profiles))" >&2 + exit 1 +fi + +TARGET_ROOT="${TARGET_ROOT:?TARGET_ROOT is required}" +GITHUB_WORKSPACE="${GITHUB_WORKSPACE:?GITHUB_WORKSPACE is required}" +GITHUB_ENV="${GITHUB_ENV:?GITHUB_ENV is required}" + +HERE=$(cd "$(dirname "$0")" && pwd) +export PIP_CONSTRAINT="$(cd "$HERE/.." && pwd)/constraints-npu.txt" +# This runner pool reaches huggingface.co directly. The engine's +# hf-mirror default 308-redirects namespace-less ids such as roberta-base +# to huggingface.co, a redirect huggingface_hub does not follow. +export HF_ENDPOINT=https://huggingface.co + +source /usr/local/Ascend/ascend-toolkit/set_env.sh + +select_pip_index +python -m pip install -U pip setuptools wheel +ensure_torch_stack +prepare_fixtures + +"setup_${PROFILE}"