From ed5841e78197b22722c37618d477fc27cb2a93d2 Mon Sep 17 00:00:00 2001 From: cyb <694642680@qq.com> Date: Mon, 12 Oct 2026 09:44:59 +0800 Subject: [PATCH] add deepspeed examples guard --- .github/workflows/deepspeed-examples.yml | 60 + examples-guards/deepspeed/constraints-npu.txt | 6 + .../deepspeed/examples_manifest.yaml | 759 ++++++++ .../deepspeed/fixtures/ci_alpaca_16.json | 18 + .../deepspeed/fixtures/ci_dpo_8.json | 8 + .../deepspeed/fixtures/ci_gsm8k_8.jsonl | 8 + .../deepspeed/fixtures/ci_mmlu_8.jsonl | 8 + .../deepspeed/fixtures/ci_mnli_8.jsonl | 8 + .../deepspeed/fixtures/ci_rac_8.jsonl | 8 + .../deepspeed/fixtures/ci_rlhf_8.json | 8 + .../deepspeed/fixtures/ci_rw_8.json | 8 + .../deepspeed/fixtures/ci_sft_8.json | 8 + .../deepspeed/scripts/run_example.sh | 1228 +++++++++++++ .../deepspeed/scripts/setup_example.sh | 1598 +++++++++++++++++ 14 files changed, 3733 insertions(+) create mode 100644 .github/workflows/deepspeed-examples.yml create mode 100644 examples-guards/deepspeed/constraints-npu.txt create mode 100644 examples-guards/deepspeed/examples_manifest.yaml create mode 100644 examples-guards/deepspeed/fixtures/ci_alpaca_16.json create mode 100644 examples-guards/deepspeed/fixtures/ci_dpo_8.json create mode 100644 examples-guards/deepspeed/fixtures/ci_gsm8k_8.jsonl create mode 100644 examples-guards/deepspeed/fixtures/ci_mmlu_8.jsonl create mode 100644 examples-guards/deepspeed/fixtures/ci_mnli_8.jsonl create mode 100644 examples-guards/deepspeed/fixtures/ci_rac_8.jsonl create mode 100644 examples-guards/deepspeed/fixtures/ci_rlhf_8.json create mode 100644 examples-guards/deepspeed/fixtures/ci_rw_8.json create mode 100644 examples-guards/deepspeed/fixtures/ci_sft_8.json create mode 100644 examples-guards/deepspeed/scripts/run_example.sh create mode 100644 examples-guards/deepspeed/scripts/setup_example.sh diff --git a/.github/workflows/deepspeed-examples.yml b/.github/workflows/deepspeed-examples.yml new file mode 100644 index 000000000..55d863bf3 --- /dev/null +++ b/.github/workflows/deepspeed-examples.yml @@ -0,0 +1,60 @@ +# deepspeed examples guard - project thin trigger. +# +# Calls the common engine .github/workflows/examples-template.yml; this +# file only declares what varies per project: schedule, concurrency, +# upstream/examples repo, dispatch input forwarding, and matrix +# parallelism. The guard loop (release-only monitor / manifest-check / +# run-example matrix / validate-results / save-monitor-state) lives in +# the engine. See .github/workflows/examples-template.yml for the engine contract. +# +# Split mode (examples_repo set): the monitored upstream is the main +# DeepSpeed repo (which publishes releases); the example scripts live in +# the separate DeepSpeedExamples repo (no releases; follows its default +# branch master). DeepSpeed source is installed from the main-repo +# checkout; examples run from the DeepSpeedExamples checkout. + +name: deepspeed-examples + +concurrency: + # Serialize schedule runs; cancel stale runs only within the same project PR. + # Each manual run gets its own group. + group: ${{ github.event_name == 'schedule' && 'deepspeed-examples-schedule' || (github.event_name == 'pull_request' && format('deepspeed-examples-pr-{0}', github.event.pull_request.number)) || format('manual-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +on: + # Daily at 02:00 Beijing time (UTC 18:00). + schedule: + - cron: '0 18 * * *' + pull_request: + branches: [main] + types: [opened, synchronize, reopened] + paths: + - 'examples-guards/deepspeed/**' + - '.github/workflows/deepspeed-examples.yml' + workflow_dispatch: + inputs: + target_ref: + description: Branch, tag, or SHA of deepspeedai/DeepSpeed (empty selects latest release, falls back to master) + required: false + default: '' + type: string + +# Engine jobs query the GitHub Job API (validate-results); called workflows +# run with the caller's GITHUB_TOKEN permissions. +permissions: + actions: read + contents: read + +jobs: + deepspeed-examples: + uses: ./.github/workflows/examples-template.yml + with: + # Namespaces monitor-state cache keys, artifact names, and the + # scheduling paths under examples-guards/deepspeed. + project: deepspeed + upstream_repo: deepspeedai/DeepSpeed + # Latest-release lookup may fail (e.g. API HTTP 403); DeepSpeed has no main branch. + default_branch: master + examples_repo: deepspeedai/DeepSpeedExamples + target_ref: ${{ inputs.target_ref }} + max_parallel: 4 diff --git a/examples-guards/deepspeed/constraints-npu.txt b/examples-guards/deepspeed/constraints-npu.txt new file mode 100644 index 000000000..227fe4513 --- /dev/null +++ b/examples-guards/deepspeed/constraints-npu.txt @@ -0,0 +1,6 @@ +# Keep the scientific/data stack compatible throughout every setup install. +# SciPy 1.18 requires NumPy 2 and must not survive a NumPy 1.26 downgrade. +numpy==1.26.4 +scipy==1.16.3 +pyarrow==20.0.0 +pandas==2.2.3 diff --git a/examples-guards/deepspeed/examples_manifest.yaml b/examples-guards/deepspeed/examples_manifest.yaml new file mode 100644 index 000000000..e7a676b4a --- /dev/null +++ b/examples-guards/deepspeed/examples_manifest.yaml @@ -0,0 +1,759 @@ +version: 1 +# 主仓 deepspeedai/DeepSpeed 提供被测源码；example 路径相对 deepspeedai/DeepSpeedExamples。 +# supported 为 NPU CI 看护入口（新增项待实跑）；unsupported 仅登记尚未接入的设备用例。 +# 库、安装脚本、单元测试、数据准备及结果处理工具不作为 example 登记。 +# 扫描无 exclude；同一 example 的启动包装合并到主入口。 +scan: + root: '' + include_extensions: ['.sh', '.py'] +supported: + # RoBERTa MLM 训练；支持单卡 NPU，2 层模型/10 步验证 ZeRO-1 与 CPU offload。 + - path: training/HelloDeepSpeed/run_ds.sh + profile: deepspeed + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--num_layers 2' + - '--num_heads 2' + - '--h_dim 64' + - '--ff_dim 128' + - '--batch_size 2' + - '--num_iterations 10' + - '--log_every 5' + - '--dtype bf16' + timeout_minutes: 60 + # SFT 训练；支持单卡 NPU，使用本地 OPT-125M 和 8 行数据。 + - path: applications/DeepSpeed-Chat/training/step1_supervised_finetuning + exec: applications/DeepSpeed-Chat/training/step1_supervised_finetuning/main.py + profile: ds_chat_sft + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model_name_or_path ${OPT_125M_PATH}' + - '--data_path local/jsonfile' + - '--num_train_epochs 1' + - '--per_device_train_batch_size 4' + - '--per_device_eval_batch_size 4' + - '--max_seq_len 128' + - '--zero_stage 2' + - '--dtype bf16' + - '--print_loss' + - '--deepspeed' + - '--output_dir ${CI_OUTPUT_DIR}' + timeout_minutes: 120 + # 奖励模型训练；支持单卡 NPU，使用本地 OPT-125M 和小型偏好数据。 + - path: applications/DeepSpeed-Chat/training/step2_reward_model_finetuning + exec: applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/main.py + profile: ds_chat_rw + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model_name_or_path ${OPT_125M_PATH}' + - '--data_path local/jsonfile' + - '--num_train_epochs 1' + - '--per_device_train_batch_size 4' + - '--per_device_eval_batch_size 4' + - '--max_seq_len 128' + - '--zero_stage 2' + - '--dtype bf16' + - '--deepspeed' + - '--output_dir ${CI_OUTPUT_DIR}' + timeout_minutes: 120 + # DPO 偏好训练；支持单卡 NPU，使用本地 OPT-125M 和小批量数据。 + - path: applications/DeepSpeed-Chat/training/step2_dpo_finetuning + exec: applications/DeepSpeed-Chat/training/step2_dpo_finetuning/main.py + profile: ds_chat_dpo + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model_name_or_path ${OPT_125M_PATH}' + - '--data_path local/jsonfile' + - '--num_train_epochs 1' + - '--per_device_train_batch_size 2' + - '--per_device_eval_batch_size 2' + - '--max_seq_len 128' + - '--zero_stage 2' + - '--dtype bf16' + - '--beta 0.1' + - '--deepspeed' + - '--output_dir ${CI_OUTPUT_DIR}' + timeout_minutes: 150 + # RLHF 训练；支持单卡 NPU，OPT-125M actor/critic 使用官方 5 步测试模式。 + - path: applications/DeepSpeed-Chat/training/step3_rlhf_finetuning + exec: applications/DeepSpeed-Chat/training/step3_rlhf_finetuning/main.py + profile: ds_chat_rlhf + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--actor_model_name_or_path ${OPT_125M_PATH}' + - '--critic_model_name_or_path ${OPT_125M_PATH}' + - '--data_path local/jsonfile' + - '--enable_test_mode' + - '--test_stop_step 5' + - '--actor_zero_stage 2' + - '--critic_zero_stage 2' + - '--dtype bf16' + - '--deepspeed' + timeout_minutes: 180 + # 文本生成；支持 NPU 上的 HF baseline，本地 OPT-125M 绕过 DS kernel 注入。 + - path: inference/huggingface/text-generation/inference-test.py + profile: ds_infer + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model ${OPT_125M_PATH}' + - '--hf_baseline' + - '--dtype float32' + - '--max_new_tokens 8' + timeout_minutes: 60 + # CIFAR-10 分类；支持单卡 NPU，上游 launcher 配合 ModelScope 预置数据。 + - path: training/cifar + exec: training/cifar/run_ds.sh + profile: ds_cifar + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--epochs 1' + - '--log-interval 50' + - '--dtype bf16' + - '--stage 0' + timeout_minutes: 90 + # ZeRO 状态 offload；支持单卡 NPU，使用小模型和合成数据。 + - path: training/offload_states/offload_states.py + profile: deepspeed + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--hidden_dim 1024' + - '--nlayers 2' + timeout_minutes: 60 + # CIFAR MoE 训练；支持两卡 HCCL/EP=2，局部禁用 torch.compile 避开 Triton 依赖。 + - path: training/cifar/run_ds_moe.sh + profile: ds_cifar + runner: linux-aarch64-a2-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--epochs 1' + - '--log-interval 50' + - '--dtype bf16' + - '--stage 0' + timeout_minutes: 120 + # AutoTP loss 等价性；支持四卡 NPU，Qwen3-0.6B 验证 TP=1/3/4 各 5 步。 + - path: training/autotp_equivalence + exec: training/autotp_equivalence/train.py + profile: ds_autotp_equivalence + runner: linux-aarch64-a2-4 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model_name_or_path ${QWEN3_06B_PATH}' + - '--steps 5' + - '--seq_len 32' + - '--micro_batch_size 1' + - '--log_every 1' + timeout_minutes: 180 + + # HF Trainer AutoTP；八卡 TP=8、本地模型和 fixture，3 步含 2 步预热，已通过 #28。 + - path: training/tensor_parallel/hf_integration/train.py + profile: ds_hf_autotp + runner: linux-aarch64-a2-8 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model_name_or_path ${OPT_125M_PATH}' + - '--data_path ${ALPACA_CI_PATH}' + - '--output_dir ${CI_OUTPUT_DIR}/hf-autotp' + - '--max_steps 3' + - '--model_max_length 128' + - '--per_device_train_batch_size 1' + - '--gradient_accumulation_steps 1' + - '--learning_rate 2e-5' + - '--warmup_steps 2' + - '--logging_steps 1' + - '--eval_strategy no' + - '--save_strategy no' + - '--report_to none' + - '--bf16 True' + - '--tf32 False' + - '--gradient_checkpointing False' + - '--dataloader_pin_memory False' + - '--dataloader_num_workers 0' + timeout_minutes: 120 + # CIFAR PR-MoE；两卡 EP=2、残差专家 2/4，eager MoE 已通过 #27。 + - path: training/cifar/run_ds_prmoe.sh + profile: ds_cifar + runner: linux-aarch64-a2-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--epochs 1' + - '--log-interval 50' + - '--dtype bf16' + - '--stage 0' + timeout_minutes: 120 + # 动态 batch/LR；单卡合成序列打包及 2 epoch，已通过 #27。 + - path: training/data_efficiency/variable_batch_size_and_lr/variable_batch_size_and_lr_example.py + profile: ds_variable_batch + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--pipeline-num-stages 0' + timeout_minutes: 60 + # ZenFlow offload；两卡 256 维/2 层合成数据，6 次迭代已通过 #27。 + - path: training/DeepSpeed-ZenFlow/benchmark/zf_benchmark.py + profile: ds_zenflow + runner: linux-aarch64-a2-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--hidden_dim 256' + - '--nlayers 2' + - '--iteration 3' + - '--dtype torch.bfloat16' + - '--pin_memory_opts 0' + - '--topk_ratios 0.1' + - '--update_intervals 2' + - '--overlap_steps 0' + timeout_minutes: 90 + # Wanda 校准剪枝；NPU 本地 OPT-125M/JSONL，4×32 token 校准已通过 #27。 + - path: compression/reasoning_aware_compression/prune.py + profile: ds_rac_prune + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model ${OPT_125M_PATH}' + - '--device npu' + - '--dtype float32' + - '--pruning-method wanda' + - '--calibration prompt' + - '--dataset ${RAC_CI_PATH}' + - '--prompt-column prompt' + - '--no-chat-template' + - '--nsamples 4' + - '--seqlen 32' + - '--batch-size 1' + - '--scope all' + - '--layer-thirds 1' + - '--sparsity 0.5' + - '--output ${CI_OUTPUT_DIR}/rac-pruned' + - '--save-report' + timeout_minutes: 90 + + # SFT 生成评测；单卡先训练小检查点，再对比基础/微调模型，8 token，待实跑。 + - path: applications/DeepSpeed-Chat/training/step1_supervised_finetuning/prompt_eval.py + profile: ds_chat_prompt_eval + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model_name_or_path_baseline ${OPT_125M_PATH}' + - '--model_name_or_path_finetune ${CI_OUTPUT_DIR}/prompt-eval-sft' + - '--max_new_tokens 8' + - '--language English' + timeout_minutes: 120 + # 奖励评分 smoke；单卡本地 OPT-125M/初始化评分头，两组偏好 forward，待实跑。 + - path: applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/rw_eval.py + profile: ds_chat_reward_eval + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model_name_or_path ${OPT_125M_PATH}' + - '--num_padding_at_beginning 1' + timeout_minutes: 60 + # 固定长度 HF AutoTP；八卡 TP=8/序列 128/3 步，校验 padding 与 label masking，待实跑。 + - path: training/tensor_parallel/hf_integration/train_bench_length.py + profile: ds_hf_bench_length + runner: linux-aarch64-a2-8 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model_name_or_path ${OPT_125M_PATH}' + - '--data_path ${ALPACA_CI_PATH}' + - '--output_dir ${CI_OUTPUT_DIR}/hf-bench-length' + - '--max_steps 3' + - '--model_max_length 128' + - '--per_device_train_batch_size 1' + - '--gradient_accumulation_steps 1' + - '--learning_rate 2e-5' + - '--warmup_steps 2' + - '--logging_steps 1' + - '--eval_strategy no' + - '--save_strategy no' + - '--report_to none' + - '--bf16 True' + - '--tf32 False' + - '--gradient_checkpointing False' + - '--dataloader_pin_memory False' + - '--dataloader_num_workers 0' + timeout_minutes: 120 + # SuperOffload 入口；两卡 ZeRO-Offload 模式/CPU 参数和优化器卸载，3 步，待实跑。 + - path: training/DeepSpeed-SuperOffload/finetune_zero3.py + profile: ds_superoffload + runner: linux-aarch64-a2-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model_name ${OPT_125M_PATH}' + - '--dataset_name ${ALPACA_DATASET_DIR}' + - '--lr 0.001' + - '--batch_size 1' + - '--num_train_epochs 1' + - '--max_length 128' + - '--bench_steps 3' + - '--warmup_steps 1' + - '--attn_implementation eager' + - '--output_dir ${CI_OUTPUT_DIR}/superoffload' + timeout_minutes: 120 + + # 模型张量卸载对比；单卡 ZeRO-3/native mlock，合成小模型/两臂各 3 步，待实跑. + - path: benchmarks/pin_memory/model_tensor_offload/bench.py + profile: ds_pin_memory + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--hidden 4' + - '--layers 2' + - '--batch 1' + - '--seq 16' + - '--steps 3' + - '--warmup 1' + - '--zero-stage 3' + timeout_minutes: 120 + # 激活卸载对比；单卡 NPU checkpoint/native pin，两臂各 3 步，待实跑。 + - path: benchmarks/pin_memory/activation_offload/bench.py + profile: ds_pin_memory + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--hidden 64' + - '--layers 2' + - '--batch 1' + - '--seq 32' + - '--steps 3' + - '--warmup 1' + timeout_minutes: 120 + # 主机/NPU 双向拷贝；三种 1MiB 缓冲区各 3 次，不冒充 CUDA host registration，待实跑。 + - path: benchmarks/pin_memory/h2d_d2h/bench.py + profile: ds_pin_memory + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--sizes-mib 1' + - '--warmup 1' + - '--iters 3' + - '--skip-native-register' + timeout_minutes: 120 + # ZenFlow LLM 微调；OPT-125M/16 行本地 Alpaca，单卡 CPU optimizer offload 与保存，待实跑。 + - path: training/DeepSpeed-ZenFlow/finetuning/finetune_llama.py + profile: ds_zenflow_finetune + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model_name "${OPT_125M_PATH}"' + - '--lr 0.00002' + - '--batch_size 1' + - '--num_train_epochs 1' + - '--output_dir "${CI_OUTPUT_DIR}/zenflow-finetune-checkpoint"' + timeout_minutes: 120 + # OPSD 在线蒸馏；小模型/3 步 rollout、教师 CPU 缓存与 streamed KL，待实跑。 + - path: training/opsd/main.py + profile: ds_opsd + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: [] + timeout_minutes: 120 + # Student TP/ZeRO-3 smoke；两卡小模型 forward/backward/step，关闭可选 CPU optimizer offload，待实跑。 + - path: training/opsd/test_student_autotp_zero3.py + profile: ds_opsd + runner: linux-aarch64-a2-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: [] + timeout_minutes: 120 + # Teacher TP/ZeRO-3 smoke；两卡小模型参数 CPU offload 与 logit cache，待实跑。 + - path: training/opsd/test_teacher_autotp_zero3.py + profile: ds_opsd + runner: linux-aarch64-a2-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: [] + timeout_minutes: 120 + # BERT 推理计时；本地模型/4 次 DS engine forward 与 NPU 同步，待实跑。 + - path: benchmarks/inference/bert-bench.py + profile: ds_legacy_inference_bench + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model "${BERT_BASE_CASED_PATH}"' + - '--deepspeed' + - '--dtype fp32' + - '--trials 4' + timeout_minutes: 120 + # GPT 推理计时；OPT-125M/4 次×8 token，DS engine 与 NPU 同步，待实跑。 + - path: benchmarks/inference/gpt-bench.py + profile: ds_legacy_inference_bench + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model "${OPT_125M_PATH}"' + - '--deepspeed' + - '--dtype fp32' + - '--max-tokens 8' + - '--trials 4' + timeout_minutes: 120 + # HF/DS 生成一致性；OPT-125M/2 个 prompt，强制断言零 mismatch，待实跑。 + - path: inference/huggingface/text-generation/ds-hf-compare.py + profile: ds_hf_ds_compare + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model "${OPT_125M_PATH}"' + - '--dtype float32' + - '--num_inputs 2' + - '--min_length 12' + - '--max_length 16' + - '--print_outputs' + timeout_minutes: 120 + # BERT-large 填词；本地同名 ModelScope 权重，单卡短序列 DS 推理，待实跑。 + - path: inference/huggingface/fill-mask/test-bert.py + profile: ds_fill_mask_bert + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: [] + timeout_minutes: 120 + # ELECTRA 填词；本地 generator 权重，两卡 TP=2 的显式切分策略，待实跑。 + - path: inference/huggingface/fill-mask/test-electra.py + profile: ds_fill_mask_electra + runner: linux-aarch64-a2-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: [] + timeout_minutes: 120 + # RoBERTa-large 填词；本地权重与两卡 TP=2，验证 NPU 结果，待实跑。 + - path: inference/huggingface/fill-mask/test-roberta.py + profile: ds_fill_mask_roberta + runner: linux-aarch64-a2-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: [] + timeout_minutes: 120 + # T5 翻译；本地权重与两卡 TP=2，模型生成配置限制 8 token，待实跑。 + - path: inference/huggingface/translation/test-t5-base.py + profile: ds_t5_translation + runner: linux-aarch64-a2-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: [] + timeout_minutes: 120 + # Wav2Vec2 CTC 推理；两条合成音频/本地同名模型，验证 NPU logits 而非识别准确率，待实跑。 + - path: inference/huggingface/automatic-speech-recognition/test-wav2vec2.py + profile: ds_asr_ctc + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: [] + timeout_minutes: 120 + + # LLM 微调；SmolLM2-135M/匹配 rotary 的依赖，两卡 ZeRO-2/3 步，待实跑。 + - path: training/deepspeed_finetune_demo/finetune_llama.py + profile: ds_finetune_demo + runner: linux-aarch64-a2-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model_name "${SMOLLM2_135M_PATH}"' + - '--dataset_name "${ALPACA_FINETUNE_DIR}"' + - '--batch_size 2' + - '--num_train_epochs 1' + - '--max_length 128' + - '--max_steps 3' + - '--eval_steps 0' + - '--checkpoint_steps 0' + - '--output_dir "${CI_OUTPUT_DIR}/finetune-demo"' + timeout_minutes: 120 + # SD 教师 CFG 蒸馏；本地 SD1.5/8 幅图，两卡 ZeRO-2 FP32/3 步全 UNet 训练，待实跑。 + - path: training/stable_diffusion/train_sd_distil_lora.py + profile: ds_sd_distil + runner: linux-aarch64-a2-2 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--pretrained_model_name_or_path "${SD15_PATH}"' + - '--default_prompt "a colorful geometric shape"' + - '--resolution 64' + - '--center_crop' + - '--train_batch_size 1' + - '--gradient_accumulation_steps 1' + - '--max_train_steps 3' + - '--learning_rate 0.000005' + - '--lr_scheduler constant' + - '--lr_warmup_steps 0' + - '--mixed_precision no' + - '--report_to tensorboard' + - '--checkpointing_steps 1000' + - '--dataloader_num_workers 0' + - '--seed 42' + - '--output_dir "${CI_OUTPUT_DIR}/sd-distil"' + timeout_minutes: 180 + # OPSD decode 对比；小模型/16 prompt token，原生 raw 与 HybridEngine 各 2 次，待实跑。 + - path: training/opsd/benchmarks/bench_decode_1p1r.py + profile: ds_opsd_decode + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--model "${QWEN3_06B_PATH}"' + - '--prompt-len 16' + - '--max-new-tokens 4' + - '--num-warmup 1' + - '--num-iters 2' + timeout_minutes: 120 + + # GSM8K 答案生成；尝试 vLLM-Ascend 单卡，本地 0.5B 模型/8 题/16 token，待 NPU CI。 + - path: training/deepspeed_finetune_demo/evaluate/gsm8k/gen_gsm8k.py + profile: ds_vllm_eval + runner: linux-aarch64-a2-1 + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend/vllm-ascend:v0.23.0 + overlay_args: + - '--model "${VLLM_CI_MODEL_PATH}"' + - '--output "${CI_OUTPUT_DIR}/gsm8k"' + - '--tp 1' + - '--max_new_tokens 16' + timeout_minutes: 120 + # MMLU 答案生成；尝试 vLLM-Ascend 单卡，本地 0.5B 模型/8 题，待 NPU CI。 + - path: training/deepspeed_finetune_demo/evaluate/mmlu/gen_mmlu.py + profile: ds_vllm_eval + runner: linux-aarch64-a2-1 + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend/vllm-ascend:v0.23.0 + overlay_args: + - '--model "${VLLM_CI_MODEL_PATH}"' + - '--output "${CI_OUTPUT_DIR}/mmlu"' + - '--tp 1' + timeout_minutes: 120 + # RAC 推理轨迹采集；原生 vLLM 后端，本地 0.5B 模型/8 prompt/16 token，待 NPU CI。 + - path: compression/reasoning_aware_compression/collect_traces.py + profile: ds_vllm_traces + runner: linux-aarch64-a2-1 + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend/vllm-ascend:v0.23.0 + overlay_args: + - '--backend vllm' + - '--model "${VLLM_CI_MODEL_PATH}"' + - '--dataset "${RAC_CI_PATH}"' + - '--prompt-column prompt' + - '--max-prompts 8' + - '--trace-tokens 1024' + - '--max-new-tokens 16' + - '--batch-size 2' + - '--output "${CI_OUTPUT_DIR}/rac-traces.jsonl"' + timeout_minutes: 180 + + # OpenAI 服务压测；原生客户端连接本地 vLLM-Ascend，单卡/4 请求，待 NPU CI。 + - path: benchmarks/inference/mii/run_benchmark.py + profile: ds_vllm_bench + runner: linux-aarch64-a2-1 + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend/vllm-ascend:v0.23.0 + overlay_args: + - '--backend openai' + - '--model ci-model' + - '--num_clients 1' + - '--num_requests 4' + - '--max_prompt_length 64' + - '--mean_prompt_length 32' + - '--mean_max_new_tokens 16' + - '--prompt_length_var 0' + - '--max_new_tokens_var 0' + - '--warmup 0' + - '--out_json_dir "${CI_OUTPUT_DIR}/benchmark"' + timeout_minutes: 120 + # 推理服务并发压测；原生 OpenAI 客户端连接 NPU 服务，单客户端/4 请求，待 NPU CI。 + - path: benchmarks/inference/deepspeedometer/src/deepspeedometer/benchmark_runner.py + profile: ds_vllm_bench + runner: linux-aarch64-a2-1 + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend/vllm-ascend:v0.23.0 + overlay_args: + - '--api openai' + - '--model ci-model' + - '--num_clients 1' + - '--min_requests 4' + - '--num_requests_per_client 4' + - '--warmup_requests 0' + - '--max_prompt_length 64' + - '--prompt_length 32' + - '--max_new_tokens 16' + - '--prompt_length_var 0' + - '--max_new_tokens_var 0' + - '--result_dir "${CI_OUTPUT_DIR}/benchmark"' + timeout_minutes: 120 + # async_io 设备读写；复用原配方 ds_io 的 accelerator 路径，1MiB/2 次读写，待 NPU CI。 + - path: deepnvme/ds_io/ds_io_sweep.sh + profile: ds_aio + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--io_size 1M' + - '--gpu' + - '--use_accelerator_pin_memory' + - '--io_parallel 1' + - '--block_size 1M' + - '--queue_depth 4' + - '--loops 2' + - '--warmup_loops 0' + timeout_minutes: 90 + + # MATH-500 评测配方；本地 8 题/16 token，原生数学评分，不设准确率门槛，待 NPU CI。 + - path: compression/reasoning_aware_compression/bash_script/eval_math500.sh + profile: ds_math500 + runner: linux-aarch64-a2-1 + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend/vllm-ascend:v0.23.0 + overlay_args: + - '--max-samples 8' + - '--dataset-loading-processes 1' + - '--output-dir "${CI_OUTPUT_DIR}/math500"' + - '--save-details' + timeout_minutes: 120 + + # BERT 自动调参；合并 base/large 启动配方，本地 BERT-base/8 行分类数据/小 batch 搜索，待 NPU CI。 + - path: training/autotuning/hf/bert-base/test_tune.sh + profile: ds_autotune_bert + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--max_seq_length 32' + timeout_minutes: 180 + # DeBERTa 自动调参；本地 DeBERTa-v3-small/8 行分类数据/小 batch 搜索，待 NPU CI。 + - path: training/autotuning/hf/deberta/test_tune.sh + profile: ds_autotune_deberta + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--max_seq_length 32' + timeout_minutes: 180 + # DistilBERT 自动调参；本地模型/8 行分类数据/小 batch 搜索，待 NPU CI。 + - path: training/autotuning/hf/distilbert/test_tune.sh + profile: ds_autotune_distilbert + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--max_seq_length 32' + timeout_minutes: 180 + # GPT-2 自动调参；合并 medium/large/xl 启动配方，本地 GPT-2/8 行文本/小 batch 搜索，待 NPU CI。 + - path: training/autotuning/hf/gpt2/test_tune.sh + profile: ds_autotune_gpt2 + runner: linux-aarch64-a2-1 + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: + - '--block_size 32' + timeout_minutes: 180 + +unsupported: + # DeepCompile 的原生算子为 CUDA 专用：is_deepcompile_supported() 要求 device_name 为 cuda， + # csrc/compile 直接使用 at::cuda 流、c10::kCUDA 与 ATen/native/cuda/Resize.h；无补丁的昇腾环境无法加载。 + - benchmarks/deepcompile/run_bench_lm.py + # HybridEngine rollout 性能测试；#30 的 OPT 注入路径触发 NPU softmax_context_bf16 参数 16/19 不匹配，需上游修复。 + - benchmarks/opsd/benchmark_hybrid_engine_rollout.py + # BF16 主权重训练对比；torch.cuda 和 CUDA autocast 硬绑定。 + - training/bf16_master_weight/train.py + # CIFAR/AlexNet 流水线训练；硬绑定 CUDA 设备和 NCCL。 + - training/pipeline_parallelism/train.py + # 交互式文本聊天；pipeline 硬编码 cuda:0，需已训练模型。 + - applications/DeepSpeed-Chat/inference/chatbot.py + # 交互式视觉聊天；硬绑定 .cuda()，需 Llama-2/视觉模型检查点。 + - applications/DeepSpeed-VisualChat/chat/chat.py + # 视觉聊天批量评测；硬绑定 .cuda()，需预训练检查点。 + - applications/DeepSpeed-VisualChat/eval/batch_generation.py + # 视觉聊天训练；硬绑定 CUDA，需 Llama-2、CLIP 和大规模图文数据。 + - applications/DeepSpeed-VisualChat/training/main.py + # AutoSP 正确性编排；子训练必经 ring_attention/flash_attn CUDA 扩展。 + - benchmarks/autosp/correctness/correctness.sh + # AutoSP loss 对比；调用依赖 CUDA flash_attn 的 run.py，换小模型仍受阻。 + - benchmarks/autosp/correctness/correctness_run.py + # 序列并行训练；必经 import ring_attention/flash_attn，eager 参数也不能绕过 CUDA 扩展。 + - benchmarks/autosp/run.py + # all_gather 吞吐基准；torch.cuda.Event 计时硬绑定 CUDA。 + - benchmarks/communication/all_gather.py + # all_reduce 吞吐基准；torch.cuda.Event 计时硬绑定 CUDA。 + - benchmarks/communication/all_reduce.py + # all_to_all 吞吐基准；torch.cuda.Event 计时硬绑定 CUDA。 + - benchmarks/communication/all_to_all.py + # broadcast 吞吐基准；torch.cuda.Event 计时硬绑定 CUDA。 + - benchmarks/communication/broadcast.py + # 双 rank 点对点吞吐基准；torch.cuda.Event 计时硬绑定 CUDA。 + - benchmarks/communication/pt2pt.py + # 多算子通信基准汇总；硬绑定 CUDA 计时，非训练验证。 + - benchmarks/communication/run_all.py + # BERT Triton 延迟基准；硬绑定 torch.cuda/NCCL 与 CUDA kernel。 + - benchmarks/inference/triton-bert-benchmark.py + # async_io 读取设备张量；可准备小文件，但结果硬调用 .cuda()。 + - deepnvme/file_access/aio_load_gpu_tensor.py + # async_io 写入设备张量；可准备小文件，但源张量硬编码 device='cuda'。 + - deepnvme/file_access/aio_store_gpu_tensor.py + # GDS 读取设备张量；缓冲区硬编码 CUDA，且依赖 CUDA GPUDirect Storage。 + - deepnvme/file_access/gds_load_gpu_tensor.py + # GDS 写入设备张量；缓冲区硬编码 CUDA，且依赖 CUDA GPUDirect Storage。 + - deepnvme/file_access/gds_store_gpu_tensor.py + # Python 读取设备张量；可准备小文件，但结果硬调用 .cuda()。 + - deepnvme/file_access/py_load_gpu_tensor.py + # Python 写入设备张量；可准备小文件，但源张量硬编码 device='cuda'。 + - deepnvme/file_access/py_store_gpu_tensor.py + # DeepSpeed 检查点 IO；必经 NCCL、pdb 断点与 CUDA GDS，换小文件不能绕过。 + - deepnvme/model_checkpoint/deepspeed_save_model.py + # 模型保存恢复校验；输入、模型及恢复张量硬编码 CUDA。 + - deepnvme/model_checkpoint/torch_save_load_model.py + # 模型检查点 IO；设备模式强制 CUDA GDS，关闭 gpu 后仅测 CPU。 + - deepnvme/model_checkpoint/torch_save_model.py + # 张量检查点 IO；设备模式强制 CUDA GDS，关闭 gpu 后仅测 CPU。 + - deepnvme/model_checkpoint/torch_save_tensor.py + # HumanEval 代码生成评测；硬绑定 CUDA，需 MII、7B 模型及评测数据。 + - evaluation/inference/human_eval/run_human_eval.py + # Stable Diffusion 图像生成；硬绑定 CUDA/CUDA graph，模型权重较大。 + - inference/huggingface/stable-diffusion/test-stable-diffusion.py + # GPT 文本生成；model.cuda() 和 torch.cuda.synchronize() 硬绑定 CUDA。 + - inference/huggingface/text-generation/run-generation-script/test-run-generation.py + # T5 文本生成；硬编码四卡 TP/CUDA 设备，无 CLI 缩规模接口。 + - inference/huggingface/text2text-generation/test-t5.py + # 大模型 ZeRO offload 推理；硬绑定 NCCL 和 torch.cuda.current_device。 + - inference/huggingface/zero_inference/run_model.py + # Falcon 推理；MII 必经 Inference V2 的 CUDA activation/kernel，缩模型不能绕过。 + - inference/mii/non-persistent/falcon.py + # Llama 推理；MII 必经 Inference V2 的 CUDA activation/kernel，另有受限权重。 + - inference/mii/non-persistent/llama2.py + # Mixtral 推理；MII 必经 Inference V2 的 CUDA activation/kernel，缩模型不能绕过。 + - inference/mii/non-persistent/mixtral.py + # MII 非持久推理；build_hf_engine 必经 CUDA activation/kernel，无 NPU 实现。 + - inference/mii/non-persistent/pipeline.py + # MII 持久服务；build_hf_engine 必经 CUDA activation/kernel，无 NPU 实现。 + - inference/mii/persistent/serve.py + # SGLang ZeRO 推理基准；需 tjruwase/sglang zero-inference 分支，配方含 CUDA GDS 卸载。 + - inference/sglang/run_llama3_1B.sh + # SGLang ZeRO 推理基准；需专用 zero-inference 分支与 CUDA GDS，换小模型不能解决后端约束。 + - inference/sglang/run_llama3_70B.sh + # SGLang ZeRO 推理基准；需专用 zero-inference 分支，CPU/NVMe/GDS 对比尚无 NPU 配方。 + - inference/sglang/run_llama3_8B.sh + # BERT-base GLUE 微调；硬绑定 CUDA，数据路径固定为 /data/GlueData。 + - training/BingBertGlue/run_glue_classifier_bert_base.py + # BERT-large GLUE 微调；硬绑定 CUDA，并含私人源码/数据路径。 + - training/BingBertGlue/run_glue_classifier_bert_large.py + # BERT-large SQuAD 基线训练；硬绑定 CUDA，需外部权重和数据。 + - training/BingBertSquad/nvidia_run_squad_baseline.py + # BERT-large SQuAD DeepSpeed 训练；上游配方需多机 MPI/NCCL 及 CUDA kernel。 + - training/BingBertSquad/nvidia_run_squad_deepspeed.py + # SQuAD HF 启动包装；目标 nvidia_run_squad_mpi.py 不在仓库内。 + - training/BingBertSquad/run_hf.sh + # Domino GPT-3 预训练；缺少 Megatron 子模块，需大模型和外部语料。 + - training/DeepSpeed-Domino/pretrain_gpt.py + # BingBERT 预训练；硬绑定 CUDA/NCCL，需多机及内部语料。 + - training/bing_bert/deepspeed_train.py + # BERT-base GLUE 微调；硬绑定 CUDA，数据路径固定为 /data/GlueData。 + - training/bing_bert/run_glue_classifier_bert_base.py + # BERT-large GLUE 微调；硬绑定 CUDA，数据路径固定为 /data/GlueData。 + - training/bing_bert/run_glue_classifier_bert_large.py + # GPT random-LTD 微调；本地数据可行，但必经 torch.device('cuda')/cuda.set_device。 + - training/data_efficiency/gpt_finetuning/run_clm_no_trainer.py + # CIFAR ViT 微调；损失与数据传输硬绑定 .cuda()，待适配。 + - training/data_efficiency/vit_finetuning/main_cifar.py + # ImageNet ViT 微调；本地小数据可行，但 criterion.cuda 和设备设置必经绑定。 + - training/data_efficiency/vit_finetuning/main_imagenet.py + # 训练评测编排；固定八卡训练/转换且不透传生成规模，子训练和 GSM8K/MMLU 已独立看护。 + - training/deepspeed_finetune_demo/run_and_evaluate.sh + # AutoEP MoE 训练；可缩层数和步数，但 metrics.reduce_max 必经 torch.cuda.current_device。 + - training/expert_parallel/train.py + # DCGAN 基线；可用 fake 数据，但设备仅 CUDA/CPU 且无条件 cuda.synchronize，无法原生 NPU。 + - training/gan/gan_baseline_train.py + # DCGAN DeepSpeed 训练；硬绑定 torch.cuda.set_device，需 CelebA 数据。 + - training/gan/gan_deepspeed_train.py + # ResNet 分类；本地 ImageFolder 可行，但 cuda.set_device/losses.cuda 无切换参数。 + - training/imagenet/main.py + # Stable Diffusion 多卡推理；设备与随机生成器硬绑定 CUDA。 + - training/stable_diffusion/inf_txt2img_loop.py + # 基础 AutoTP 训练；设备硬编码 cuda，默认 Llama-3.1-8B。 + - training/tensor_parallel/basic_example/autotp_example.py + # AutoTP 显存对比；硬绑定 torch.cuda，默认 Llama-3.1-8B。 + - training/tensor_parallel/basic_example/autotp_memory_compare.py + # 自定义 AutoTP 切分训练；设备硬绑定 CUDA，默认 Pythia-6.9B。 + - training/tensor_parallel/custom_patterns/autotp_custom_patterns.py + # DeepCompile 激活卸载；compile.enabled 与主仓配置不符且未调用 engine.compile，需上游修复。 + - benchmarks/pin_memory/deepcompile_activation/bench.py diff --git a/examples-guards/deepspeed/fixtures/ci_alpaca_16.json b/examples-guards/deepspeed/fixtures/ci_alpaca_16.json new file mode 100644 index 000000000..c0174cb6b --- /dev/null +++ b/examples-guards/deepspeed/fixtures/ci_alpaca_16.json @@ -0,0 +1,18 @@ +[ + {"instruction": "Add the numbers.", "input": "2 + 3", "output": "5."}, + {"instruction": "Subtract the numbers.", "input": "9 - 4", "output": "5."}, + {"instruction": "Multiply the numbers.", "input": "3 * 4", "output": "12."}, + {"instruction": "Divide the numbers.", "input": "8 / 2", "output": "4."}, + {"instruction": "Name the next number.", "input": "1, 2, 3", "output": "4."}, + {"instruction": "Choose the larger number.", "input": "7 or 2", "output": "7."}, + {"instruction": "Choose the smaller number.", "input": "6 or 9", "output": "6."}, + {"instruction": "Count the words.", "input": "The cat sleeps", "output": "3."}, + {"instruction": "Write the plural.", "input": "book", "output": "books."}, + {"instruction": "Write the opposite.", "input": "hot", "output": "cold."}, + {"instruction": "Name a primary color.", "input": "", "output": "Blue."}, + {"instruction": "Name a common fruit.", "input": "", "output": "Apple."}, + {"instruction": "Complete the sentence.", "input": "Birds can", "output": "fly."}, + {"instruction": "Write a greeting.", "input": "", "output": "Hello."}, + {"instruction": "Translate into English.", "input": "bonjour", "output": "Hello."}, + {"instruction": "Return the last letter.", "input": "cat", "output": "t."} +] diff --git a/examples-guards/deepspeed/fixtures/ci_dpo_8.json b/examples-guards/deepspeed/fixtures/ci_dpo_8.json new file mode 100644 index 000000000..759c7d4b7 --- /dev/null +++ b/examples-guards/deepspeed/fixtures/ci_dpo_8.json @@ -0,0 +1,8 @@ +{"prompt": " Human: q0 Assistant:", "chosen": " c0", "rejected": " r0"} +{"prompt": " Human: q1 Assistant:", "chosen": " c1", "rejected": " r1"} +{"prompt": " Human: q2 Assistant:", "chosen": " c2", "rejected": " r2"} +{"prompt": " Human: q3 Assistant:", "chosen": " c3", "rejected": " r3"} +{"prompt": " Human: q4 Assistant:", "chosen": " c4", "rejected": " r4"} +{"prompt": " Human: q5 Assistant:", "chosen": " c5", "rejected": " r5"} +{"prompt": " Human: q6 Assistant:", "chosen": " c6", "rejected": " r6"} +{"prompt": " Human: q7 Assistant:", "chosen": " c7", "rejected": " r7"} diff --git a/examples-guards/deepspeed/fixtures/ci_gsm8k_8.jsonl b/examples-guards/deepspeed/fixtures/ci_gsm8k_8.jsonl new file mode 100644 index 000000000..2b2a1006e --- /dev/null +++ b/examples-guards/deepspeed/fixtures/ci_gsm8k_8.jsonl @@ -0,0 +1,8 @@ +{"question":"There are 3 red balls and 4 blue balls. How many balls are there?","answer":"3 + 4 = 7.\n#### 7"} +{"question":"A box has 12 pencils. Sam takes 5. How many pencils remain?","answer":"12 - 5 = 7.\n#### 7"} +{"question":"Each bag has 4 apples. How many apples are in 3 bags?","answer":"4 * 3 = 12.\n#### 12"} +{"question":"Eight cookies are shared equally by 2 children. How many does each get?","answer":"8 / 2 = 4.\n#### 4"} +{"question":"A bus has 6 people. Two more get on. How many people are on the bus?","answer":"6 + 2 = 8.\n#### 8"} +{"question":"Lee has 10 coins and spends 3. How many coins are left?","answer":"10 - 3 = 7.\n#### 7"} +{"question":"A shelf holds 5 books. How many books are on 2 such shelves?","answer":"5 * 2 = 10.\n#### 10"} +{"question":"Nine flowers are put equally into 3 vases. How many flowers are in each vase?","answer":"9 / 3 = 3.\n#### 3"} diff --git a/examples-guards/deepspeed/fixtures/ci_mmlu_8.jsonl b/examples-guards/deepspeed/fixtures/ci_mmlu_8.jsonl new file mode 100644 index 000000000..ff1e9b926 --- /dev/null +++ b/examples-guards/deepspeed/fixtures/ci_mmlu_8.jsonl @@ -0,0 +1,8 @@ +{"question":"What is 2 + 3?","choices":["4","5","6","7"],"answer":1} +{"question":"Which planet is closest to the Sun?","choices":["Earth","Mars","Mercury","Jupiter"],"answer":2} +{"question":"What is the capital of France?","choices":["Paris","Rome","Tokyo","Berlin"],"answer":0} +{"question":"Which animal is a mammal?","choices":["Shark","Frog","Eagle","Whale"],"answer":3} +{"question":"How many sides does a triangle have?","choices":["2","3","4","5"],"answer":1} +{"question":"Which substance is water?","choices":["CO2","O2","H2O","NaCl"],"answer":2} +{"question":"What is 12 divided by 4?","choices":["3","4","6","8"],"answer":0} +{"question":"Which direction is opposite to north?","choices":["East","West","Up","South"],"answer":3} diff --git a/examples-guards/deepspeed/fixtures/ci_mnli_8.jsonl b/examples-guards/deepspeed/fixtures/ci_mnli_8.jsonl new file mode 100644 index 000000000..fc0c079f7 --- /dev/null +++ b/examples-guards/deepspeed/fixtures/ci_mnli_8.jsonl @@ -0,0 +1,8 @@ +{"sentence1":"A child is holding a red ball.","sentence2":"A child has a ball.","label":0} +{"sentence1":"Two people are sitting outside.","sentence2":"The people are drinking tea.","label":1} +{"sentence1":"The room is empty.","sentence2":"There are people in the room.","label":2} +{"sentence1":"A dog is running in a park.","sentence2":"An animal is running.","label":0} +{"sentence1":"A woman is reading a book.","sentence2":"The book is about history.","label":1} +{"sentence1":"The door is closed.","sentence2":"The door is open.","label":2} +{"sentence1":"Three students are studying together.","sentence2":"Some students are studying.","label":0} +{"sentence1":"A man walks along a street.","sentence2":"The man is going to work.","label":1} diff --git a/examples-guards/deepspeed/fixtures/ci_rac_8.jsonl b/examples-guards/deepspeed/fixtures/ci_rac_8.jsonl new file mode 100644 index 000000000..41195f5fb --- /dev/null +++ b/examples-guards/deepspeed/fixtures/ci_rac_8.jsonl @@ -0,0 +1,8 @@ +{"prompt":"A garden has three rows of flowers with four flowers in each row. Two flowers are picked and three new flowers are planted. Calculate the number of flowers remaining, showing the multiplication, subtraction, and addition in order."} +{"prompt":"A train leaves the station at nine in the morning and takes two hours to reach the next city. After a thirty minute stop it travels for another hour. Determine its final arrival time and explain each time interval."} +{"prompt":"A shop sells notebooks for two dollars each and pencils for one dollar each. Maya buys three notebooks and four pencils, then pays with a twenty dollar note. Calculate the total cost and the change she should receive."} +{"prompt":"A rectangle is eight meters long and five meters wide. A square with sides of two meters is removed from one corner. Find the area of the original rectangle, the removed square, and the remaining shape using simple arithmetic."} +{"prompt":"Three friends share eighteen apples equally. One friend gives two of her apples to another friend. State how many apples each friend has before and after the transfer, and verify that the total number of apples remains unchanged."} +{"prompt":"A library has five shelves containing ten books each. Six books are borrowed on Monday and four books are returned on Tuesday. Calculate how many books are now on the shelves and describe why borrowing decreases the total while returns increase it."} +{"prompt":"A runner completes one lap in three minutes. She runs four laps, rests for two minutes, and then runs two more laps at the same speed. Calculate the total running time and the total elapsed time including the rest."} +{"prompt":"A farmer collects twenty four eggs and packs them into boxes holding six eggs each. Two boxes are sold and one box is given to a neighbor. Work out the number of boxes initially filled and the number of eggs still available afterward."} diff --git a/examples-guards/deepspeed/fixtures/ci_rlhf_8.json b/examples-guards/deepspeed/fixtures/ci_rlhf_8.json new file mode 100644 index 000000000..759c7d4b7 --- /dev/null +++ b/examples-guards/deepspeed/fixtures/ci_rlhf_8.json @@ -0,0 +1,8 @@ +{"prompt": " Human: q0 Assistant:", "chosen": " c0", "rejected": " r0"} +{"prompt": " Human: q1 Assistant:", "chosen": " c1", "rejected": " r1"} +{"prompt": " Human: q2 Assistant:", "chosen": " c2", "rejected": " r2"} +{"prompt": " Human: q3 Assistant:", "chosen": " c3", "rejected": " r3"} +{"prompt": " Human: q4 Assistant:", "chosen": " c4", "rejected": " r4"} +{"prompt": " Human: q5 Assistant:", "chosen": " c5", "rejected": " r5"} +{"prompt": " Human: q6 Assistant:", "chosen": " c6", "rejected": " r6"} +{"prompt": " Human: q7 Assistant:", "chosen": " c7", "rejected": " r7"} diff --git a/examples-guards/deepspeed/fixtures/ci_rw_8.json b/examples-guards/deepspeed/fixtures/ci_rw_8.json new file mode 100644 index 000000000..759c7d4b7 --- /dev/null +++ b/examples-guards/deepspeed/fixtures/ci_rw_8.json @@ -0,0 +1,8 @@ +{"prompt": " Human: q0 Assistant:", "chosen": " c0", "rejected": " r0"} +{"prompt": " Human: q1 Assistant:", "chosen": " c1", "rejected": " r1"} +{"prompt": " Human: q2 Assistant:", "chosen": " c2", "rejected": " r2"} +{"prompt": " Human: q3 Assistant:", "chosen": " c3", "rejected": " r3"} +{"prompt": " Human: q4 Assistant:", "chosen": " c4", "rejected": " r4"} +{"prompt": " Human: q5 Assistant:", "chosen": " c5", "rejected": " r5"} +{"prompt": " Human: q6 Assistant:", "chosen": " c6", "rejected": " r6"} +{"prompt": " Human: q7 Assistant:", "chosen": " c7", "rejected": " r7"} diff --git a/examples-guards/deepspeed/fixtures/ci_sft_8.json b/examples-guards/deepspeed/fixtures/ci_sft_8.json new file mode 100644 index 000000000..759c7d4b7 --- /dev/null +++ b/examples-guards/deepspeed/fixtures/ci_sft_8.json @@ -0,0 +1,8 @@ +{"prompt": " Human: q0 Assistant:", "chosen": " c0", "rejected": " r0"} +{"prompt": " Human: q1 Assistant:", "chosen": " c1", "rejected": " r1"} +{"prompt": " Human: q2 Assistant:", "chosen": " c2", "rejected": " r2"} +{"prompt": " Human: q3 Assistant:", "chosen": " c3", "rejected": " r3"} +{"prompt": " Human: q4 Assistant:", "chosen": " c4", "rejected": " r4"} +{"prompt": " Human: q5 Assistant:", "chosen": " c5", "rejected": " r5"} +{"prompt": " Human: q6 Assistant:", "chosen": " c6", "rejected": " r6"} +{"prompt": " Human: q7 Assistant:", "chosen": " c7", "rejected": " r7"} diff --git a/examples-guards/deepspeed/scripts/run_example.sh b/examples-guards/deepspeed/scripts/run_example.sh new file mode 100644 index 000000000..3baef647d --- /dev/null +++ b/examples-guards/deepspeed/scripts/run_example.sh @@ -0,0 +1,1228 @@ +#!/usr/bin/env bash +# Run one example from a CI working copy of the examples tree. +# $1 is the manifest entry path. EXEC, when set, names the launchable file +# relative to the examples root; otherwise path itself must be launchable. +# Overlay CLI args come from OVERLAY_ARGS (JSON array). Never git add/commit/push. +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +EXAMPLE_REL="$1" +TARGET_ROOT="${TARGET_ROOT:?TARGET_ROOT is required}" +CI_OUTPUT_DIR="${CI_OUTPUT_DIR:?CI_OUTPUT_DIR is required}" + +# Examples tree root: split mode sets EXAMPLES_ROOT (the DeepSpeedExamples +# checkout); non-split it falls back to TARGET_ROOT. +EXAMPLES_ROOT="${EXAMPLES_ROOT:-$TARGET_ROOT}" + +EXAMPLE_PATH="$EXAMPLES_ROOT/$EXAMPLE_REL" +if [[ ! -e "$EXAMPLE_PATH" ]]; then + echo "example not found: $EXAMPLE_PATH" >&2 + exit 1 +fi + +# Resolve the launchable file: EXEC (relative to the examples root) when set, +# otherwise path itself. +if [[ -n "${EXEC:-}" ]]; then + LAUNCH_PATH="$EXAMPLES_ROOT/$EXEC" +else + LAUNCH_PATH="$EXAMPLE_PATH" +fi +if [[ ! -f "$LAUNCH_PATH" ]]; then + echo "launchable file not found: $LAUNCH_PATH" >&2 + exit 1 +fi + +mkdir -p "$CI_OUTPUT_DIR" + +source /usr/local/Ascend/ascend-toolkit/set_env.sh + +if command -v python3 >/dev/null 2>&1; then + PYTHON=python3 +else + PYTHON=python +fi + +expand_overlay() { + "$PYTHON" - <<'PY' +import json +import os +import shlex + +raw = os.environ.get('OVERLAY_ARGS', '').strip() +if not raw or raw in ('null', '""'): + raise SystemExit(0) +try: + items = json.loads(raw) +except json.JSONDecodeError as exc: + raise SystemExit(f'OVERLAY_ARGS is not valid JSON: {exc}') from exc +if items in (None, ''): + raise SystemExit(0) +if not isinstance(items, list): + raise SystemExit( + f'OVERLAY_ARGS must be a JSON array, got {type(items).__name__}') +tokens = [] +for item in items: + if not isinstance(item, str): + raise SystemExit( + f'OVERLAY_ARGS items must be strings, got {type(item).__name__}') + tokens.extend(shlex.split(os.path.expandvars(item), posix=True)) +print(' '.join(shlex.quote(token) for token in tokens)) +PY +} + +eval "EXTRA_ARGS=( $(expand_overlay) )" + +echo "running $LAUNCH_PATH with ${#EXTRA_ARGS[@]} overlay args" +if ((${#EXTRA_ARGS[@]})); then + printf 'overlay arg: %q\n' "${EXTRA_ARGS[@]}" +fi + +cd "$(dirname "$LAUNCH_PATH")" +export CI_OUTPUT_DIR +export HF_ENDPOINT="${HF_ENDPOINT:-https://hf-mirror.com}" + +entry_key="${EXEC:-$EXAMPLE_REL}" +if [[ "$entry_key" == benchmarks/opsd/benchmark_hybrid_engine_rollout.py ]]; then + echo 'unsupported: native NPU softmax_context interface mismatch (Run #30); upstream fix required' >&2 + exit 2 +fi + +require_visible_devices() { + local required="$1" + local default_devices="$2" + local -a devices + + if [[ -z "${ASCEND_RT_VISIBLE_DEVICES:-}" ]]; then + export ASCEND_RT_VISIBLE_DEVICES="$default_devices" + fi + IFS=',' read -r -a devices <<< "$ASCEND_RT_VISIBLE_DEVICES" + if ((${#devices[@]} < required)); then + echo "insufficient NPU devices for $entry_key: required=$required visible=$ASCEND_RT_VISIBLE_DEVICES" >&2 + exit 1 + fi + echo "NPU devices for $entry_key: required=$required visible=$ASCEND_RT_VISIBLE_DEVICES" +} + +first_visible_devices() { + local count="$1" + local -a devices + IFS=',' read -r -a devices <<< "$ASCEND_RT_VISIBLE_DEVICES" + local selected="${devices[0]}" + local index + for ((index = 1; index < count; index++)); do + selected+=",${devices[index]}" + done + printf '%s\n' "$selected" +} + +master_port_for() { + local offset="$1" + local run_id="${GITHUB_RUN_ID:-0}" + printf '%s\n' "$((20000 + run_id % 20000 + offset))" +} + +require_overlay_path() { + local flag="$1" + local kind="$2" + local index value='' + for ((index = 0; index < ${#EXTRA_ARGS[@]}; index++)); do + if [[ "${EXTRA_ARGS[index]}" == "$flag" ]]; then + value="${EXTRA_ARGS[index + 1]:-}" + break + fi + done + if [[ -z "$value" ]] || + { [[ "$kind" == directory ]] && [[ ! -d "$value" ]]; } || + { [[ "$kind" == file ]] && [[ ! -f "$value" ]]; }; then + echo "missing local $kind for $entry_key: $flag=${value:-}" >&2 + exit 1 + fi +} + +run_cifar_moe() { + require_visible_devices 2 '0,1' + local devices + devices="$(first_visible_devices 2)" + echo "reproducing upstream run_ds_moe.sh on devices $devices" + echo "disabling optional torch.compile for DeepSpeed MoE helpers; eager MoE/EP execution is preserved" + TORCH_COMPILE_DISABLE=1 \ + ASCEND_RT_VISIBLE_DEVICES="$devices" \ + deepspeed \ + --master_port "$(master_port_for 2)" \ + --num_nodes 1 \ + --num_gpus 2 \ + --bind_cores_to_rank \ + "$EXAMPLES_ROOT/training/cifar/cifar10_deepspeed.py" \ + --deepspeed \ + --moe \ + --ep-world-size 2 \ + --num-experts 2 \ + --top-k 1 \ + --noisy-gate-policy RSample \ + --moe-param-group \ + "${EXTRA_ARGS[@]}" +} + +run_cifar_prmoe() { + require_visible_devices 2 '0,1' + local devices + devices="$(first_visible_devices 2)" + echo "reproducing upstream run_ds_prmoe.sh on devices $devices: EP=2, experts=2/4, residual MoE" + echo "disabling optional torch.compile for DeepSpeed MoE helpers; eager PR-MoE/EP execution is preserved" + TORCH_COMPILE_DISABLE=1 \ + ASCEND_RT_VISIBLE_DEVICES="$devices" \ + deepspeed \ + --master_port "$(master_port_for 3)" \ + --num_nodes 1 \ + --num_gpus 2 \ + "$EXAMPLES_ROOT/training/cifar/cifar10_deepspeed.py" \ + --deepspeed \ + --moe \ + --ep-world-size 2 \ + --num-experts 2 4 \ + --top-k 1 \ + --mlp-type residual \ + --noisy-gate-policy RSample \ + --moe-param-group \ + "${EXTRA_ARGS[@]}" +} + +run_hf_autotp() { + require_visible_devices 8 '0,1,2,3,4,5,6,7' + require_overlay_path --model_name_or_path directory + require_overlay_path --data_path file + local devices config_path + devices="$(first_visible_devices 8)" + config_path="$CI_OUTPUT_DIR/hf-autotp-config.json" + "$PYTHON" - "$EXAMPLES_ROOT/training/tensor_parallel/hf_integration/configs/ds_config_temp.json" "$config_path" <<'PY' +import json +import sys +from pathlib import Path + +template = Path(sys.argv[1]).read_text() +config = json.loads(template.replace('${zero_stage}', '0').replace('${autotp_size}', '8')) +if config['zero_optimization']['stage'] != 0 or config['tensor_parallel']['autotp_size'] != 8: + raise SystemExit('HF AutoTP CI requires ZeRO=0 and AutoTP=8') +Path(sys.argv[2]).write_text(json.dumps(config, indent=2) + '\n') +PY + # The upstream example caches tokenized data in its working directory and + # unconditionally saves the final model. Keep all generated files in CI output. + if [[ ! -s "$CI_OUTPUT_DIR/hf-autotp-work/dataset_dict.pkl" ]]; then + echo "HF AutoTP shared tokenization cache missing; run ds_hf_autotp setup first" >&2 + exit 1 + fi + mkdir -p "$CI_OUTPUT_DIR/hf-autotp-work" + cd "$CI_OUTPUT_DIR/hf-autotp-work" + echo "running HF Trainer AutoTP=8 on devices $devices with config $config_path" + ASCEND_RT_VISIBLE_DEVICES="$devices" WANDB_MODE=disabled deepspeed \ + --master_port "$(master_port_for 28)" \ + --num_gpus 8 \ + --no_local_rank \ + "$LAUNCH_PATH" \ + --deepspeed "$config_path" \ + "${EXTRA_ARGS[@]}" +} + +overlay_value() { + local flag="$1" index + for ((index = 0; index < ${#EXTRA_ARGS[@]}; index++)); do + if [[ "${EXTRA_ARGS[index]}" == "$flag" ]]; then + printf '%s\n' "${EXTRA_ARGS[index + 1]:-}" + return 0 + fi + done + return 1 +} + +run_chat_prompt_eval() { + require_visible_devices 1 '0' + require_overlay_path --model_name_or_path_baseline directory + local devices baseline checkpoint finetune + devices="$(first_visible_devices 1)" + baseline="$(overlay_value --model_name_or_path_baseline)" + checkpoint="$CI_OUTPUT_DIR/prompt-eval-sft" + finetune="$(overlay_value --model_name_or_path_finetune)" + if [[ "$finetune" != "$checkpoint" ]]; then + echo "prompt eval requires the same-job SFT checkpoint: expected=$checkpoint actual=$finetune" >&2 + exit 1 + fi + if [[ -e "$checkpoint/config.json" ]] || [[ -e "$checkpoint/pytorch_model.bin" ]]; then + echo "prompt eval checkpoint already exists; refusing to reuse stale training output: $checkpoint" >&2 + exit 1 + fi + echo "training a genuine SFT checkpoint before baseline/finetuned prompt evaluation" + ASCEND_RT_VISIBLE_DEVICES="$devices" deepspeed \ + --master_port "$(master_port_for 51)" --num_gpus 1 \ + "$EXAMPLES_ROOT/applications/DeepSpeed-Chat/training/step1_supervised_finetuning/main.py" \ + --model_name_or_path "$baseline" --data_path local/jsonfile \ + --num_train_epochs 1 --per_device_train_batch_size 4 \ + --per_device_eval_batch_size 4 --max_seq_len 128 \ + --zero_stage 2 --dtype bf16 --print_loss --deepspeed \ + --data_output_path "$CI_OUTPUT_DIR/prompt-eval-data" \ + --output_dir "$checkpoint" + # save_hf_format in the unchanged SFT entry writes these exact two files. + if [[ ! -s "$checkpoint/config.json" ]] || [[ ! -s "$checkpoint/pytorch_model.bin" ]]; then + echo "SFT did not produce a loadable checkpoint for prompt eval: $checkpoint" >&2 + exit 1 + fi + echo "evaluating baseline against the newly trained SFT checkpoint on NPU $devices" + ASCEND_RT_VISIBLE_DEVICES="$devices" "$PYTHON" "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" +} + +run_chat_reward_eval() { + require_visible_devices 1 '0' + require_overlay_path --model_name_or_path directory + local devices + devices="$(first_visible_devices 1)" + echo "running upstream reward-head initialization/forward smoke on NPU $devices; not trained-head restoration" + ASCEND_RT_VISIBLE_DEVICES="$devices" "$PYTHON" "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" \ + | tee "$CI_OUTPUT_DIR/reward-eval.log" + "$PYTHON" - "$CI_OUTPUT_DIR/reward-eval.log" "$CI_OUTPUT_DIR/reward-eval-scores.json" <<'PY' +import json +import math +import re +import sys +from pathlib import Path + +scores = [] +for line in Path(sys.argv[1]).read_text().splitlines(): + match = re.fullmatch(r'\s*(good_ans|bad_ans) score:\s*(\S+)\s*', line) + if match: + value = float(match.group(2)) + if not math.isfinite(value): + raise SystemExit('reward evaluation produced a non-finite score') + scores.append({'kind': match.group(1), 'score': value}) +if [score['kind'] for score in scores] != ['good_ans', 'bad_ans'] * 2: + raise SystemExit('reward evaluation must produce four scores for the two upstream preference pairs') +# An initialized reward head has no quality ordering guarantee; finite forward +# results, not good>bad, are the smoke acceptance criterion. +Path(sys.argv[2]).write_text(json.dumps(scores, indent=2, allow_nan=False) + '\n') +print('validated four finite reward scores (initialized-head forward smoke)') +PY +} + +run_hf_bench_length() { + require_visible_devices 8 '0,1,2,3,4,5,6,7' + require_overlay_path --model_name_or_path directory + require_overlay_path --data_path file + local devices config_path + devices="$(first_visible_devices 8)" + config_path="$CI_OUTPUT_DIR/hf-bench-length-config.json" + if [[ "$(overlay_value --model_max_length)" != 128 ]]; then + echo "HF fixed-length CI recipe requires --model_max_length 128" >&2 + exit 1 + fi + if [[ ! -s "$CI_OUTPUT_DIR/hf-bench-length-work/dataset_dict128.pkl" ]]; then + echo "HF fixed-length shared tokenization cache missing; run ds_hf_bench_length setup first" >&2 + exit 1 + fi + "$PYTHON" - "$EXAMPLES_ROOT/training/tensor_parallel/hf_integration/configs/ds_config_temp.json" "$config_path" <<'PY' +import json +import sys +from pathlib import Path + +template = Path(sys.argv[1]).read_text() +config = json.loads(template.replace('${zero_stage}', '0').replace('${autotp_size}', '8')) +if config['zero_optimization']['stage'] != 0 or config['tensor_parallel']['autotp_size'] != 8: + raise SystemExit('HF fixed-length CI requires ZeRO=0 and AutoTP=8') +Path(sys.argv[2]).write_text(json.dumps(config, indent=2) + '\n') +PY + cd "$CI_OUTPUT_DIR/hf-bench-length-work" + echo "running fixed-length HF Trainer AutoTP=8 on devices $devices" + ASCEND_RT_VISIBLE_DEVICES="$devices" WANDB_MODE=disabled deepspeed \ + --master_port "$(master_port_for 61)" --num_gpus 8 --no_local_rank \ + "$LAUNCH_PATH" --deepspeed "$config_path" "${EXTRA_ARGS[@]}" +} + +run_superoffload() { + require_visible_devices 2 '0,1' + require_overlay_path --model_name directory + require_overlay_path --dataset_name directory + local devices config_path + devices="$(first_visible_devices 2)" + config_path="$CI_OUTPUT_DIR/superoffload-config.json" + "$PYTHON" - "$config_path" <<'PY' +import json +import sys +from pathlib import Path + +# The unchanged entry creates DeepSpeedCPUAdam itself (LR=0.001); do not +# provide a duplicate optimizer or claim that the logging-only --lr sets it. +config = { + 'train_batch_size': 2, + 'train_micro_batch_size_per_gpu': 1, + 'gradient_accumulation_steps': 1, + 'bf16': {'enabled': True}, + 'zero_optimization': { + 'stage': 3, + 'overlap_comm': False, + 'contiguous_gradients': True, + 'reduce_bucket_size': 65536, + 'stage3_prefetch_bucket_size': 65536, + 'stage3_param_persistence_threshold': 4096, + 'offload_param': {'device': 'cpu', 'pin_memory': False}, + 'offload_optimizer': {'device': 'cpu', 'pin_memory': False}, + }, + 'steps_per_print': 1, + 'wall_clock_breakdown': True, +} +Path(sys.argv[1]).write_text(json.dumps(config, indent=2) + '\n') +PY + mkdir -p "$CI_OUTPUT_DIR/superoffload-work" + cd "$CI_OUTPUT_DIR/superoffload-work" + echo "running SuperOffload entry / ZeRO-Offload smoke on devices $devices; native super_offload is not enabled" + ASCEND_RT_VISIBLE_DEVICES="$devices" WANDB_MODE=disabled deepspeed \ + --master_port "$(master_port_for 71)" --num_gpus 2 \ + "$LAUNCH_PATH" --deepspeed --deepspeed_config "$config_path" "${EXTRA_ARGS[@]}" \ + 2>&1 | tee "$CI_OUTPUT_DIR/superoffload.log" + "$PYTHON" - "$CI_OUTPUT_DIR/superoffload.log" "$CI_OUTPUT_DIR/superoffload-losses.json" <<'PY' +import json +import math +import re +import sys +from pathlib import Path + +rank_zero_losses = {} +for line in Path(sys.argv[1]).read_text().splitlines(): + match = re.search(r'Step\s+(\d+)\s*\|\s*Loss:\s*(\S+)\s*\|', line) + if not match: + continue + step, loss = int(match.group(1)), float(match.group(2)) + if not math.isfinite(loss): + raise SystemExit(f'SuperOffload entry produced a non-finite loss at step {step}') + # Upstream adds the timestamped handler only on rank 0. Its root logging + # handler can repeat messages on other ranks, so tolerate duplicates and + # check every reported loss, but require rank-0 evidence for all 3 steps. + if ' - finetune_zero3 - INFO - ' in line: + rank_zero_losses[step] = loss +if sorted(rank_zero_losses) != [1, 2, 3]: + raise SystemExit('SuperOffload entry must report rank-0 finite losses for steps 1, 2 and 3') +report = [{'step': step, 'loss': rank_zero_losses[step]} for step in sorted(rank_zero_losses)] +Path(sys.argv[2]).write_text(json.dumps(report, indent=2, allow_nan=False) + '\n') +print('validated three finite SuperOffload entry train-step losses (ZeRO-Offload smoke)') +PY +} + +run_variable_batch() { + require_visible_devices 1 '0' + local devices + devices="$(first_visible_devices 1)" + mkdir -p "$CI_OUTPUT_DIR/variable-batch-work" + cd "$CI_OUTPUT_DIR/variable-batch-work" + echo "running upstream dynamic sequence packing and LR scaling on device $devices" + ASCEND_RT_VISIBLE_DEVICES="$devices" deepspeed \ + --master_port "$(master_port_for 31)" \ + --num_gpus 1 \ + --no_local_rank \ + "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" +} + +run_zenflow() { + require_visible_devices 2 '0,1' + local devices + devices="$(first_visible_devices 2)" + mkdir -p "$CI_OUTPUT_DIR/zenflow-work" + cd "$CI_OUTPUT_DIR/zenflow-work" + echo "running two-rank ZenFlow optimizer-offload smoke on devices $devices" + ASCEND_RT_VISIBLE_DEVICES="$devices" deepspeed \ + --master_port "$(master_port_for 42)" \ + --num_gpus 2 \ + "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" +} + +run_rac_prune() { + require_visible_devices 1 '0' + require_overlay_path --model directory + require_overlay_path --dataset file + local devices + devices="$(first_visible_devices 1)" + echo "running upstream calibration-based pruning on NPU $devices" + # Import the vendor backend normally before executing the unchanged entry. + # runpy needs the sibling rac package on sys.path, as ordinary python does. + ASCEND_RT_VISIBLE_DEVICES="$devices" "$PYTHON" -c ' +import os +import runpy +import sys +import torch_npu +sys.argv = sys.argv[1:] +sys.path.insert(0, os.path.dirname(sys.argv[0])) +runpy.run_path(sys.argv[0], run_name="__main__") +' "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" +} + +run_autotp_equivalence() { + require_visible_devices 4 '0,1,2,3' + : "${QWEN3_06B_PATH:?QWEN3_06B_PATH was not exported by setup}" + + local example_dir="$EXAMPLES_ROOT/training/autotp_equivalence" + local metrics_dir="$CI_OUTPUT_DIR/autotp-equivalence" + mkdir -p "$metrics_dir" + + local size devices + for size in 1 3 4; do + devices="$(first_visible_devices "$size")" + echo "running AutoTP=$size on ASCEND_RT_VISIBLE_DEVICES=$devices" + ASCEND_RT_VISIBLE_DEVICES="$devices" deepspeed \ + --master_port "$(master_port_for "$((10 + size))")" \ + --num_gpus "$size" \ + "$LAUNCH_PATH" \ + --deepspeed_config "$example_dir/configs/autotp${size}.json" \ + --metrics_file "$metrics_dir/autotp${size}.jsonl" \ + "${EXTRA_ARGS[@]}" + done + + "$PYTHON" "$example_dir/compare_loss.py" \ + "$metrics_dir/autotp1.jsonl" "$metrics_dir/autotp3.jsonl" \ + --print-every 1 + "$PYTHON" "$example_dir/compare_loss.py" \ + "$metrics_dir/autotp1.jsonl" "$metrics_dir/autotp4.jsonl" \ + --print-every 1 +} + +validate_expansion() { + "$PYTHON" - "$@" <<'PY' +"""Check upstream example outputs; no NPU execution or source interception here.""" +from __future__ import annotations + +import json +import math +from pathlib import Path +import re +import sys + + +def positive(value): + return isinstance(value, (float, int)) and math.isfinite(value) and value > 0 + + +def validate(kind: str, text: str, output: Path | None = None) -> object: + if kind in {"model_tensor_offload", "activation_offload"}: + summaries = [json.loads(line.split("=", 1)[1]) for line in text.splitlines() + if line.startswith("DRIVERRESULT=")] + if len(summaries) != 1: + raise ValueError("upstream driver must report exactly one two-arm summary") + report = summaries[0] + unpinned = "unpinned" if kind == "model_tensor_offload" else "pageable" + for name in (unpinned, "pinned"): + row = report[name] + pin_key = "pin_memory" if kind == "model_tensor_offload" else "use_pin_memory" + if (row["device"] != "npu" or row["steps"] != 3 or + bool(row[pin_key]) != (name == "pinned") or + row["experiment"] != kind or + not all(positive(row[key]) for key in ("step_avg_s", "step_min_s"))): + raise ValueError(f"invalid NPU {kind} arm: {row}") + if kind == "model_tensor_offload" and row["zero_stage"] != 3: + raise ValueError("model tensor offload must exercise ZeRO-3") + if not positive(report["speedup"]): + raise ValueError("speedup must be finite/positive, not necessarily greater than 1") + elif kind == "h2d_d2h": + report = [json.loads(line.split("=", 1)[1]) for line in text.splitlines() + if line.startswith("RESULT=")] + expected = {(arm, size) for arm in ("pageable", "torch", "native-unregistered") + for size in (1,)} + if len(report) != len(expected) or {(r["arm"], r["size_mib"]) for r in report} != expected: + raise ValueError("copy benchmark must report all three 1-MiB buffer modes") + for row in report: + if row["experiment"] != kind or not all(positive(row[k]) for k in ("h2d_gbps", "d2h_gbps")): + raise ValueError(f"invalid copy measurement: {row}") + if bool(row["accelerator_is_pinned"]) != (row["arm"] != "pageable"): + raise ValueError(f"buffer pinning did not match the requested arm: {row}") + elif kind == "zenflow_finetune": + report = {int(step): float(loss) for step, loss in + re.findall(r"Step\s+(\d+), Loss:\s*([^,\s]+)", text)} + if sorted(report) != list(range(1, 17)) or not all(math.isfinite(v) for v in report.values()): + raise ValueError("ZenFlow finetune must complete 16 finite-loss optimizer updates") + if "Training complete!" not in text: + raise ValueError("ZenFlow did not complete checkpoint/tokenizer saving") + if output is not None: + if not (output / "latest").is_file() or not list(output.rglob("*model_states.pt")): + raise ValueError("missing upstream DeepSpeed checkpoint") + if not (output / "tokenizer_config.json").is_file(): + raise ValueError("missing saved upstream tokenizer") + elif kind == "opsd_student": + matches = re.findall(r"STUDENT_OK loss=(\S+) mem=\S+ offload=0 autotp=2 ws=2", text) + if len(matches) != 1 or not math.isfinite(float(matches[0])): + raise ValueError("student must complete finite-loss TP=2/ZeRO-3 forward/backward/step") + report = {"loss": float(matches[0]), "autotp": 2, "world_size": 2, "offload": False} + elif kind == "opsd_teacher": + matches = re.findall(r"TEACHER_OK shape=\(1, 12, (\d+)\) mem=\S+ offload=True autotp=2 ws=2", text) + if len(matches) != 1 or int(matches[0]) <= 0: + raise ValueError("teacher must create CPU logit cache from TP=2/ZeRO-3 offload forward") + report = {"cache_shape": [1, 12, int(matches[0])], "autotp": 2, "world_size": 2, "offload": True} + elif kind == "opsd": + report = [] + for step, loss, tokens in re.findall( + r"\[opsd\]\[step (\d+)\] loss=(\S+).*?resp_tok=(\d+)", text): + row = {"step": int(step), "loss": float(loss), "response_tokens": int(tokens)} + if not math.isfinite(row["loss"]) or row["response_tokens"] <= 0: + raise ValueError(f"invalid OPSD rollout/teacher/student step: {row}") + report.append(row) + if [row["step"] for row in report] != [0, 1, 2]: + raise ValueError("OPSD must complete exactly three rollout/distillation steps") + else: + raise ValueError(f"unknown expansion result kind: {kind}") + return report + + +if __name__ == "__main__": + kind, log, report_file, *extra = sys.argv[1:] + report = validate(kind, Path(log).read_text(), Path(extra[0]) if extra else None) + Path(report_file).write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + print(f"validated upstream {kind} results") +PY +} + +run_pin_memory_example() { + require_visible_devices 1 '0' + local experiment="$1" devices log + devices="$(first_visible_devices 1)" + log="$CI_OUTPUT_DIR/$experiment.log" + # Limits are per-process: setup's soft limit does not carry to this step. + local memlock_hard + memlock_hard="$(ulimit -Hl)" + ulimit -Sl "$memlock_hard" + # Upstream drivers own the arm subprocesses and allocate fresh rendezvous + # ports. Do not wrap the two-arm driver in a multi-rank launcher. + ASCEND_RT_VISIBLE_DEVICES="$devices" "$PYTHON" "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" 2>&1 | tee "$log" + validate_expansion \ + "$experiment" "$log" "$CI_OUTPUT_DIR/$experiment-results.json" +} + +run_zenflow_finetune() { + require_visible_devices 1 '0' + require_overlay_path --model_name directory + : "${ZENFLOW_WORK_DIR:?ZENFLOW_WORK_DIR was not exported by setup}" + [[ -s "$ZENFLOW_WORK_DIR/tatsu-lab/alpaca/train.parquet" ]] || { + echo 'ZenFlow local Alpaca parquet fixture is missing' >&2; exit 1; + } + local config="$CI_OUTPUT_DIR/zenflow-finetune-config.json" devices + devices="$(first_visible_devices 1)" + "$PYTHON" - "$config" <<'PY' +import json +from pathlib import Path +import sys +config = { + "train_batch_size": 1, "train_micro_batch_size_per_gpu": 1, + "gradient_accumulation_steps": 1, "gradient_clipping": 1.0, + "bf16": {"enabled": True}, "zero_allow_untested_optimizer": True, + "zero_optimization": {"stage": 2, + "offload_optimizer": {"device": "cpu", "pin_memory": False}, + "zenflow": {"topk_ratio": 0.1, "update_interval": 2, + "full_warm_up_rounds": 0, "overlap_step": False}}, + "optimizer": {"type": "AdamW", "params": {"lr": 2e-5, + "betas": [0.9, 0.999], "eps": 1e-8, "weight_decay": 0.01}}, +} +Path(sys.argv[1]).write_text(json.dumps(config, indent=2) + '\n') +PY + cd "$ZENFLOW_WORK_DIR" + # Native datasets local-directory resolution of the original fixed ID. + # Offline mode makes any accidental fallback to the Hub fail explicitly. + ASCEND_RT_VISIBLE_DEVICES="$devices" HF_DATASETS_OFFLINE=1 HF_HUB_OFFLINE=1 \ + deepspeed --master_port "$(master_port_for 81)" --num_gpus 1 \ + "$LAUNCH_PATH" --deepspeed --deepspeed_config "$config" "${EXTRA_ARGS[@]}" \ + 2>&1 | tee "$CI_OUTPUT_DIR/zenflow-finetune.log" + validate_expansion zenflow_finetune \ + "$CI_OUTPUT_DIR/zenflow-finetune.log" "$CI_OUTPUT_DIR/zenflow-finetune-results.json" \ + "$(overlay_value --output_dir)" +} + +run_opsd() { + require_visible_devices 1 '0' + : "${QWEN25_STUDENT_PATH:?QWEN25_STUDENT_PATH was not exported by setup}" + : "${QWEN25_TEACHER_PATH:?QWEN25_TEACHER_PATH was not exported by setup}" + : "${OPSD_CI_PATH:?OPSD_CI_PATH was not exported by setup}" + [[ -d "$QWEN25_STUDENT_PATH" && -d "$QWEN25_TEACHER_PATH" && -s "$OPSD_CI_PATH" ]] || { + echo 'OPSD requires two local ModelScope models and the local prompt fixture' >&2; exit 1; + } + local config="$CI_OUTPUT_DIR/opsd-config.json" devices + devices="$(first_visible_devices 1)" + "$PYTHON" - "$config" <<'PY' +import json +import os +from pathlib import Path +import sys +output = Path(os.environ['CI_OUTPUT_DIR']) +ds_config = output / 'opsd-student-config.json' +ds_config.write_text(json.dumps({ + 'train_micro_batch_size_per_gpu': 1, 'gradient_accumulation_steps': 1, + 'bf16': {'enabled': True}, 'zero_optimization': {'stage': 0}, + 'optimizer': {'type': 'AdamW', 'params': {'lr': 1e-6}}, + 'gradient_clipping': 1.0, +}, indent=2) + '\n') +cfg = { + 'student': {'model_name_or_path': os.environ['QWEN25_STUDENT_PATH'], 'dtype': 'bfloat16'}, + 'teacher': {'model_name_or_path': os.environ['QWEN25_TEACHER_PATH'], 'dtype': 'bfloat16', + 'offload_to_cpu': True, 'autotp_size': 1}, + 'rollout': {'engine': 'hybrid_engine', 'max_prompt_length': 512, + 'max_response_length': 8, 'temperature': 0.0, 'n_samples_per_prompt': 1}, + 'distillation': {'loss_type': 'reverse_kl', 'temperature': 1.0, 'chunk_size': 8}, + 'training': {'micro_batch_size_per_gpu': 1, 'gradient_accumulation_steps': 1, + 'learning_rate': 1e-6, 'num_train_epochs': 1, 'max_steps': 3, + 'logging_steps': 1, 'save_steps': 500, 'save_dir': str(output / 'opsd-checkpoints')}, + 'data': {'path': os.environ['OPSD_CI_PATH'], 'prompt_field': 'prompt', 'shuffle': False}, + 'deepspeed_config': str(ds_config), +} +Path(sys.argv[1]).write_text(json.dumps(cfg, indent=2) + '\n') +PY + # Default rollout delegates to the original student module.generate(), + # then the original trainer does teacher-cache + streamed KL backward. + ASCEND_RT_VISIBLE_DEVICES="$devices" HF_HUB_OFFLINE=1 \ + deepspeed --master_port "$(master_port_for 91)" --num_gpus 1 \ + "$LAUNCH_PATH" --config "$config" "${EXTRA_ARGS[@]}" 2>&1 | tee "$CI_OUTPUT_DIR/opsd.log" + validate_expansion opsd \ + "$CI_OUTPUT_DIR/opsd.log" "$CI_OUTPUT_DIR/opsd-results.json" +} + +run_opsd_isolated() { + local role="$1" devices log + require_visible_devices 2 '0,1' + devices="$(first_visible_devices 2)" + : "${QWEN25_STUDENT_PATH:?QWEN25_STUDENT_PATH was not exported by setup}" + : "${QWEN25_TEACHER_PATH:?QWEN25_TEACHER_PATH was not exported by setup}" + [[ -d "$QWEN25_STUDENT_PATH" && -d "$QWEN25_TEACHER_PATH" ]] || { + echo 'isolated OPSD requires local ModelScope model paths' >&2; exit 1; + } + log="$CI_OUTPUT_DIR/opsd-$role.log" + # The upstream ENV knobs are the interface; no model/source alias is needed. + # Student's optional CPU optimizer offload is disabled because its explicitly + # selected torch AdamW is not the ZeRO CPUAdam recipe; TP=2 + ZeRO-3 remain. + # Legacy mem= output uses uninitialized CUDA stats (zero), not NPU memory. + ASCEND_RT_VISIBLE_DEVICES="$devices" AUTOTP=2 \ + OFFLOAD="$([[ "$role" == teacher ]] && printf 1 || printf 0)" \ + STUDENT_MODEL="$QWEN25_STUDENT_PATH" TEACHER_MODEL="$QWEN25_TEACHER_PATH" \ + deepspeed --master_port "$(master_port_for "$([[ "$role" == teacher ]] && printf 93 || printf 92)")" \ + --num_gpus 2 --no_local_rank "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" 2>&1 | tee "$log" + validate_expansion "opsd_$role" "$log" \ + "$CI_OUTPUT_DIR/opsd-$role-results.json" +} + +run_expansion_example() { + case "$entry_key" in + benchmarks/pin_memory/model_tensor_offload/bench.py) run_pin_memory_example model_tensor_offload ;; + benchmarks/pin_memory/activation_offload/bench.py) run_pin_memory_example activation_offload ;; + benchmarks/pin_memory/h2d_d2h/bench.py) run_pin_memory_example h2d_d2h ;; + training/DeepSpeed-ZenFlow/finetuning/finetune_llama.py) run_zenflow_finetune ;; + training/opsd/main.py) run_opsd ;; + training/opsd/test_student_autotp_zero3.py) run_opsd_isolated student ;; + training/opsd/test_teacher_autotp_zero3.py) run_opsd_isolated teacher ;; + *) return 1 ;; + esac + # Called outside a conditional so errexit/pipefail remain active. +} +case "$entry_key" in + benchmarks/pin_memory/model_tensor_offload/bench.py|benchmarks/pin_memory/activation_offload/bench.py|benchmarks/pin_memory/h2d_d2h/bench.py|training/DeepSpeed-ZenFlow/finetuning/finetune_llama.py|training/opsd/main.py|training/opsd/test_student_autotp_zero3.py|training/opsd/test_teacher_autotp_zero3.py) + run_expansion_example + exit 0 + ;; +esac + +is_new_inference_entry() { + case "$entry_key" in + benchmarks/inference/bert-bench.py|benchmarks/inference/gpt-bench.py|\ + inference/huggingface/text-generation/ds-hf-compare.py|\ + inference/huggingface/fill-mask/test-bert.py|\ + inference/huggingface/fill-mask/test-electra.py|\ + inference/huggingface/fill-mask/test-roberta.py|\ + inference/huggingface/translation/test-t5-base.py|\ + inference/huggingface/automatic-speech-recognition/test-wav2vec2.py) return 0 ;; + *) return 1 ;; + esac +} + +write_inference_bootstrap() { + local output="$1" + "$PYTHON" - "$output" <<'PY' +import sys +from pathlib import Path + +Path(sys.argv[1]).write_text(r'''import json +import math +import os +from pathlib import Path +import runpy +import sys +import torch +import torch_npu +from deepspeed.accelerator import get_accelerator + +kind = os.environ['DS_INFERENCE_KIND'] +source = os.environ['DS_INFERENCE_SOURCE'] +if get_accelerator().device_name() != 'npu' or not torch.npu.is_available(): + raise SystemExit('inference smoke requires real NPU execution') +get_accelerator().set_device(int(os.environ.get('LOCAL_RANK', '0'))) +sys.argv[0] = source +sys.path.insert(0, str(Path(source).parent)) +state = runpy.run_path(source, run_name='__main__') + +def require(condition, message): + if not condition: + raise SystemExit(message) + +def finite(value): + return isinstance(value, (int, float)) and math.isfinite(value) + +if kind == 'asr': + model = state['model'] + require(next(model.parameters()).device.type == 'npu', 'CTC model weights are not NPU') + result = state['result'] + require(len(result) == 2 and all(isinstance(x['transcription'], str) for x in result), 'CTC did not process both synthetic waveforms') + score = state['wer'](result['text'], result['transcription']) + require(finite(score) and score >= 0, 'CTC fixture WER must be finite, not an accuracy assertion') + # The original maps argmax predictions; additionally reject NaN/Inf logits + # on one original waveform without replacing its dataset/model functions. + speech = state['librispeech_eval'][0]['speech'] + values = state['processor'](speech, sampling_rate=16000, return_tensors='pt', padding='longest').input_values + with torch.no_grad(): + logits = model(values.to(state['device'])).logits + require(logits.numel() > 0 and bool(torch.isfinite(logits).all()), 'CTC forward logits must be nonempty and finite') +else: + pipe = state.get('pipe', state.get('translator')) + require(pipe is not None and pipe.device.type == 'npu', 'pipeline device is not NPU') + require(next(pipe.model.parameters()).device.type == 'npu', 'model weights are not on NPU') + if kind in ('bert-bench', 'gpt-bench'): + require(len(state['times']) == 4 and all(finite(x) and x > 0 for x in state['times']), 'missing/invalid four NPU timings') + require(state['mtimes'] and all(finite(x) and x >= 0 for x in state['mtimes']), 'missing/invalid DS model timings') + require(len(state['responses']) == 4 and all(state['responses']), 'missing benchmark predictions') + elif kind == 'compare': + require(state['match_count'] == 2 and state['mismatch_count'] == 0, 'HF/DS outputs must match for both CI prompts') + elif kind == 'translation': + require(state['translation'] and state['translation'][0].get('translation_text', '').strip(), 'empty T5 translation') + elif kind == 'fill-mask': + predictions = state['output'] + require(predictions and all(finite(x['score']) and x['token_str'].strip() for x in predictions), 'missing/invalid fill-mask predictions') + else: + raise SystemExit(f'unknown inference assertion kind: {kind}') +print(f'CI inference assertions passed: {kind}, device=npu') +''', encoding='utf-8') +PY +} + +run_new_inference() { + local kind cards=1 alias='' offset=70 + case "$entry_key" in + benchmarks/inference/bert-bench.py) + kind=bert-bench; require_overlay_path --model directory ;; + benchmarks/inference/gpt-bench.py) + kind=gpt-bench; offset=71; require_overlay_path --model directory ;; + inference/huggingface/text-generation/ds-hf-compare.py) + kind=compare; offset=72; require_overlay_path --model directory ;; + inference/huggingface/fill-mask/test-bert.py) + kind=fill-mask; alias=bert-large-cased; offset=73 ;; + inference/huggingface/fill-mask/test-electra.py) + kind=fill-mask; alias=google/electra-base-generator; cards=2; offset=74 ;; + inference/huggingface/fill-mask/test-roberta.py) + kind=fill-mask; alias=roberta-large; cards=2; offset=75 ;; + inference/huggingface/translation/test-t5-base.py) + kind=translation; alias=t5-base; cards=2; offset=76 ;; + inference/huggingface/automatic-speech-recognition/test-wav2vec2.py) + kind=asr; alias=facebook/wav2vec2-base-960h; offset=78 ;; + *) return 1 ;; + esac + require_visible_devices "$cards" "$([[ "$cards" == 2 ]] && printf '0,1' || printf '0')" + local devices bootstrap + devices="$(first_visible_devices "$cards")" + if [[ -n "$alias" ]]; then + if [[ -z "${INFERENCE_CI_WORK:-}" || ! -f "$INFERENCE_CI_WORK/$alias/config.json" ]]; then + echo "local inference alias missing for $entry_key: $alias" >&2 + exit 1 + fi + cd "$INFERENCE_CI_WORK" + else + mkdir -p "$CI_OUTPUT_DIR/inference-work" + cd "$CI_OUTPUT_DIR/inference-work" + fi + bootstrap="$CI_OUTPUT_DIR/inference-entry.py" + write_inference_bootstrap "$bootstrap" || exit "$?" + # no_local_rank works with every native entry, including those with no parser. + # LOCAL_RANK/WORLD_SIZE are still injected by DeepSpeed's launcher. + # Explicit || exit is essential: this function is called from an if-condition, + # which otherwise disables bash errexit throughout the function body. + DS_INFERENCE_KIND="$kind" DS_INFERENCE_SOURCE="$LAUNCH_PATH" \ + ASCEND_RT_VISIBLE_DEVICES="$devices" HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \ + deepspeed --master_port "$(master_port_for "$offset")" --num_nodes 1 \ + --num_gpus "$cards" --no_local_rank "$bootstrap" "${EXTRA_ARGS[@]}" || exit "$?" + return 0 +} +if is_new_inference_entry; then + run_new_inference + exit 0 +fi + +is_training_expansion_entry() { + case "$entry_key" in + training/deepspeed_finetune_demo/finetune_llama.py|training/stable_diffusion/train_sd_distil_lora.py|training/opsd/benchmarks/bench_decode_1p1r.py) return 0 ;; + *) return 1 ;; + esac +} + +run_training_expansion_executor() { + case "$entry_key" in + training/deepspeed_finetune_demo/finetune_llama.py) run_finetune_demo_ci ;; + training/stable_diffusion/train_sd_distil_lora.py) run_sd_distil_ci ;; + training/opsd/benchmarks/bench_decode_1p1r.py) run_opsd_decode_ci ;; + *) echo "unknown training expansion entry: $entry_key" >&2; return 1 ;; + esac +} + +run_finetune_demo_ci() { + require_visible_devices 2 '0,1' + require_overlay_path --model_name directory + require_overlay_path --dataset_name directory + local devices config log + devices="$(first_visible_devices 2)" + config="$CI_OUTPUT_DIR/finetune-demo-config.json" + log="$CI_OUTPUT_DIR/finetune-demo.log" + "$PYTHON" - "$config" <<'PY' +import json +from pathlib import Path +import sys + +config = { + 'train_batch_size': 2, 'train_micro_batch_size_per_gpu': 1, + 'gradient_accumulation_steps': 1, 'bf16': {'enabled': True}, + 'zero_optimization': {'stage': 2}, + 'optimizer': {'type': 'AdamW', 'params': {'lr': 2e-5, 'torch_adam': True}}, + 'gradient_clipping': 1.0, 'steps_per_print': 1, +} +Path(sys.argv[1]).write_text(json.dumps(config, indent=2) + '\n') +PY + mkdir -p "$CI_OUTPUT_DIR/finetune-demo-work" + cd "$CI_OUTPUT_DIR/finetune-demo-work" + echo "running native SmolLM2/Llama finetune recipe: two-rank DP/ZeRO-2, three updates" + ASCEND_RT_VISIBLE_DEVICES="$devices" WANDB_MODE=disabled deepspeed \ + --master_port "$(master_port_for 111)" --num_gpus 2 \ + "$LAUNCH_PATH" --deepspeed --deepspeed_config "$config" "${EXTRA_ARGS[@]}" \ + 2>&1 | tee "$log" + "$PYTHON" - "$log" "$CI_OUTPUT_DIR/finetune-demo-losses.json" <<'PY' +import json +import math +from pathlib import Path +import re +import sys + +text = Path(sys.argv[1]).read_text() +if not re.search(r'First batch valid shifted labels: [1-9]\d*, finite params: True, finite logits: True,.*finite loss: True', text): + raise SystemExit('finetune demo did not prove valid labels and finite first-batch tensors') +losses = {} +for match in re.finditer(r'Step\s+(\d+), Loss:\s*([^,\s]+)', text): + step, value = int(match[1]), float(match[2]) + if not math.isfinite(value): + raise SystemExit(f'finetune demo produced nonfinite loss at step {step}') + losses[step] = value +if sorted(losses) != [1, 2, 3] or 'Training complete!' not in text: + raise SystemExit('finetune demo must finish exactly three finite updates') +Path(sys.argv[2]).write_text(json.dumps(losses, allow_nan=False) + '\n') +print('validated three finite finetune demo updates') +PY +} + +run_sd_distil_ci() { + require_visible_devices 2 '0,1' + require_overlay_path --pretrained_model_name_or_path directory + local devices config output log + devices="$(first_visible_devices 2)" + config="$CI_OUTPUT_DIR/sd-distil-config.json" + log="$CI_OUTPUT_DIR/sd-distil.log" + output="$(overlay_value --output_dir)" + [[ "$output" == "$CI_OUTPUT_DIR/sd-distil" ]] || { + echo "SD distillation must save its fresh pipeline under CI output" >&2; return 1; + } + [[ ! -e "$output/model_index.json" ]] || { + echo "SD distillation refuses to reuse a stale output pipeline" >&2; return 1; + } + [[ -s "$CI_OUTPUT_DIR/sd-distil-work/poloclub/diffusiondb/train.parquet" ]] || { + echo "SD local eight-image dataset missing; run ds_sd_distil setup first" >&2; return 1; + } + [[ "$(overlay_value --mixed_precision)" == no ]] || { + echo "SD unwrapped teacher requires the FP32 CI recipe (--mixed_precision no)" >&2; return 1; + } + "$PYTHON" - "$config" <<'PY' +import json +from pathlib import Path +import sys + +config = { + 'train_batch_size': 2, 'train_micro_batch_size_per_gpu': 1, + 'gradient_accumulation_steps': 1, 'zero_optimization': {'stage': 2}, + 'fp16': {'enabled': False}, 'bf16': {'enabled': False}, + 'gradient_clipping': 1.0, +} +Path(sys.argv[1]).write_text(json.dumps(config, indent=2) + '\n') +PY + cd "$CI_OUTPUT_DIR/sd-distil-work" + echo "running teacher-CFG full-UNet distillation using native local images, FP32 and two-rank DeepSpeed" + ASCEND_RT_VISIBLE_DEVICES="$devices" WANDB_MODE=disabled TQDM_MININTERVAL=0 \ + accelerate launch --use_deepspeed --num_processes 2 --num_machines 1 \ + --mixed_precision no --deepspeed_config_file "$config" \ + --main_process_port "$(master_port_for 121)" \ + "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" 2>&1 | tee "$log" + "$PYTHON" - "$log" "$output" "$CI_OUTPUT_DIR/sd-distil-losses.json" <<'PY' +import json +import math +from pathlib import Path +import re +import sys + +text = Path(sys.argv[1]).read_text() +losses = {} +# set_postfix(refresh=True) in the original entry prints the current loss +# after each progress update. Parse CR-separated tqdm frames, not wall-clock speed. +for frame in re.split(r'[\r\n]', text): + match = re.search(r'\b([123])/3\b.*?\bloss=([^,\]\s]+)', frame) + if match: + step, value = int(match[1]), float(match[2]) + if not math.isfinite(value): + raise SystemExit(f'SD distillation nonfinite loss at step {step}') + losses[step] = value +if sorted(losses) != [1, 2, 3]: + raise SystemExit('SD distillation must display three finite train-step losses') +output = Path(sys.argv[2]) +for name in ('model_index.json', 'unet/config.json', 'unet/diffusion_pytorch_model.safetensors'): + file = output / name + if not file.is_file() or not file.stat().st_size: + raise SystemExit(f'SD distillation failed to save a fresh trained pipeline: {file}') +Path(sys.argv[3]).write_text(json.dumps(losses, allow_nan=False) + '\n') +print('validated three finite SD distillation updates and trained pipeline save') +PY +} + +run_opsd_decode_ci() { + require_visible_devices 1 '0' + require_overlay_path --model directory + local devices log + devices="$(first_visible_devices 1)" + log="$CI_OUTPUT_DIR/opsd-decode.log" + mkdir -p "$CI_OUTPUT_DIR/opsd-decode-work" + cd "$CI_OUTPUT_DIR/opsd-decode-work" + # Normal backend import and runpy preserve every original API and function. + ASCEND_RT_VISIBLE_DEVICES="$devices" "$PYTHON" -c ' +import os, runpy, sys +import torch, torch_npu +from deepspeed.accelerator import get_accelerator +index = get_accelerator().current_device() +assert torch.empty(1, device="cpu").to(index).device.type == "npu" +assert torch.randint(10, 1000, (1, 2), device=index).device.type == "npu" +sys.argv = sys.argv[1:] +sys.path.insert(0, os.path.dirname(sys.argv[0])) +runpy.run_path(sys.argv[0], run_name="__main__") +' "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" 2>&1 | tee "$log" + "$PYTHON" - "$log" "$CI_OUTPUT_DIR/opsd-decode-timings.json" <<'PY' +import json +import math +from pathlib import Path +import re +import sys + +text = Path(sys.argv[1]).read_text() +timings = {} +for label in ('Raw decode loop', 'HybridEngine rollout'): + match = re.search(re.escape(label) + r':\s*([^\s]+)\s+ms', text) + if not match: + raise SystemExit(f'OPSD decode missing completed timing: {label}') + value = float(match[1]) + if not math.isfinite(value) or value <= 0: + raise SystemExit(f'OPSD decode invalid timing: {label}={value}') + timings[label] = value +Path(sys.argv[2]).write_text(json.dumps(timings, allow_nan=False) + '\n') +print('validated raw sampling/decode and HybridEngine rollout completion; no speed threshold') +PY +} +if is_training_expansion_entry; then + run_training_expansion_executor + exit 0 +fi + +run_vllm_benchmark() { + require_visible_devices 1 '0' + : "${VLLM_CI_MODEL_PATH:?local vLLM model is required}" + : "${VLLM_CI_BENCH_ROOT:?benchmark tokenizer directory is required}" + export ASCEND_RT_VISIBLE_DEVICES="$(first_visible_devices 1)" + export VLLM_WORKER_MULTIPROC_METHOD=spawn + export OPENAI_API_KEY=ci-local-not-a-credential + local port server_pid log ready=0 + port="$(master_port_for 130)" + log="$CI_OUTPUT_DIR/vllm-server.log" + vllm serve "$VLLM_CI_MODEL_PATH" --served-model-name ci-model \ + --host 127.0.0.1 --port "$port" --max-model-len 256 \ + --max-num-seqs 4 --gpu-memory-utilization 0.5 --enforce-eager > "$log" 2>&1 & + server_pid=$! + trap 'kill "$server_pid" 2>/dev/null || true; wait "$server_pid" 2>/dev/null || true' EXIT + for ((attempt=0; attempt<180; attempt++)); do + if curl --noproxy '*' -fsS "http://127.0.0.1:$port/health" >/dev/null 2>&1; then ready=1; break; fi + if ! kill -0 "$server_pid" 2>/dev/null; then cat "$log"; exit 1; fi + sleep 2 + done + if ((ready == 0)); then cat "$log"; echo 'vLLM server did not become ready' >&2; exit 1; fi + cd "$VLLM_CI_BENCH_ROOT" + local url="http://127.0.0.1:$port/v1/completions" + if [[ "$entry_key" == benchmarks/inference/mii/run_benchmark.py ]]; then + # The client's barrier can keep forked workers alive after a parent error. + timeout --kill-after=10s 5m "$PYTHON" "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" \ + --openai_api_url "$url" --openai_api_key "$OPENAI_API_KEY" + else + PYTHONPATH="$EXAMPLES_ROOT/benchmarks/inference/deepspeedometer/src${PYTHONPATH:+:$PYTHONPATH}" \ + "$PYTHON" -m deepspeedometer.benchmark_runner "${EXTRA_ARGS[@]}" --url "$url" + fi + # Clean up while the local PID is in scope; preserve native failures above. + kill "$server_pid" + wait "$server_pid" || true + trap - EXIT +} + +case "$entry_key" in + training/autotuning/hf/*/test_tune.sh) + require_visible_devices 1 '0' + : "${AUTOTUNE_WORK:?autotuning dependencies/data were not exported}" + : "${AUTOTUNE_MODEL_PATH:?local autotuning model is required}" + cd "$CI_OUTPUT_DIR" + "$PYTHON" - "$CI_OUTPUT_DIR" <<'PY' +import json +from pathlib import Path +import sys + +output = Path(sys.argv[1]) +config = {'train_micro_batch_size_per_gpu': [1, 2], 'gradient_accumulation_steps': 1, + 'zero_optimization': {'stage': 0}, + 'autotuning': {'enabled': True, 'fast': True, 'start_profile_step': 1, 'end_profile_step': 3, + 'results_dir': str(output / 'tuning-results'), 'exps_dir': str(output / 'tuning-experiments'), + 'arg_mappings': {'train_micro_batch_size_per_gpu': '--per_device_train_batch_size', + 'gradient_accumulation_steps': '--gradient_accumulation_steps'}}} +(output / 'autotuning.json').write_text(json.dumps(config)) +(output / 'hostfile').write_text('localhost slots=1\n') +PY + # evaluate.load('accuracy') finds the unchanged official metric locally. + cd "$AUTOTUNE_WORK" + deepspeed --autotuning run --hostfile "$CI_OUTPUT_DIR/hostfile" --num_nodes 1 --num_gpus 1 \ + --master_port "$(master_port_for 132)" "$AUTOTUNE_WORK/train.py" \ + --deepspeed "$CI_OUTPUT_DIR/autotuning.json" --model_name_or_path "$AUTOTUNE_MODEL_PATH" \ + --train_file "$AUTOTUNE_WORK/train.json" --validation_file "$AUTOTUNE_WORK/train.json" \ + --do_train --max_steps 5 \ + --per_device_train_batch_size 1 --save_strategy no --report_to none \ + --output_dir "$CI_OUTPUT_DIR/train" "${EXTRA_ARGS[@]}" + # The native tuner can log failed experiments and exit zero without a result. + test -s "$CI_OUTPUT_DIR/tuning-results/ds_config_optimal.json" + exit 0 + ;; + compression/reasoning_aware_compression/bash_script/eval_math500.sh) + require_visible_devices 1 '0' + : "${VLLM_CI_MODEL_PATH:?local vLLM model is required}" + : "${MATH500_CI_WORK:?local MATH-500 task/data are required}" + export VLLM_WORKER_MULTIPROC_METHOD=spawn + # Preserve the shell recipe's model/sampling/metric semantics, using the + # current native CLI and custom-task hook to select eight local problems. + lighteval vllm \ + "model_name=$VLLM_CI_MODEL_PATH,max_model_length=256,gpu_memory_utilization=0.8,override_chat_template=true,generation_parameters={max_new_tokens:16,temperature:0.6,top_p:0.95}" \ + 'ci_math500|0' --custom-tasks "$MATH500_CI_WORK/task.py" "${EXTRA_ARGS[@]}" + exit 0 + ;; + benchmarks/inference/mii/run_benchmark.py|benchmarks/inference/deepspeedometer/src/deepspeedometer/benchmark_runner.py) + run_vllm_benchmark + exit 0 + ;; + deepnvme/ds_io/ds_io_sweep.sh) + require_visible_devices 1 '0' + ulimit -Sl "$(ulimit -Hl)" + mkdir -p "$CI_OUTPUT_DIR/aio" + # Same ds_io accelerator recipe as the sweep, reduced to one write/read pair. + ds_io "${EXTRA_ARGS[@]}" --folder_to_device_mapping "$CI_OUTPUT_DIR/aio:0" + ds_io "${EXTRA_ARGS[@]}" --folder_to_device_mapping "$CI_OUTPUT_DIR/aio:0" --read + exit 0 + ;; + training/deepspeed_finetune_demo/evaluate/gsm8k/gen_gsm8k.py|training/deepspeed_finetune_demo/evaluate/mmlu/gen_mmlu.py|compression/reasoning_aware_compression/collect_traces.py) + require_visible_devices 1 '0' + require_overlay_path --model directory + export ASCEND_RT_VISIBLE_DEVICES="$(first_visible_devices 1)" + export VLLM_WORKER_MULTIPROC_METHOD=spawn + # Fail rather than silently exercising CPU/CUDA instead of Ascend. + "$PYTHON" -c 'import torch, torch_npu; from vllm.platforms import current_platform; assert current_platform.device_type == "npu" and torch.npu.is_available(), "vLLM must select available NPU"' + if [[ "$entry_key" == compression/reasoning_aware_compression/collect_traces.py ]]; then + require_overlay_path --dataset file + else + : "${VLLM_CI_DATA_ROOT:?native local evaluation datasets were not exported by setup}" + [[ -d "$VLLM_CI_DATA_ROOT" ]] || { echo "local evaluation dataset root missing: $VLLM_CI_DATA_ROOT" >&2; exit 1; } + # load_dataset('openai/gsm8k' or 'cais/mmlu') natively finds the staged + # local directories from this cwd; no monkeypatch or source edit. + cd "$VLLM_CI_DATA_ROOT" + fi + # Keep the original program's exit status; no answer/accuracy threshold. + "$PYTHON" "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" + exit 0 + ;; + training/cifar/run_ds_moe.sh) + run_cifar_moe + exit 0 + ;; + training/cifar/run_ds_prmoe.sh) + run_cifar_prmoe + exit 0 + ;; + training/tensor_parallel/hf_integration/train.py) + run_hf_autotp + exit 0 + ;; + applications/DeepSpeed-Chat/training/step1_supervised_finetuning/prompt_eval.py) + run_chat_prompt_eval + exit 0 + ;; + applications/DeepSpeed-Chat/training/step2_reward_model_finetuning/rw_eval.py) + run_chat_reward_eval + exit 0 + ;; + training/tensor_parallel/hf_integration/train_bench_length.py) + run_hf_bench_length + exit 0 + ;; + training/DeepSpeed-SuperOffload/finetune_zero3.py) + run_superoffload + exit 0 + ;; + training/data_efficiency/variable_batch_size_and_lr/variable_batch_size_and_lr_example.py) + run_variable_batch + exit 0 + ;; + training/DeepSpeed-ZenFlow/benchmark/zf_benchmark.py) + run_zenflow + exit 0 + ;; + compression/reasoning_aware_compression/prune.py) + run_rac_prune + exit 0 + ;; + training/autotp_equivalence/train.py) + run_autotp_equivalence + exit 0 + ;; + *) + require_visible_devices 1 '0' + ;; +esac + +case "$LAUNCH_PATH" in + *.sh) + bash "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" + ;; + *.py) + # DeepSpeed-Chat reads torch.distributed.get_rank() even when local_rank + # keeps its default -1, so it still needs the launcher to initialize a + # one-rank process group. Its upstream training_scripts wrappers hardcode + # large models/configs and do not pass arbitrary overlay args through; + # invoke the same launcher directly with our CI-sized manifest arguments. + case "${EXEC:-}" in + applications/DeepSpeed-Chat/training/*/main.py) + deepspeed --master_port "$(master_port_for 1)" --num_gpus 1 \ + "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" + ;; + *) + "$PYTHON" "$LAUNCH_PATH" "${EXTRA_ARGS[@]}" + ;; + esac + ;; + *) + echo "unsupported example type: $LAUNCH_PATH" >&2 + exit 1 + ;; +esac diff --git a/examples-guards/deepspeed/scripts/setup_example.sh b/examples-guards/deepspeed/scripts/setup_example.sh new file mode 100644 index 000000000..a1bce6bc8 --- /dev/null +++ b/examples-guards/deepspeed/scripts/setup_example.sh @@ -0,0 +1,1598 @@ +#!/usr/bin/env bash +# Prepare the CI environment for one supported example. +# $1 is the manifest profile. Unknown profiles fail before any install. +# DeepSpeed source is installed from the main-repo checkout (TARGET_ROOT in +# split mode; DEEPSPEED_SOURCE_ROOT overrides). Examples run from the +# DeepSpeedExamples checkout (EXAMPLES_ROOT). +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +PROFILE="$1" +export PIP_CONSTRAINT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/constraints-npu.txt" + +ASCEND_PIP_INDEX=https://repo.huaweicloud.com/ascend/repos/pypi +FALLBACK_PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple +CLUSTER_PIP_HOST=cache-service.nginx-pypi-cache.svc.cluster.local +export CLUSTER_PIP_INDEX="http://${CLUSTER_PIP_HOST}/pypi/simple" + +pip_ascend() { + python -m pip install --extra-index-url "$ASCEND_PIP_INDEX" "$@" +} + +select_pip_index() { + if python -c " +import urllib.error +import urllib.request +try: + urllib.request.urlopen('${CLUSTER_PIP_INDEX}', timeout=3) +except urllib.error.HTTPError: + pass +" 2>/dev/null; then + export PIP_INDEX_URL="$CLUSTER_PIP_INDEX" + export PIP_TRUSTED_HOST="$CLUSTER_PIP_HOST" + else + export PIP_INDEX_URL="$FALLBACK_PIP_INDEX" + unset PIP_TRUSTED_HOST + fi + echo "pip index: $PIP_INDEX_URL" +} + +ensure_torch_stack() { + if python -c " +import torch, torch_npu +print('found torch', torch.__version__, 'torch_npu', torch_npu.__version__) +raise SystemExit(0 if torch.__version__.startswith('2.9.0') and torch_npu.__version__.startswith('2.9.0') else 1) +"; then + echo "reusing image torch stack" + return + fi + echo "installing torch==2.9.0 torch_npu==2.9.0.post2" + pip_ascend torch==2.9.0 torch_npu==2.9.0.post2 +} + +# Install DeepSpeed from the main-repo checkout (TARGET_ROOT). +install_deepspeed_source() { + local src="${DEEPSPEED_SOURCE_ROOT:-$TARGET_ROOT}" + echo "installing DeepSpeed from source at $src" + python -m pip install -e "$src" + python -c " +import deepspeed +print('DeepSpeed version:', deepspeed.__version__) +" + ds_report 2>&1 | grep -i 'npu' || { + echo 'WARNING: ds_report did not list npu accelerator' + } + echo "installing MPI runtime for deepspeed.initialize distributed discovery" + apt-get update && apt-get install -y libopenmpi-dev numactl + python -m pip install mpi4py +} + +# Download models from ModelScope and expose their local paths to the run step +# via GITHUB_ENV (same pattern as examples-guards/trl). overlay_args reference these +# variables so examples receive concrete local directories. +ms_download_models() { + # modelscope>=1.38 splits hub code into modelscope-hub; pin the last pre-split + # release because the runner mirror may only expose an older hub for the latest wheel. + python -m pip install -q "modelscope==1.37.0" safetensors + TQDM_MININTERVAL="${TQDM_MININTERVAL:-15}" \ + MODELSCOPE_DOWNLOAD_PARALLELS=1 MODELSCOPE_ENABLE_DEFAULT_HASH_VALIDATION=true \ + python - "$@" <<'PY' +import json +import os +from pathlib import Path +import sys +import tempfile +import time +from modelscope import snapshot_download +from safetensors import safe_open + + +def validate_snapshot(local): + root = Path(local) + if not (root / 'config.json').is_file(): + raise RuntimeError(f'model config missing: {root}') + json.loads((root / 'config.json').read_text()) + shards = sorted(root.glob('*.safetensors')) + index = root / 'model.safetensors.index.json' + if index.is_file(): + expected = set(json.loads(index.read_text())['weight_map'].values()) + if not expected.issubset({shard.name for shard in shards}): + raise RuntimeError(f'model safetensors shards missing: {root}') + for shard in shards: + with safe_open(str(shard), framework='np') as weights: + if not list(weights.keys()): + raise RuntimeError(f'empty model safetensors: {shard}') + if not shards and not list(root.glob('pytorch_model*.bin')): + raise RuntimeError(f'no PyTorch model weights: {root}') + print(f'validated model assets: {root}, {len(shards)} safetensors shard(s)', flush=True) + + +# A shared snapshot can change after validation while another job downloads +# into it. Keep every model in this setup's private cache through the run step. +MODEL_CACHE = tempfile.mkdtemp(prefix='deepspeed-modelscope-', dir='/tmp') +print('private ModelScope cache:', MODEL_CACHE, flush=True) +for pair in sys.argv[1:]: + env_name, model_id = pair.split("=", 1) + cache = MODEL_CACHE + ignored = ['*.h5', '*.msgpack', '*.onnx', '*.ot', '*.ckpt', '*.tflite'] + # This mirror contains safetensors; don't fetch its redundant .bin weights. + if model_id == 'AI-ModelScope/bert-base-cased': + ignored.append('*.bin') + for attempt in range(3): + try: + local = snapshot_download(model_id, cache_dir=cache, + ignore_file_pattern=ignored, max_workers=2, + enable_file_lock=True) + validate_snapshot(local) + break + except Exception as exc: + print(f'ModelScope {model_id} attempt {attempt + 1}/3 failed: {exc}', flush=True) + if attempt == 2: + raise + # A fresh job-owned cache avoids trusting a corrupt cached shard. + # Keep the old files untouched; never purge the runner-wide cache. + cache = tempfile.mkdtemp(prefix='deepspeed-modelscope-retry-', dir='/tmp') + time.sleep(2 * (attempt + 1)) + with open(os.environ["GITHUB_ENV"], "a") as fh: + fh.write(f"{env_name}={local}\n") +PY + # GITHUB_ENV is applied in the next step; expose only requested model names + # in this setup process so fixture checks can use the downloaded tokenizer. + local pair env_name local_path + for pair in "$@"; do + env_name="${pair%%=*}" + [[ "$env_name" =~ ^[A-Z_][A-Z0-9_]*$ ]] || { + echo "invalid model environment name: $env_name" >&2 + return 1 + } + local_path="$(awk -v key="$env_name" ' + index($0, key "=") == 1 { value = substr($0, length(key) + 2) } + END { print value } + ' "$GITHUB_ENV")" + [[ -d "$local_path" ]] || { + echo "downloaded model directory missing for $env_name: $local_path" >&2 + return 1 + } + export "$env_name=$local_path" + done +} + +# Redirect the renamed wikitext dataset at interpreter startup without editing +# the DeepSpeedExamples checkout. The shim is scoped to this CI job through +# GITHUB_ENV and leaves every other datasets.load_dataset call unchanged. +install_hello_dataset_shim() { + local shim_dir="$GITHUB_WORKSPACE/.ci/deepspeed-sitecustomize" + mkdir -p "$shim_dir" + cat > "$shim_dir/sitecustomize.py" <<'PY' +import datasets as _datasets + +_original_load_dataset = _datasets.load_dataset + + +def _patched_load_dataset(path, *args, **kwargs): + if path == "wikitext": + path = "Salesforce/wikitext" + return _original_load_dataset(path, *args, **kwargs) + + +_datasets.load_dataset = _patched_load_dataset +PY + echo "PYTHONPATH=$shim_dir${PYTHONPATH:+:$PYTHONPATH}" >> "$GITHUB_ENV" + echo "installed wikitext runtime redirect in $shim_dir" +} + +# Copy a fixture into the DeepSpeed-Chat data/ dir that local/jsonfile reads. +# $1 = fixture name under $FIXTURE_DIR. +plant_chat_fixture() { + local fixture="$1" + local chat_data="$EXAMPLES_ROOT/applications/DeepSpeed-Chat/data" + mkdir -p "$chat_data" + cp "$FIXTURE_DIR/$fixture" "$chat_data/train.json" + cp "$FIXTURE_DIR/$fixture" "$chat_data/eval.json" + echo "planted chat fixture $fixture -> $chat_data/{train,eval}.json" +} + +# DeepSpeed-Chat's setup.py uses find_packages(include=['dschat']), while the +# dschat root is a PEP 420 namespace package without __init__.py. pip therefore +# installs distribution metadata but does not make the source package +# importable when the launcher changes into a training/step* directory. Expose +# the upstream source root explicitly without editing the examples checkout. +install_chat_source_path() { + local chat_root="$EXAMPLES_ROOT/applications/DeepSpeed-Chat" + case ":${PYTHONPATH:-}:" in + *":$chat_root:"*) ;; + *) export PYTHONPATH="$chat_root${PYTHONPATH:+:$PYTHONPATH}" ;; + esac + echo "PYTHONPATH=$PYTHONPATH" >> "$GITHUB_ENV" + python - <<'PY' +import dschat + +paths = list(dschat.__path__) +if not paths: + raise SystemExit("dschat namespace package has no source path") +print("runtime dschat namespace:", paths) +PY +} + +setup_deepspeed() { + install_deepspeed_source + echo "installing dependencies from the HelloDeepSpeed and CIFAR requirements baselines" + PIP_INDEX_URL="https://pypi.tuna.tsinghua.edu.cn/simple" \ + python -m pip install "tokenizers>=0.22.0,<0.23" "transformers<5" datasets \ + fire loguru "sh==1.14.2" tqdm pytz tensorboard \ + "torchvision==0.24.0" "pillow>=7.1.0" matplotlib + # CIFAR profiles reuse these packages but never load wikitext. + if [[ "$PROFILE" == deepspeed ]]; then + install_hello_dataset_shim + fi +} + +# Pre-stage CIFAR-10 from a pinned ModelScope dataset revision. Both the +# single-card and MoE examples use torchvision with root=./data and +# download=True; a complete local tree makes torchvision skip its unreachable +# Toronto download URL. Verify both the archive SHA-256 and torchvision's +# official per-file MD5 list. +plant_cifar10() { + local data_dir="$EXAMPLES_ROOT/training/cifar/data" + CIFAR10_DATA_DIR="$data_dir" python - <<'PY' +import hashlib +import os +from pathlib import Path +import time +import urllib.request +import zipfile + +from torchvision.datasets import CIFAR10 + +URL = ( + "https://modelscope.cn/api/v1/datasets/studyhard1/" + "cifar10-dataset/repo?Revision=" + "9231e736fd8d53f7158165a07d801429b4414993&" + "FilePath=cifar-10-batches-py.zip" +) +EXPECTED_SHA256 = "4f287e6733e987d5c1ab2af557413cf0f5bc78f293765d87dc5633788246e5d7" +data_dir = Path(os.environ["CIFAR10_DATA_DIR"]).resolve() +archive = data_dir / "cifar-10-batches-py.zip" +partial = archive.with_suffix(".zip.part") +data_dir.mkdir(parents=True, exist_ok=True) + + +def complete() -> bool: + dataset = CIFAR10.__new__(CIFAR10) + dataset.root = str(data_dir) + return dataset._check_integrity() + + +if complete(): + print("CIFAR-10 fixture already passes torchvision integrity checks:", data_dir) + raise SystemExit(0) + +for attempt in range(1, 4): + digest = hashlib.sha256() + try: + request = urllib.request.Request(URL, headers={"User-Agent": "cosdt-ci/1.0"}) + with urllib.request.urlopen(request, timeout=120) as response, partial.open("wb") as output: + while chunk := response.read(1024 * 1024): + output.write(chunk) + digest.update(chunk) + actual = digest.hexdigest() + if actual != EXPECTED_SHA256: + raise RuntimeError( + f"CIFAR-10 archive sha256 mismatch: expected {EXPECTED_SHA256}, got {actual}" + ) + partial.replace(archive) + break + except Exception: + partial.unlink(missing_ok=True) + if attempt == 3: + raise + print(f"CIFAR-10 download attempt {attempt}/3 failed; retrying", flush=True) + time.sleep(2 * attempt) + +with zipfile.ZipFile(archive) as bundle: + for member in bundle.infolist(): + destination = (data_dir / member.filename).resolve() + if destination != data_dir and data_dir not in destination.parents: + raise RuntimeError(f"unsafe CIFAR-10 archive member: {member.filename}") + bundle.extractall(data_dir) +archive.unlink() + +if not complete(): + raise SystemExit("extracted CIFAR-10 data failed torchvision integrity checks") +print("planted verified CIFAR-10 fixture in", data_dir) +PY +} + +setup_ds_cifar() { + setup_deepspeed + plant_cifar10 +} + +# DS-Chat reloads the tokenizer from config._name_or_path even when its model +# argument is local. Keep that lookup local in a job-owned asset view, without +# changing the upstream code or the shared ModelScope snapshot. +prepare_chat_model_view() { + python - <<'PY' +import json +import os +from pathlib import Path +import tempfile +from transformers import AutoTokenizer + +snapshot = Path(os.environ['OPT_125M_PATH']).resolve() +views = Path(os.environ['GITHUB_WORKSPACE']) / '.ci' / 'deepspeed-chat-models' +views.mkdir(parents=True, exist_ok=True) +view = Path(tempfile.mkdtemp(prefix='opt-125m-', dir=views)) +for asset in snapshot.iterdir(): + if asset.name != 'config.json': + (view / asset.name).symlink_to(asset, target_is_directory=asset.is_dir()) +config = json.loads((snapshot / 'config.json').read_text()) +config['_name_or_path'] = str(view) +(view / 'config.json').write_text(json.dumps(config)) +AutoTokenizer.from_pretrained(str(view), local_files_only=True) +with open(os.environ['GITHUB_ENV'], 'a') as env: + env.write(f'OPT_125M_PATH={view}\n') +print('DS-Chat offline model/tokenizer view:', view) +PY + local model_path + model_path="$(awk '/^OPT_125M_PATH=/ { value = substr($0, 15) } END { print value }' "$GITHUB_ENV")" + [[ -d "$model_path" ]] || { echo 'DS-Chat local model view missing' >&2; return 1; } + export OPT_125M_PATH="$model_path" +} + +# Shared DeepSpeed-Chat setup: DS source + transformers + opt-125m + fixture. +setup_ds_chat() { + local fixture="$1" + install_deepspeed_source + # Install the declared dependencies explicitly so pip cannot replace the NPU + # torch stack or the source DeepSpeed under test. The upstream editable + # package is intentionally not used: its find_packages() call produces an + # empty package for the PEP 420 dschat namespace. + python -m pip install "transformers>=4.31.0,<5,!=4.33.2" \ + "datasets>=2.8.0" "accelerate>=0.15.0" "sentencepiece>=0.1.97" \ + "protobuf==3.20.3" tensorboard + install_chat_source_path + ms_download_models "OPT_125M_PATH=facebook/opt-125m" + prepare_chat_model_view + plant_chat_fixture "$fixture" +} + +setup_ds_chat_sft() { setup_ds_chat ci_sft_8.json; } +setup_ds_chat_rw() { setup_ds_chat ci_rw_8.json; } +setup_ds_chat_dpo() { setup_ds_chat ci_dpo_8.json; } +setup_ds_chat_rlhf() { setup_ds_chat ci_rlhf_8.json; } + +# Evaluation entries use the same upstream Chat baseline with protected runtime +# dependencies. Leave the already-validated training profiles unchanged. +setup_ds_chat_eval_dependencies() { + install_deepspeed_source + install_example_dependencies "transformers>=4.31.0,<5,!=4.33.2" \ + "datasets>=2.8.0" "accelerate>=0.15.0" "sentencepiece>=0.1.97" \ + "protobuf==3.20.3" tensorboard + install_chat_source_path + ms_download_models "OPT_125M_PATH=facebook/opt-125m" + prepare_chat_model_view + python - <<'PY' +import importlib.util +import os +from pathlib import Path +import torch_npu +from transformers import AutoTokenizer + +root = Path(os.environ["EXAMPLES_ROOT"]) / "applications/DeepSpeed-Chat/training" +for step, filename in ( + ("step1_supervised_finetuning", "prompt_eval.py"), + ("step2_reward_model_finetuning", "rw_eval.py"), +): + spec = importlib.util.spec_from_file_location(f"ds_ci_{filename[:-3]}", root / step / filename) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) +tokenizer = AutoTokenizer.from_pretrained(os.environ["OPT_125M_PATH"]) +print("Chat evaluation local tokenizer:", type(tokenizer).__name__) +PY +} + +setup_ds_chat_prompt_eval() { + setup_ds_chat_eval_dependencies + # The runner trains a genuine small SFT checkpoint before prompt comparison. + plant_chat_fixture ci_sft_8.json +} + +setup_ds_chat_reward_eval() { + setup_ds_chat_eval_dependencies + # rw_eval's default create_critic_model() constructs a fresh value head; this + # entry is an honest reward-forward smoke, not trained checkpoint restoration. +} + +setup_ds_infer() { + install_deepspeed_source + python -m pip install "transformers<5" accelerate + ms_download_models "OPT_125M_PATH=facebook/opt-125m" +} + +setup_ds_autotp_equivalence() { + install_deepspeed_source + # Qwen3 support is present in current 4.x transformers; keep the upper bound + # below the next major release to avoid unreviewed API changes. + python -m pip install "transformers>=4.51.0,<5" safetensors + ms_download_models "QWEN3_06B_PATH=Qwen/Qwen3-0.6B" +} + +# Install only each example's non-runtime requirements. Constraints also stop +# transitive dependencies from replacing the image NPU stack or source DS. +install_example_dependencies() { + local constraint_dir="$GITHUB_WORKSPACE/.ci/deepspeed-dependencies" + local constraint_file="$constraint_dir/protected-stack.txt" + mkdir -p "$constraint_dir" + python - "$constraint_file" <<'PY' +from importlib.metadata import version +from pathlib import Path +import os +import sys + +protected = ("torch", "torch-npu", "deepspeed") +constraints = "".join(f"{name}=={version(name)}\n" for name in protected) +# Reuse the global scientific stack pins; keep SciPy and NumPy paired. +constraints += Path(os.environ['PIP_CONSTRAINT']).read_text(encoding='utf-8') +Path(sys.argv[1]).write_text(constraints) +print("preserving installed runtime dependencies:\n" + constraints) +PY + python -m pip install --constraint "$constraint_file" "$@" +} + +# New examples accept explicit fixture paths, so stage their data outside the +# examples checkout instead of changing any upstream source or dataset loader. +plant_ci_fixture() { + local fixture="$1" + local env_name="$2" + local ci_dir="$GITHUB_WORKSPACE/.ci/deepspeed-fixtures" + mkdir -p "$ci_dir" + cp "$FIXTURE_DIR/$fixture" "$ci_dir/$fixture" + export "$env_name=$ci_dir/$fixture" + echo "$env_name=$ci_dir/$fixture" >> "$GITHUB_ENV" + echo "staged $fixture -> $ci_dir/$fixture ($env_name)" +} + +setup_ds_hf_autotp() { + install_deepspeed_source + # train.py imports Trainer directly; the extra evaluation/OpenAI packages in + # the upstream requirements belong to utils.py, which this entry never imports. + install_example_dependencies "transformers>=4.51.0,<5" "accelerate>=1.10.1,<2" \ + sentencepiece psutil numpy safetensors + ms_download_models "OPT_125M_PATH=facebook/opt-125m" + plant_ci_fixture ci_alpaca_16.json ALPACA_CI_PATH + python - <<'PY' +import accelerate +import importlib.util +import os +from pathlib import Path +import psutil +import sys +import sentencepiece +import torch_npu +import transformers +from transformers import AutoTokenizer, Trainer + +entry = Path(os.environ["EXAMPLES_ROOT"]) / "training/tensor_parallel/hf_integration/train.py" +spec = importlib.util.spec_from_file_location("ds_ci_hf_autotp", entry) +upstream = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = upstream +spec.loader.exec_module(upstream) +tokenizer = AutoTokenizer.from_pretrained(os.environ["OPT_125M_PATH"], + model_max_length=128, padding_side="right", use_fast=False) +special_tokens = {} +for kind in ("pad", "eos", "bos", "unk"): + if getattr(tokenizer, f"{kind}_token") is None: + special_tokens[f"{kind}_token"] = getattr(upstream, f"DEFAULT_{kind.upper()}_TOKEN") +tokenizer.add_special_tokens(special_tokens) +# SupervisedDataset writes a fixed dataset_dict.pkl in cwd. Precompute it once +# before eight ranks start, in the output directory used by the project runner. +# The engine supplies CI_OUTPUT_DIR only to run, whose default is workspace/output. +output_root = Path(os.environ.get("CI_OUTPUT_DIR", str(Path(os.environ["GITHUB_WORKSPACE"]) / "output"))) +work_dir = output_root / "hf-autotp-work" +work_dir.mkdir(parents=True, exist_ok=True) +cache = work_dir / "dataset_dict.pkl" +# A runner may retain output from a previous job. This file is our generated +# tokenization cache; rebuild it from this run's fixture before workers read it. +cache.unlink(missing_ok=True) +previous_cwd = Path.cwd() +try: + os.chdir(work_dir) + dataset = upstream.SupervisedDataset(os.environ["ALPACA_CI_PATH"], tokenizer) +finally: + os.chdir(previous_cwd) +counts = [(labels != upstream.IGNORE_INDEX).sum().item() for labels in dataset.labels] +if len(dataset) != 16 or any(count == 0 for count in counts): + raise SystemExit(f"HF AutoTP fixture lost training labels at length 128: {counts}") +if not cache.is_file() or cache.stat().st_size == 0: + raise SystemExit(f"HF AutoTP pre-tokenized cache missing: {cache}") +print("HF AutoTP shared pre-tokenized cache:", cache) +print("HF AutoTP non-masked target tokens per fixture row:", counts) + +print("HF AutoTP dependencies:", "transformers", transformers.__version__, + "accelerate", accelerate.__version__, "psutil", psutil.__version__) +PY +} + +setup_ds_hf_bench_length() { + install_deepspeed_source + # The sibling utils.py imports the legacy OpenAIObject API at module import. + # This dependency enables ordinary imports only; this recipe makes no API call. + install_example_dependencies "transformers>=4.51.0,<5" "accelerate>=1.10.1,<2" \ + "openai==0.28.1" sentencepiece psutil numpy safetensors + ms_download_models "OPT_125M_PATH=facebook/opt-125m" + plant_ci_fixture ci_alpaca_16.json ALPACA_CI_PATH + python - <<'PY' +import accelerate +import importlib.util +import os +from pathlib import Path +import sys + +import numpy +import openai +from openai import openai_object +import torch_npu +import transformers +from transformers import AutoTokenizer + +entry = Path(os.environ["EXAMPLES_ROOT"]) / "training/tensor_parallel/hf_integration/train_bench_length.py" +sys.path.insert(0, str(entry.parent)) +spec = importlib.util.spec_from_file_location("ds_ci_hf_bench_length", entry) +upstream = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = upstream +spec.loader.exec_module(upstream) +tokenizer = AutoTokenizer.from_pretrained(os.environ["OPT_125M_PATH"], + model_max_length=128, padding_side="right", use_fast=False) +special_tokens = {} +for kind in ("pad", "eos", "bos", "unk"): + if getattr(tokenizer, f"{kind}_token") is None: + special_tokens[f"{kind}_token"] = getattr(upstream, f"DEFAULT_{kind.upper()}_TOKEN") +tokenizer.add_special_tokens(special_tokens) +output_root = Path(os.environ.get("CI_OUTPUT_DIR", str(Path(os.environ["GITHUB_WORKSPACE"]) / "output"))) +work_dir = output_root / "hf-bench-length-work" +work_dir.mkdir(parents=True, exist_ok=True) +cache = work_dir / "dataset_dict128.pkl" +# Our generated shared cache is rebuilt before the eight readers start. +cache.unlink(missing_ok=True) +previous_cwd = Path.cwd() +try: + os.chdir(work_dir) + dataset = upstream.SupervisedDataset(os.environ["ALPACA_CI_PATH"], tokenizer) +finally: + os.chdir(previous_cwd) +counts = [(labels != upstream.IGNORE_INDEX).sum().item() for labels in dataset.labels] +if len(dataset) != 16 or any(count == 0 for count in counts): + raise SystemExit(f"fixed-length HF fixture lost training labels: {counts}") +for ids, labels in zip(dataset.input_ids, dataset.labels): + if ids.numel() != 128 or labels.numel() != 128: + raise SystemExit("HF benchmark cache does not contain fixed 128-token rows") + if not bool((labels[ids == tokenizer.pad_token_id] == upstream.IGNORE_INDEX).all()): + raise SystemExit("HF benchmark cache contains unmasked padding targets") +if not cache.is_file() or cache.stat().st_size == 0: + raise SystemExit(f"fixed-length HF pre-tokenized cache missing: {cache}") +print("HF benchmark shared pre-tokenized cache:", cache) +print("HF benchmark non-masked target tokens:", counts) +print("HF benchmark dependencies:", "transformers", transformers.__version__, + "accelerate", accelerate.__version__, "openai", openai.__version__, "numpy", numpy.__version__) +PY +} + +setup_ds_superoffload() { + install_deepspeed_source + # finetune_zero3 imports wandb unconditionally, but the runner never enables + # its network logging. CPU Adam is compiled up front rather than on every rank. + install_example_dependencies "transformers>=4.56.1,<5" "accelerate>=1.10.1,<2" \ + "datasets>=4,<5" "numpy>=1.21.0" packaging psutil pyarrow \ + sentencepiece safetensors wandb ninja + apt-get install -y build-essential + ms_download_models "OPT_125M_PATH=facebook/opt-125m" + plant_ci_fixture ci_alpaca_16.json ALPACA_CI_PATH + export ALPACA_DATASET_DIR="$GITHUB_WORKSPACE/.ci/deepspeed-fixtures/alpaca-parquet" + mkdir -p "$ALPACA_DATASET_DIR" + echo "ALPACA_DATASET_DIR=$ALPACA_DATASET_DIR" >> "$GITHUB_ENV" + python - <<'PY' +import importlib.util +import json +import logging +import os +from pathlib import Path +import sys + +import datasets +import torch_npu +import transformers +import wandb +from datasets import Dataset, load_dataset +from deepspeed.ops.op_builder import CPUAdamBuilder + +rows = json.loads(Path(os.environ["ALPACA_CI_PATH"]).read_text()) +if len(rows) != 16 or any(set(row) != {"instruction", "input", "output"} for row in rows): + raise SystemExit("SuperOffload fixture must contain 16 Alpaca instruction/input/output rows") +dataset_dir = Path(os.environ["ALPACA_DATASET_DIR"]) +Dataset.from_list(rows).to_parquet(str(dataset_dir / "train.parquet")) +# The upstream entry passes a directory directly to load_dataset(), not the +# JSON loader or load_from_disk(). Validate precisely that public loading path. +dataset = load_dataset(str(dataset_dir)) +if len(dataset["train"]) != 16 or set(dataset["train"].column_names) != {"instruction", "input", "output"}: + raise SystemExit("local parquet directory did not resolve to the 16-row train split") +entry = Path(os.environ["EXAMPLES_ROOT"]) / "training/DeepSpeed-SuperOffload/finetune_zero3.py" +spec = importlib.util.spec_from_file_location("ds_ci_superoffload", entry) +upstream = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = upstream +spec.loader.exec_module(upstream) +tokenizer = upstream.load_tokenizer(os.environ["OPT_125M_PATH"], logging.getLogger("ci-superoffload")) +for row in dataset["train"]: + batch = upstream.preprocess_alpaca_example(row, tokenizer, max_length=128) + if len(batch["input_ids"]) != 128 or len(batch["labels"]) != 128: + raise SystemExit("SuperOffload fixture does not produce 128-token training rows") + if sum(batch["attention_mask"]) < 2 or batch["labels"] != batch["input_ids"]: + raise SystemExit("SuperOffload fixture lost the upstream causal language-model targets") +builder = CPUAdamBuilder() +print("checking SuperOffload CPU Adam extension:", type(builder).__name__) +builder.load(verbose=True) +print("SuperOffload CPU Adam extension loaded successfully") +print("SuperOffload local parquet train split:", len(dataset["train"]), dataset_dir) +print("SuperOffload upstream optimizer LR:", upstream.DEFAULT_OPTIMIZER_LR) +print("SuperOffload dependencies:", "datasets", datasets.__version__, + "transformers", transformers.__version__, "wandb", wandb.__version__) +PY +} + +setup_ds_variable_batch() { + install_deepspeed_source + # DeepSpeed's DataAnalyzer/data-sampling path uses these packages. The + # example itself constructs its small model and synthetic dataset locally. + install_example_dependencies numpy pandas + python - <<'PY' +import numpy +import pandas +import torch_npu +from deepspeed.runtime.data_pipeline.data_sampling.variable_batch_size_and_lr import ( + get_dataloader_and_lr_scheduler_for_variable_batch_size_deepspeed, +) + +print("dynamic batch dependencies:", "numpy", numpy.__version__, + "pandas", pandas.__version__) +PY +} + +setup_ds_zenflow() { + install_deepspeed_source + # benchmark/requirements.txt without its torch/deepspeed entries. CPU Adam + # is imported here; its actual NPU/offload execution is checked by the run. + install_example_dependencies "datasets>=2.14.1" "transformers>=4.37.2,<5" \ + "numpy>=1.21.0" tabulate pandas ninja + echo "installing C++ build tools for upstream ZenFlow CPU Adam" + apt-get install -y build-essential + python - <<'PY' +import datasets +import numpy +import pandas +import tabulate +import torch_npu +import transformers +from deepspeed.ops.adam import DeepSpeedCPUAdam +from deepspeed.ops.op_builder import CPUAdamBuilder + +builder = CPUAdamBuilder() +print("checking ZenFlow CPU Adam extension:", type(builder).__name__) +builder.load(verbose=True) +print("ZenFlow CPU Adam extension loaded successfully") + +print("ZenFlow dependencies:", "datasets", datasets.__version__, + "transformers", transformers.__version__, "numpy", numpy.__version__, + "pandas", pandas.__version__, "tabulate", tabulate.__version__) +PY +} + +setup_ds_rac_prune() { + install_deepspeed_source + # Local prompt JSONL takes the standard-library branch of the calibration + # loader. No datasets, vLLM, evaluation backend or runtime patch is needed. + install_example_dependencies "transformers>=4.45.0,<5" "accelerate>=0.30" \ + sentencepiece safetensors + ms_download_models "OPT_125M_PATH=facebook/opt-125m" + plant_ci_fixture ci_rac_8.jsonl RAC_CI_PATH + python - <<'PY' +import os +from pathlib import Path +import sys + +import accelerate +import torch_npu +import transformers + +sys.path.insert(0, str(Path(os.environ["EXAMPLES_ROOT"]) / + "compression/reasoning_aware_compression")) +from rac import build_calibration_samples, sequential_prune +from rac.wanda import Wanda +from transformers import AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained(os.environ["OPT_125M_PATH"]) +samples = build_calibration_samples( + "prompt", tokenizer, nsamples=4, seqlen=32, seed=0, + dataset_name=os.environ["RAC_CI_PATH"], prompt_column="prompt", + use_chat_template=False, +) +if len(samples) != 4 or any(sample.numel() != 32 for sample in samples): + raise SystemExit("RAC fixture cannot fill the required 4 x 32 token windows") +print("RAC calibration fixture:", len(samples), "windows x", samples[0].numel(), "tokens") + +print("Wanda pruning dependencies:", "transformers", transformers.__version__, + "accelerate", accelerate.__version__) +PY +} + +setup_ds_pin_memory() { + install_deepspeed_source + install_example_dependencies numpy ninja + apt-get install -y build-essential + # Adjust the shell so its Python/compiler subprocesses inherit the allowed + # limit. The later run step independently makes the same bounded adjustment. + local memlock_hard + memlock_hard="$(ulimit -Hl)" + ulimit -Sl "$memlock_hard" + echo "pin-memory shell memlock soft/hard KiB: $(ulimit -Sl)/$memlock_hard" + python - <<'PY' +import os +import resource + +import torch +import torch_npu +from deepspeed.accelerator import get_accelerator +from deepspeed.ops.op_builder import CPUAdamBuilder, PinMemoryBuilder +from deepspeed.utils.pin_memory import get_native_pinned_memory + +# Never increase the container's hard limit or require extra privileges. +soft, hard = resource.getrlimit(resource.RLIMIT_MEMLOCK) +print("pin-memory RLIMIT_MEMLOCK soft/hard bytes:", soft, hard) +required = 4 * 1024 * 1024 +if soft != resource.RLIM_INFINITY and soft < required: + raise SystemExit(f"small native pin-memory CI recipes need at least {required} locked bytes; container limit is {soft}") +for builder in (CPUAdamBuilder(), PinMemoryBuilder()): + print("preloading pin-memory benchmark extension:", type(builder).__name__) + builder.load(verbose=True) +os.environ["DS_PIN_MEMORY_BACKEND"] = "native" +os.environ["DS_PIN_MEMORY_REGISTER_DEVICE"] = "0" +manager = get_native_pinned_memory() +raw = torch.arange(1024 * 1024 // 4, dtype=torch.float32) +locked = manager.pin(raw) +try: + if not manager.is_pinned(locked) or not torch.equal(locked, raw): + raise SystemExit("native page-locked allocation failed the pin/content preflight") + accelerator = get_accelerator() + device = locked.to(accelerator.current_device_name()) + accelerator.synchronize() + if not torch.equal(device.cpu(), raw): + raise SystemExit("native page-locked host/NPU copy lost tensor contents") +finally: + manager.unpin(locked) +print("native mlock host allocation and NPU copy verified; this is not CUDA host registration") +PY +} + +setup_ds_zenflow_finetune() { + install_deepspeed_source + install_example_dependencies "transformers==4.57.6" "datasets>=4,<5" \ + "accelerate>=1.10.1,<2" numpy pyarrow sentencepiece safetensors ninja + apt-get install -y build-essential + ms_download_models "OPT_125M_PATH=facebook/opt-125m" + plant_ci_fixture ci_alpaca_16.json ALPACA_CI_PATH + export ZENFLOW_WORK_DIR="${CI_OUTPUT_DIR:-$GITHUB_WORKSPACE/output}/zenflow-finetune-work" + mkdir -p "$ZENFLOW_WORK_DIR/tatsu-lab/alpaca" + echo "ZENFLOW_WORK_DIR=$ZENFLOW_WORK_DIR" >> "$GITHUB_ENV" + echo "HF_DATASETS_OFFLINE=1" >> "$GITHUB_ENV" + HF_DATASETS_OFFLINE=1 python - <<'PY' +import importlib.util +import json +import os +from pathlib import Path +import sys + +import datasets +import torch_npu +import transformers +from datasets import Dataset, load_dataset +from deepspeed.ops.op_builder import CPUAdamBuilder +from transformers import AutoTokenizer, default_data_collator + +rows = json.loads(Path(os.environ["ALPACA_CI_PATH"]).read_text()) +if len(rows) != 16 or any(set(row) != {"instruction", "input", "output"} for row in rows): + raise SystemExit("ZenFlow fixture must contain 16 original Alpaca-schema rows") +work = Path(os.environ["ZENFLOW_WORK_DIR"]).resolve() +Dataset.from_list(rows).to_parquet(str(work / "tatsu-lab/alpaca/train.parquet")) +os.chdir(work) +# Match the unmodified entry's hardcoded public API call precisely. The local +# directory is resolved natively; no dataset shim or source rewrite is used. +dataset = load_dataset("tatsu-lab/alpaca") +if len(dataset["train"]) != 16 or set(dataset["train"].column_names) != {"instruction", "input", "output"}: + raise SystemExit("ZenFlow's exact hardcoded dataset name did not resolve to the local fixture") +entry = Path(os.environ["EXAMPLES_ROOT"]) / "training/DeepSpeed-ZenFlow/finetuning/finetune_llama.py" +spec = importlib.util.spec_from_file_location("ds_ci_zenflow_finetune", entry) +upstream = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = upstream +spec.loader.exec_module(upstream) +tokenizer = AutoTokenizer.from_pretrained(os.environ["OPT_125M_PATH"]) +if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token +# Keep the original string columns just as the entry does. The official +# collator ignores strings, so the engine receives only model input tensors. +tokenized = dataset["train"].map(lambda row: upstream.preprocess_alpaca(row, tokenizer), batched=False) +for row in tokenized: + if len(row["input_ids"]) != 512 or row["labels"] != row["input_ids"] or sum(row["attention_mask"]) < 2: + raise SystemExit("ZenFlow's original preprocessing produced invalid causal training targets") +batch = default_data_collator([tokenized[0], tokenized[1]]) +if set(batch) != {"input_ids", "attention_mask", "labels"} or batch["input_ids"].shape != (2, 512): + raise SystemExit(f"ZenFlow's original collator did not drop auxiliary strings: {list(batch)}") +CPUAdamBuilder().load(verbose=True) +print("ZenFlow local offline name resolution:", work / "tatsu-lab/alpaca", "rows:", len(tokenized)) +print("ZenFlow finetuning dependencies:", "transformers", transformers.__version__, "datasets", datasets.__version__) +PY +} + +setup_ds_opsd() { + install_deepspeed_source + install_example_dependencies "transformers==4.57.6" "accelerate>=1.10.1,<2" \ + numpy sentencepiece safetensors ninja + apt-get install -y build-essential + ms_download_models "QWEN25_STUDENT_PATH=Qwen/Qwen2.5-0.5B-Instruct" \ + "QWEN25_TEACHER_PATH=Qwen/Qwen2.5-0.5B" + plant_ci_fixture ci_rac_8.jsonl OPSD_CI_PATH + python - <<'PY' +import importlib.util +import json +import os +from pathlib import Path +import sys + +import torch_npu +import transformers +from deepspeed.ops.op_builder import CPUAdamBuilder +from deepspeed.runtime.rollout import RolloutConfig, build_rollout +from transformers import AutoConfig, AutoTokenizer + +student = AutoTokenizer.from_pretrained(os.environ["QWEN25_STUDENT_PATH"], padding_side="left") +teacher = AutoTokenizer.from_pretrained(os.environ["QWEN25_TEACHER_PATH"], padding_side="left") +# KL compares the same token index in each logit tensor. Validate the entire +# mapping, including added tokens, rather than merely comparing its size. +if student.get_vocab() != teacher.get_vocab() or student.get_added_vocab() != teacher.get_added_vocab(): + raise SystemExit("OPSD student/teacher token IDs differ; token-wise distillation is invalid") +student_config = AutoConfig.from_pretrained(os.environ["QWEN25_STUDENT_PATH"]) +teacher_config = AutoConfig.from_pretrained(os.environ["QWEN25_TEACHER_PATH"]) +if student_config.vocab_size != teacher_config.vocab_size or max(student.get_vocab().values()) >= student_config.vocab_size: + raise SystemExit("OPSD model logit vocabularies do not align with their token IDs") +if not student.chat_template: + raise SystemExit("OPSD instruction student must provide its own chat template") +entry = Path(os.environ["EXAMPLES_ROOT"]) / "training/opsd/main.py" +sys.path.insert(0, str(entry.parent)) +spec = importlib.util.spec_from_file_location("ds_ci_opsd", entry) +upstream = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = upstream +spec.loader.exec_module(upstream) +dataset = upstream.PromptDataset(os.environ["OPSD_CI_PATH"], tokenizer=student, + max_prompt_length=128, prompt_field="prompt") +if len(dataset) != 8 or any(not isinstance(dataset[index], str) or not dataset[index] for index in range(len(dataset))): + raise SystemExit("OPSD fixture must produce eight nonempty native chat-templated prompts") +collator = upstream.LeftPaddedPromptCollator(student, max_prompt_length=128) +batch = collator([dataset[0], dataset[1]]) +if batch["prompt_ids"].shape[0] != 2 or not bool((batch["prompt_attention_mask"].sum(dim=1) > 0).all()): + raise SystemExit("OPSD native prompt collator lost valid prompt tokens") +CPUAdamBuilder().load(verbose=True) +print("OPSD shared token mapping verified:", len(student), "model vocab size:", student_config.vocab_size) +print("OPSD student/teacher use different checkpoint weights; student's chat/EOS policy drives rollout") +print("OPSD rollout API available:", RolloutConfig.__name__, build_rollout.__name__) +print("OPSD local prompt fixture:", len(dataset), "transformers:", transformers.__version__) +PY +} + +verify_data_stack() { + python - <<'PY' +import numpy as np +import pandas as pd +import pyarrow as pa +import scipy +from scipy.optimize import linear_sum_assignment +from scipy.sparse import csr_matrix + +versions = {'numpy': np.__version__, 'scipy': scipy.__version__, + 'pyarrow': pa.__version__, 'pandas': pd.__version__} +print('scientific/data stack:', versions) +expected = {'numpy': '1.26.4', 'scipy': '1.16.3', + 'pyarrow': '20.0.0', 'pandas': '2.2.3'} +if versions != expected: + raise SystemExit(f'incompatible scientific/data stack: {versions}; expected {expected}') +matrix = csr_matrix(np.eye(2)) +assert matrix.sum() == 2 +rows, columns = linear_sum_assignment([[0, 1], [1, 0]]) +assert rows.tolist() == columns.tolist() == [0, 1] +table = pa.Table.from_pandas(pd.DataFrame({'value': [1, 2]})) +assert table.column('value').to_pylist() == [1, 2] +print('SciPy sparse/optimize and Arrow/Pandas checks passed') +PY +} + +verify_installed_runtime() { + python - "$PROFILE" <<'PY' +import os +from pathlib import Path +import sys + +import deepspeed +import torch +import torch_npu +from deepspeed.accelerator import get_accelerator + +source_root = Path(os.environ["TARGET_ROOT"]).resolve() +deepspeed_file = Path(deepspeed.__file__).resolve() +print("runtime torch:", torch.__version__) +print("runtime torch_npu:", torch_npu.__version__) +print("runtime deepspeed:", deepspeed.__version__, deepspeed_file) +if sys.argv[1] in {"ds_cifar", "ds_hf_autotp", "ds_variable_batch", "ds_zenflow", "ds_rac_prune", + "ds_chat_prompt_eval", "ds_chat_reward_eval", "ds_hf_bench_length", "ds_superoffload", + "ds_pin_memory", "ds_zenflow_finetune", "ds_opsd", "ds_finetune_demo", + "ds_sd_distil", "ds_opsd_decode", "ds_legacy_inference_bench", "ds_hf_ds_compare", + "ds_fill_mask_bert", "ds_fill_mask_electra", "ds_fill_mask_roberta", + "ds_t5_translation", "ds_hybrid_rollout", "ds_asr_ctc"}: + accelerator = get_accelerator()._name + available = torch.npu.is_available() + print("runtime accelerator:", accelerator, "NPU available:", available) + if accelerator != "npu" or not available: + raise SystemExit(f"new DeepSpeed NPU example requires available npu accelerator, got {accelerator}") + # Availability alone does not exercise the CANN Python compiler imports. + probe = torch.ones(2, device=get_accelerator().device_name()).sum() + if probe.item() != 2: + raise SystemExit('NPU scientific-stack preflight produced an unexpected result') + print('NPU allocation/reduction preflight passed') +try: + deepspeed_file.relative_to(source_root) +except ValueError as exc: + raise SystemExit( + f"DeepSpeed was not imported from target source {source_root}: " + f"{deepspeed_file}") from exc +PY +} + +# Optional profiles use the same protected dependency and source-install helpers. +setup_inference_dependencies() { + install_deepspeed_source + # 4.51+ resets pipeline.device to a CPU-loaded model.device when distributed + # is already initialized; these older recipes require the pre-change behavior. + install_example_dependencies 'transformers==4.44.2' 'accelerate>=0.30,<2' \ + sentencepiece protobuf safetensors numpy +} + +plant_inference_alias() { + local snapshot="$1" alias="$2" + local work="$GITHUB_WORKSPACE/.ci/deepspeed-inference/$PROFILE" + [[ -d "$snapshot" && "$alias" != /* && "$alias" != *'..'* ]] || { + echo "invalid local inference alias: $alias -> $snapshot" >&2 + return 1 + } + mkdir -p "$work/$(dirname "$alias")" + if [[ -e "$work/$alias" || -L "$work/$alias" ]]; then + [[ -L "$work/$alias" && "$(readlink "$work/$alias")" == "$snapshot" ]] || { + echo "refusing to replace existing inference asset: $work/$alias" >&2 + return 1 + } + else + ln -s "$snapshot" "$work/$alias" + fi + export INFERENCE_CI_WORK="$work" + printf 'INFERENCE_CI_WORK=%s\nHF_HUB_OFFLINE=1\nTRANSFORMERS_OFFLINE=1\n' "$work" >> "$GITHUB_ENV" +} + +setup_ds_legacy_inference_bench() { + setup_inference_dependencies + case "${EXEC:-${EXAMPLE_PATH:-}}" in + benchmarks/inference/bert-bench.py) ms_download_models 'BERT_BASE_CASED_PATH=AI-ModelScope/bert-base-cased' ;; + benchmarks/inference/gpt-bench.py) ms_download_models 'OPT_125M_PATH=facebook/opt-125m' ;; + *) echo 'unknown legacy inference benchmark entry' >&2; return 1 ;; + esac +} + +setup_ds_hf_ds_compare() { + setup_inference_dependencies + ms_download_models 'OPT_125M_PATH=facebook/opt-125m' +} + +setup_ds_fill_mask_bert() { + setup_inference_dependencies + ms_download_models 'BERT_LARGE_CASED_PATH=AI-ModelScope/bert-large-cased' + plant_inference_alias "$BERT_LARGE_CASED_PATH" 'bert-large-cased' +} + +setup_ds_fill_mask_electra() { + setup_inference_dependencies + ms_download_models 'ELECTRA_GENERATOR_PATH=google/electra-base-generator' + plant_inference_alias "$ELECTRA_GENERATOR_PATH" 'google/electra-base-generator' +} + +setup_ds_fill_mask_roberta() { + setup_inference_dependencies + ms_download_models 'ROBERTA_LARGE_PATH=AI-ModelScope/roberta-large' + plant_inference_alias "$ROBERTA_LARGE_PATH" 'roberta-large' +} + +setup_ds_t5_translation() { + setup_inference_dependencies + ms_download_models 'T5_BASE_PATH=AI-ModelScope/t5-base' + # A separate asset view bounds generation, without writing ModelScope cache + # or changing the upstream script. Weights/tokenizer files remain symlinks. + local work="$GITHUB_WORKSPACE/.ci/deepspeed-inference/$PROFILE" + mkdir -p "$work/t5-base" + python - "$T5_BASE_PATH" "$work/t5-base" <<'PY' +import json +import sys +from pathlib import Path +import torch +from transformers import AutoConfig, AutoModelForSeq2SeqLM, GenerationConfig + +snapshot, view = map(Path, sys.argv[1:]) +for source in snapshot.iterdir(): + if source.name in ('generation_config.json', 'config.json'): + continue + target = view / source.name + if target.is_symlink(): + if target.resolve() != source.resolve(): + raise SystemExit(f'unexpected T5 asset alias: {target}') + elif target.exists(): + raise SystemExit(f'refusing to replace T5 asset: {target}') + else: + target.symlink_to(source, target_is_directory=source.is_dir()) +config = AutoConfig.from_pretrained(snapshot, local_files_only=True) +if config.model_type != 't5' or config.num_heads % 2: + raise SystemExit('translation requires a T5 model with heads divisible by TP=2') +with torch.device('meta'): + model = AutoModelForSeq2SeqLM.from_config(config) +names = [name for name, _ in model.named_modules()] +for suffix in ('SelfAttention.o', 'EncDecAttention.o', 'DenseReluDense.wo'): + if not any(name.endswith(suffix) for name in names): + raise SystemExit(f'T5 injection target missing: {suffix}') +generation = GenerationConfig.from_model_config(config) +generation.max_new_tokens = 8 +generation.do_sample = False +# Pipeline applies legacy task_specific_params after generation_config loading. +# Keep its translation task semantics, but supply a legal short min_length. +generation.min_length = 0 +for name, task in (config.task_specific_params or {}).items(): + if name.startswith('translation'): + task['min_length'] = 0 + task['max_length'] = 16 +config.save_pretrained(view) +generation.save_pretrained(view) +print('T5 TP policy checked; local generation_config limits output to 8 new tokens') +PY + export INFERENCE_CI_WORK="$work" + printf 'INFERENCE_CI_WORK=%s\nHF_HUB_OFFLINE=1\nTRANSFORMERS_OFFLINE=1\n' "$work" >> "$GITHUB_ENV" +} + +setup_ds_asr_ctc() { + setup_inference_dependencies + # datasets 4 returns Column objects for result['text']; this original script + # and jiwer 3 expect ordinary lists. Keep the supported list-valued API. + install_example_dependencies 'datasets==3.6.0' 'jiwer>=3,<4' soundfile pyarrow + ms_download_models 'WAV2VEC2_PATH=AI-ModelScope/wav2vec2-base-960h' + plant_inference_alias "$WAV2VEC2_PATH" 'facebook/wav2vec2-base-960h' + # Native local-directory resolution satisfies the upstream's fixed + # load_dataset("librispeech_asr", "clean", split="test") call. These are + # synthetic waveforms, deliberately a CTC forward smoke, not LibriSpeech WER. + python - "$INFERENCE_CI_WORK" <<'PY' +import json +import math +from pathlib import Path +import struct +import sys +import wave +from datasets import load_dataset +import pyarrow as pa +import pyarrow.parquet as pq + +work = Path(sys.argv[1]) +dataset = work / 'librispeech_asr' +dataset.mkdir(exist_ok=True) +rows = [] +for index in range(2): + audio = dataset / f'ci-wave-{index}.wav' + values = [int(2500 * math.sin(2 * math.pi * (220 + index * 110) * i / 16000)) for i in range(16000)] + with wave.open(str(audio), 'wb') as handle: + handle.setnchannels(1) + handle.setsampwidth(2) + handle.setframerate(16000) + handle.writeframes(struct.pack('<' + 'h' * len(values), *values)) + rows.append({'file': str(audio), 'text': 'HELLO WORLD'}) +pq.write_table(pa.Table.from_pylist(rows), dataset / 'test.parquet') +(dataset / 'README.md').write_text('---\nconfigs:\n- config_name: clean\n data_files:\n - split: test\n path: test.parquet\n---\nSynthetic CTC CI inputs, not LibriSpeech.\n') +# Validate the exact original loader contract, not an alternative builder. +import os +os.chdir(work) +loaded = load_dataset('librispeech_asr', 'clean', split='test') +if len(loaded) != 2 or not all(Path(row['file']).is_file() for row in loaded): + raise SystemExit('native clean/test CTC fixture loader contract failed') +if not isinstance(loaded['text'], list): + raise SystemExit('CTC original jiwer call requires a list-valued dataset column') +print('prepared two synthetic 16 kHz CTC inputs; not a LibriSpeech accuracy benchmark') +PY +} +setup_ds_finetune_demo() { + install_deepspeed_source + # The upstream rotary reset requires base/dim but sets max_seq_len_cached=None. + # Llama 4.42 uses base/dim and position_ids without a cached-length comparison; + # newer Llama/Qwen and older Qwen rotary implementations do not meet both rules. + install_example_dependencies 'transformers==4.42.4' 'accelerate>=1.0,<2' \ + 'datasets>=4,<5' safetensors sentencepiece wandb + ms_download_models 'SMOLLM2_135M_PATH=HuggingFaceTB/SmolLM2-135M' + plant_ci_fixture ci_alpaca_16.json ALPACA_CI_PATH + export ALPACA_FINETUNE_DIR="$GITHUB_WORKSPACE/.ci/deepspeed-fixtures/finetune-demo" + echo "ALPACA_FINETUNE_DIR=$ALPACA_FINETUNE_DIR" >> "$GITHUB_ENV" + python - <<'PY' +import importlib.util +import json +import os +from pathlib import Path +import sys + +from datasets import Dataset, load_dataset +import torch +import torch_npu +import transformers +from transformers import AutoConfig, AutoTokenizer, LlamaConfig, LlamaForCausalLM + +if transformers.__version__ != '4.42.4': + raise SystemExit('finetune demo requires Transformers 4.42.4 rotary semantics') +config = AutoConfig.from_pretrained(os.environ['SMOLLM2_135M_PATH'], local_files_only=True) +if config.model_type != 'llama' or config.rope_scaling is not None: + raise SystemExit('finetune demo requires the original SmolLM2 Llama/default-RoPE config') +entry = Path(os.environ['EXAMPLES_ROOT']) / 'training/deepspeed_finetune_demo/finetune_llama.py' +sys.path.insert(0, str(entry.parent)) +spec = importlib.util.spec_from_file_location('ds_ci_finetune_demo', entry) +upstream = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = upstream +spec.loader.exec_module(upstream) +# Execute the exact upstream reset, then a real CPU forward/backward. This +# preflight detects deterministic dependency/API incompatibilities before NPU launch. +with torch.device('cpu'): + tiny = LlamaForCausalLM(LlamaConfig(vocab_size=128, hidden_size=32, + intermediate_size=64, num_hidden_layers=2, + num_attention_heads=4, num_key_value_heads=2, + rope_scaling=None, max_position_embeddings=128)) +upstream._reset_rotary_embeddings(tiny) +ids = torch.arange(16, device='cpu').reshape(1, 16) +loss = tiny(input_ids=ids, labels=ids).loss +if not torch.isfinite(loss): + raise SystemExit('upstream rotary-reset CPU preflight produced nonfinite loss') +loss.backward() +print('real upstream Llama rotary-reset CPU forward/backward:', loss.item()) +tokenizer = AutoTokenizer.from_pretrained(os.environ['SMOLLM2_135M_PATH'], local_files_only=True) +if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token +rows = json.loads(Path(os.environ['ALPACA_CI_PATH']).read_text()) +directory = Path(os.environ['ALPACA_FINETUNE_DIR']) +directory.mkdir(parents=True, exist_ok=True) +Dataset.from_list(rows).to_parquet(directory / 'train.parquet') +dataset = load_dataset(str(directory))['train'] +if len(dataset) != 16: + raise SystemExit('finetune demo local dataset must contain exactly 16 rows') +for row in dataset: + encoded = upstream.preprocess_alpaca(row, tokenizer, max_length=128) + if len(encoded['input_ids']) != 128 or not any(x != -100 for x in encoded['labels'][1:]): + raise SystemExit('finetune demo fixture lost shifted training labels at length 128') +print('finetune demo native local Parquet and labels:', directory, len(dataset)) +PY +} + +download_sd15_ci_model() { + install_example_dependencies 'modelscope==1.37.0' safetensors + TQDM_MININTERVAL="${TQDM_MININTERVAL:-15}" \ + MODELSCOPE_DOWNLOAD_PARALLELS=1 MODELSCOPE_ENABLE_DEFAULT_HASH_VALIDATION=true \ + python - <<'PY' +import json +import os +from pathlib import Path +import tempfile +import time +from modelscope import snapshot_download +from safetensors import safe_open + +# Full Diffusers components, but not duplicate .bin/root .ckpt weights. The +# final unchanged entry reloads the safety checker when constructing its pipeline. +patterns = [ + 'model_index.json', 'feature_extractor/preprocessor_config.json', + 'scheduler/scheduler_config.json', 'tokenizer/*', + 'text_encoder/config.json', 'text_encoder/model.safetensors', + 'unet/config.json', 'unet/diffusion_pytorch_model.safetensors', + 'vae/config.json', 'vae/diffusion_pytorch_model.safetensors', + 'safety_checker/config.json', 'safety_checker/model.safetensors', +] +required = [name for name in patterns if '*' not in name] +required += ['tokenizer/vocab.json', 'tokenizer/merges.txt', 'tokenizer/tokenizer_config.json'] + + +def validate_components(local): + for name in required: + asset = Path(local) / name + if not asset.is_file() or not asset.stat().st_size: + raise RuntimeError(f'SD15 complete local Diffusers asset missing: {asset}') + if asset.suffix == '.json': + json.loads(asset.read_text()) + elif asset.suffix == '.safetensors': + with safe_open(str(asset), framework='np') as weights: + if not list(weights.keys()): + raise RuntimeError(f'SD15 empty component weights: {asset}') + print('validated SD15 tokenizer and all four weight components:', local, flush=True) + + +cache = tempfile.mkdtemp(prefix='deepspeed-sd15-', dir='/tmp') +print('private SD15 ModelScope cache:', cache, flush=True) +for attempt in range(3): + try: + path = snapshot_download('AI-ModelScope/stable-diffusion-v1-5', + allow_patterns=patterns, cache_dir=cache, + enable_file_lock=True, max_workers=2) + validate_components(path) + break + except Exception as exc: + print(f'SD15 attempt {attempt + 1}/3 failed: {exc}', flush=True) + if attempt == 2: + raise + cache = tempfile.mkdtemp(prefix='deepspeed-sd15-retry-', dir='/tmp') + time.sleep(2 * (attempt + 1)) +with open(os.environ['GITHUB_ENV'], 'a') as handle: + handle.write(f'SD15_PATH={path}\n') +print('complete ModelScope SD15 Diffusers components:', path) +PY + local model_path + model_path="$(awk 'index($0,"SD15_PATH=")==1 { value=substr($0,11) } END {print value}' "$GITHUB_ENV")" + [[ -d "$model_path" ]] || { echo "SD15 download did not expose a local directory" >&2; return 1; } + export SD15_PATH="$model_path" +} + +setup_ds_sd_distil() { + install_deepspeed_source + # Teacher UNet is an unwrapped FP32 module. Run the entire recipe in FP32; + # BF16 VAE output would otherwise meet FP32 teacher convolution weights. + install_example_dependencies 'transformers==4.44.2' 'diffusers==0.30.3' \ + 'accelerate==1.10.1' 'datasets>=4,<5' 'pillow>=10' 'torchvision==0.24.0' \ + safetensors sentencepiece numpy tensorboard + download_sd15_ci_model + python - <<'PY' +import importlib.util +import io +import json +import os +from pathlib import Path +import sys + +from datasets import Dataset, Features, Image as DatasetImage, Value, load_dataset +from PIL import Image, ImageDraw +import torch_npu +from transformers import AutoTokenizer + +output = Path(os.environ.get('CI_OUTPUT_DIR', str(Path(os.environ['GITHUB_WORKSPACE']) / 'output'))) +work = output / 'sd-distil-work' +directory = work / 'poloclub/diffusiondb' +directory.mkdir(parents=True, exist_ok=True) +prompts, images = [], [] +for index in range(8): + image = Image.new('RGB', (64, 64), (16 * index, 64, 128)) + draw = ImageDraw.Draw(image) + draw.rectangle((8 + index, 8, 48, 48), fill=(192, 32 * index, 64)) + buffer = io.BytesIO() + image.save(buffer, format='PNG') + images.append({'bytes': buffer.getvalue(), 'path': None}) + prompts.append(f'a colorful geometric rectangle number {index}') +features = Features({'image': DatasetImage(), 'prompt': Value('string')}) +Dataset.from_dict({'image': images, 'prompt': prompts}, features=features).to_parquet(directory / 'train.parquet') +(directory / 'README.md').write_text( + '---\nconfigs:\n- config_name: 2m_first_10k\n data_files:\n' + ' - split: train\n path: train.parquet\n---\nLocal eight-image CI fixture.\n' +) +previous = Path.cwd() +try: + os.chdir(work) + # Native local-directory resolution, not a datasets.load_dataset wrapper. + dataset = load_dataset('poloclub/diffusiondb', '2m_first_10k')['train'] +finally: + os.chdir(previous) +if len(dataset) != 8 or any(row['image'].mode != 'RGB' or row['image'].size != (64, 64) + or not row['prompt'] for row in dataset): + raise SystemExit('SD native local dataset must decode eight RGB 64x64 images/prompts') +entry = Path(os.environ['EXAMPLES_ROOT']) / 'training/stable_diffusion/train_sd_distil_lora.py' +sys.path.insert(0, str(entry.parent)) +spec = importlib.util.spec_from_file_location('ds_ci_sd_distil', entry) +upstream = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = upstream +spec.loader.exec_module(upstream) +# The original dataset reads the original entry's args global, as its normal +# __main__ does. Populate it through the original parser, without changing APIs. +upstream.args = upstream.parse_args([ + '--pretrained_model_name_or_path', os.environ['SD15_PATH'], + '--default_prompt', 'a colorful geometric shape', '--train_batch_size', '1', + '--resolution', '64', '--max_train_steps', '3', '--mixed_precision', 'no', '--report_to', 'none', +]) +tokenizer = AutoTokenizer.from_pretrained(os.environ['SD15_PATH'], subfolder='tokenizer', local_files_only=True) +native = upstream.DreamBoothDataset(dataset['prompt'], dataset['image'], tokenizer, size=64, center_crop=True) +batch = upstream.collate_fn([native[0]], with_prior_preservation=False) +if tuple(batch['pixel_values'].shape) != (1, 3, 64, 64) or batch['input_ids'].shape[0] != 1: + raise SystemExit('SD original DreamBooth dataset/collator produced invalid CI tensors') +model_index = json.loads((Path(os.environ['SD15_PATH']) / 'model_index.json').read_text()) +if model_index['_class_name'] != 'StableDiffusionPipeline': + raise SystemExit('SD CI requires a complete StableDiffusionPipeline snapshot') +print('SD native fixture/collator verified:', directory.resolve(), len(native), tuple(batch['pixel_values'].shape)) +print('SD teacher CFG distillation trains full UNet, not LoRA; dtype remains FP32') +PY +} + +setup_ds_opsd_decode() { + install_deepspeed_source + install_example_dependencies 'transformers==4.57.6' 'accelerate>=1.10.1,<2' safetensors numpy + ms_download_models 'QWEN3_06B_PATH=Qwen/Qwen3-0.6B' + python - <<'PY' +import os +import torch +import torch_npu +from deepspeed.accelerator import get_accelerator +from deepspeed.runtime.rollout.hybrid_engine_rollout import HybridEngineRollout, HybridEngineRolloutConfig +from transformers import AutoTokenizer + +index = get_accelerator().current_device() +# Torch 2.9 maps integer devices to the registered accelerator (PrivateUse1 +# first). Verify the exact tensor protocols the unchanged benchmark uses. +tensor = torch.empty(1, device='cpu').to(index) +random = torch.randint(10, 1000, (1, 2), device=index) +if tensor.device.type != 'npu' or random.device.type != 'npu': + raise SystemExit(f'OPSD decode integer device did not select NPU: {tensor.device}, {random.device}') +tokenizer = AutoTokenizer.from_pretrained(os.environ['QWEN3_06B_PATH'], local_files_only=True) +print('OPSD decode integer-device preflight:', index, tensor.device, random.device, type(tokenizer).__name__) +print('OPSD rollout APIs:', HybridEngineRollout.__name__, HybridEngineRolloutConfig.__name__) +PY +} + +# vLLM examples run in their own prebuilt image, not the training image. +# Preserve the image's torch/NPU/vLLM runtime; prepare a compatible data stack +# and select the tokenizer API required by each native client. +verify_vllm_runtime() { + python - <<'PY' +from importlib.metadata import version +import torch +import torch_npu +from vllm.platforms import current_platform + +for name, expected in {'vllm': '0.23.0', 'vllm-ascend': '0.23.0', + 'torch': '2.10.0', 'torch-npu': '2.10.0.post4'}.items(): + actual = version(name).split('+')[0] + print(f'vLLM image runtime: {name}={actual}') + if actual != expected: + raise SystemExit(f'{name}: expected {expected}, got {actual}; do not replace the image stack') +if current_platform.device_type != 'npu' or not torch.npu.is_available(): + raise SystemExit(f'vLLM must select available NPU, got {current_platform.device_type}') +if torch.ones(2, device='npu').sum().item() != 2: + raise SystemExit('vLLM NPU allocation/reduction preflight failed') +print('vLLM Ascend platform and NPU preflight passed') +PY +} + +prepare_vllm_image() { + verify_vllm_runtime + local constraint_dir="$GITHUB_WORKSPACE/.ci/deepspeed-vllm" + mkdir -p "$constraint_dir" + export PIP_CONSTRAINT="$constraint_dir/protected-serving-stack.txt" + python - "$PIP_CONSTRAINT" "$PROFILE" "${2:-}" <<'PY' +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +import sys + +protected = ('torch', 'torch-npu', 'vllm', 'vllm-ascend', + 'triton', 'triton-ascend', 'safetensors') +if not sys.argv[3]: + protected += ('transformers', 'huggingface-hub', 'tokenizers') +lines = [] +for name in protected: + try: + lines.append(f'{name}=={version(name)}\n') + except PackageNotFoundError: + pass +if sys.argv[3]: + lines += [f'transformers=={sys.argv[3]}\n', 'huggingface-hub>=0.34.0,<1.0\n', + 'tokenizers>=0.22.0,<=0.23.0\n'] +numpy_version = '2.2.6' if sys.argv[2] == 'ds_math500' else '1.26.4' +lines += [f'numpy=={numpy_version}\n', 'scipy==1.16.3\n', + 'pyarrow==20.0.0\n', 'pandas==2.2.3\n'] +Path(sys.argv[1]).write_text(''.join(lines)) +print('Serving runtime and compatible data constraints:\n' + ''.join(lines)) +PY + install_deepspeed_source + # The original GSM8K/MMLU calls include trust_remote_code; use datasets 3.x + # with native local-directory loading, without replacing load_dataset. + install_example_dependencies numpy scipy pyarrow pandas transformers "${1:-datasets==3.6.0}" + python -c 'import numpy, scipy, pandas, pyarrow; print("vLLM data stack:", numpy.__version__, scipy.__version__, pandas.__version__, pyarrow.__version__)' + ms_download_models 'VLLM_CI_MODEL_PATH=Qwen/Qwen2.5-0.5B-Instruct' + python - <<'PY' +import os +from transformers import AutoTokenizer +tokenizer = AutoTokenizer.from_pretrained(os.environ['VLLM_CI_MODEL_PATH'], local_files_only=True) +print('vLLM CI local tokenizer:', type(tokenizer).__name__) +PY +} + +setup_ds_vllm_eval() { + prepare_vllm_image + export VLLM_CI_DATA_ROOT="$GITHUB_WORKSPACE/.ci/deepspeed-vllm/data" + python - "$FIXTURE_DIR" "$VLLM_CI_DATA_ROOT" <<'PY' +import json +import os +from pathlib import Path +import sys +os.environ['HF_HUB_OFFLINE'] = '1' +os.environ['HF_DATASETS_OFFLINE'] = '1' +from datasets import Dataset, load_dataset + +fixture_root, data_root = map(Path, sys.argv[1:]) +previous = Path.cwd() +for repo, config, fixture, fields in ( + ('openai/gsm8k', 'main', 'ci_gsm8k_8.jsonl', {'question', 'answer'}), + ('cais/mmlu', 'all', 'ci_mmlu_8.jsonl', {'question', 'choices', 'answer'}), +): + rows = [json.loads(line) for line in (fixture_root / fixture).read_text().splitlines() if line.strip()] + if len(rows) != 8 or any(set(row) != fields for row in rows): + raise SystemExit(f'{fixture}: expected eight native-schema rows') + directory = data_root / repo + directory.mkdir(parents=True, exist_ok=True) + Dataset.from_list(rows).to_parquet(directory / 'test.parquet') + (directory / 'README.md').write_text( + f'---\nconfigs:\n- config_name: {config}\n data_files:\n' + ' - split: test\n path: test.parquet\n---\nLocal eight-question CI fixture.\n' + ) + try: + os.chdir(data_root) + # Same repository/config/split call as the unchanged original entry. + loaded = load_dataset(repo, config, split='test', trust_remote_code=True) + finally: + os.chdir(previous) + if loaded.to_list() != rows: + raise SystemExit(f'{repo}: native local-directory resolution changed CI data') + print(f'Native local dataset verified: {repo}/{config}, {len(loaded)} rows') +with open(os.environ['GITHUB_ENV'], 'a') as output: + output.write(f'VLLM_CI_DATA_ROOT={data_root.resolve()}\n') + output.write('HF_HUB_OFFLINE=1\nHF_DATASETS_OFFLINE=1\n') +PY +} + +setup_ds_vllm_traces() { + prepare_vllm_image + plant_ci_fixture ci_rac_8.jsonl RAC_CI_PATH +} + +setup_ds_vllm_bench() { + if [[ "${EXAMPLE_PATH:-}" == benchmarks/inference/mii/run_benchmark.py ]]; then + # This unchanged client calls batch_encode_plus, removed in Transformers 5. + prepare_vllm_image 'datasets==3.6.0' '4.57.6' + else + prepare_vllm_image + fi + install_example_dependencies loguru tabulate requests 'pydantic>=2,<3' + local work="$GITHUB_WORKSPACE/.ci/deepspeed-vllm/benchmark" + mkdir -p "$work" + ln -s "$VLLM_CI_MODEL_PATH" "$work/ci-model" + echo "VLLM_CI_BENCH_ROOT=$work" >> "$GITHUB_ENV" +} + +setup_ds_aio() { + install_deepspeed_source + apt-get install -y libaio-dev build-essential + python -c 'from deepspeed.ops.op_builder import AsyncIOBuilder; AsyncIOBuilder().load(verbose=True)' +} + +setup_ds_math500() { + prepare_vllm_image 'datasets>=4,<5' + # This lighteval commit carries PR #1211 (tokenizer/TokensPrompt APIs for + # vLLM >=0.19) plus PR #1402, which encodes strings before xxhash 4.x hashing. + # Do not install its vllm extra or replace the image engine. + install_example_dependencies \ + 'lighteval[math] @ git+https://github.com/huggingface/lighteval.git@b634749d160b080071b9a8e3f5e5a7153dd130c0' \ + ray more_itertools + local work="$GITHUB_WORKSPACE/.ci/deepspeed-math500" + python - "$work" "$FIXTURE_DIR/ci_gsm8k_8.jsonl" <<'PY' +import json +from pathlib import Path +import sys + +work, fixture = map(Path, sys.argv[1:]) +(work / 'data').mkdir(parents=True, exist_ok=True) +rows = [json.loads(line) for line in fixture.read_text().splitlines()] +with (work / 'data/test.jsonl').open('w') as output: + for row in rows: + answer = row['answer'].split('####')[-1].strip() + output.write(json.dumps({'problem': row['question'], 'solution': answer, 'answer': answer}) + '\n') +(work / 'task.py').write_text( + 'import os\nfrom dataclasses import replace\n' + 'from lighteval.tasks.tasks.math_500 import math_500\n' + 'TASKS_TABLE = [replace(math_500, name="ci_math500", ' + 'hf_repo=os.environ["MATH500_DATA_PATH"], generation_size=16)]\n' +) +PY + printf 'MATH500_CI_WORK=%s\nMATH500_DATA_PATH=%s/data\n' "$work" "$work" >> "$GITHUB_ENV" +} + +setup_ds_autotune() { + local model="$1" script="$2" + install_deepspeed_source + install_example_dependencies 'transformers==4.57.6' 'datasets==3.6.0' \ + 'accelerate>=1.10.1,<2' 'evaluate==0.4.6' scikit-learn sentencepiece protobuf tabulate + ms_download_models "AUTOTUNE_MODEL_PATH=$model" + local work="$GITHUB_WORKSPACE/.ci/deepspeed-autotune" + mkdir -p "$work" + curl -fL "https://raw.githubusercontent.com/huggingface/transformers/v4.57.6/examples/pytorch/$script" \ + -o "$work/train.py" + if [[ "$script" == text-classification/* ]]; then + cp "$FIXTURE_DIR/ci_mnli_8.jsonl" "$work/train.json" + mkdir -p "$work/accuracy" + curl -fL https://raw.githubusercontent.com/huggingface/evaluate/v0.4.6/metrics/accuracy/accuracy.py \ + -o "$work/accuracy/accuracy.py" + else + cp "$FIXTURE_DIR/ci_rac_8.jsonl" "$work/train.json" + fi + # Autotuner's native scheduler uses localhost SSH/pdsh for experiment cleanup. + # Keys and configuration belong only to this CI container, never the host. + apt-get install -y pdsh openssh-server + ssh-keygen -q -t ed25519 -N '' -f "$work/key" + ssh-keygen -q -t ed25519 -N '' -f "$work/hostkey" + mkdir -p /run/sshd + /usr/sbin/sshd -f /dev/null -p 2222 -h "$work/hostkey" \ + -o ListenAddress=127.0.0.1 -o AuthorizedKeysFile="$work/key.pub" \ + -o PermitRootLogin=prohibit-password -o StrictModes=no -o PasswordAuthentication=no \ + -o UsePAM=yes -o KbdInteractiveAuthentication=no -o PidFile="$work/sshd.pid" + # The launcher preflight and each tuner experiment call plain ssh, not pdsh. + # Ubuntu's client config includes this container-local fragment for all three. + mkdir -p /etc/ssh/ssh_config.d + printf 'Host localhost 127.0.0.1\n HostName 127.0.0.1\n Port 2222\n IdentityFile %s/key\n IdentitiesOnly yes\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' \ + "$work" > /etc/ssh/ssh_config.d/deepspeed-ci.conf + ssh -o BatchMode=yes localhost true + printf 'AUTOTUNE_WORK=%s\n' "$work" >> "$GITHUB_ENV" +} + +setup_ds_autotune_bert() { setup_ds_autotune AI-ModelScope/bert-base-cased text-classification/run_glue.py; } +setup_ds_autotune_deberta() { setup_ds_autotune microsoft/deberta-v3-small text-classification/run_glue.py; } +setup_ds_autotune_distilbert() { setup_ds_autotune AI-ModelScope/distilbert-base-uncased text-classification/run_glue.py; } +setup_ds_autotune_gpt2() { setup_ds_autotune AI-ModelScope/gpt2 language-modeling/run_clm.py; } + +supported_profiles() { + declare -F | awk '/^declare -f setup_/ { sub(/^declare -f setup_/, ""); print }' | paste -sd' ' - +} + +if ! declare -F "setup_${PROFILE}" >/dev/null 2>&1; then + echo "unknown profile: ${PROFILE} (supported: $(supported_profiles))" >&2 + exit 1 +fi + +TARGET_ROOT="${TARGET_ROOT:?TARGET_ROOT is required}" +EXAMPLES_ROOT="${EXAMPLES_ROOT:-$TARGET_ROOT}" +FIXTURE_DIR="${FIXTURE_DIR:-$GITHUB_WORKSPACE/workflows/examples-guards/deepspeed/fixtures}" +GITHUB_WORKSPACE="${GITHUB_WORKSPACE:?GITHUB_WORKSPACE is required}" +GITHUB_ENV="${GITHUB_ENV:?GITHUB_ENV is required}" + +source /usr/local/Ascend/ascend-toolkit/set_env.sh + +select_pip_index +# Do not run the training stack installer or its fixed scientific-stack +# checks for these independently scheduled vLLM image jobs. +case "$PROFILE" in + ds_vllm_eval|ds_vllm_traces|ds_vllm_bench|ds_math500) + "setup_${PROFILE}" + verify_vllm_runtime + verify_installed_runtime + exit 0 + ;; +esac +python -m pip install -U pip setuptools wheel +ensure_torch_stack +python -m pip install numpy scipy pyarrow pandas +verify_data_stack + +"setup_${PROFILE}" +verify_data_stack +verify_installed_runtime