diff --git a/.github/workflows/llama.cpp-examples.yml b/.github/workflows/llama.cpp-examples.yml new file mode 100644 index 000000000..97cdedd41 --- /dev/null +++ b/.github/workflows/llama.cpp-examples.yml @@ -0,0 +1,62 @@ +# llama.cpp examples guard - project thin trigger. +# +# Calls the common engine .github/workflows/examples-template.yml; +# this file only declares what varies per project: schedule, +# concurrency, upstream repo, dispatch input forwarding, and matrix +# parallelism. The guard loop lives in the engine. See +# examples-guards/README.md. + +name: llama.cpp-examples + +concurrency: + # Three groups: schedule runs serialize; each PR serializes per number + # so a new push cancels the stale run; each manual run stands alone. + # Group names are repository-wide, so the PR group carries the project + # name. A bare pr- would let two projects on one PR cancel + # each other. + group: ${{ github.event_name == 'schedule' && 'llama.cpp-examples-schedule' || (github.event_name == 'pull_request' && format('llama.cpp-examples-pr-{0}', github.event.pull_request.number)) || format('manual-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +on: + # Daily at 03:00 Beijing time (UTC 19:00), one hour after + # accelerate-examples. The source repo's llama.cpp-examples cron should + # be turned off once this is on main, so the two guards do not both + # take NPUs. + schedule: + - cron: '0 19 * * *' + workflow_dispatch: + inputs: + target_ref: + description: Branch, tag, or SHA to test (empty = latest upstream release; falls back to master on API failure) + required: false + default: '' + type: string + # PR runs need no merge: GitHub executes the workflow files from the + # PR merge ref. paths lists only this project's files, so a PR that + # touches only the shared engine starts no matrix here. + pull_request: + branches: [main] + types: [opened, synchronize, reopened] + paths: + - 'examples-guards/llama.cpp/**' + - '.github/workflows/llama.cpp-examples.yml' + +permissions: + actions: read + contents: read + +jobs: + # Job ids cannot contain '.'. The project name stays llama.cpp; that + # string is what artifacts and examples-guards paths use. + llama-cpp-examples: + uses: ./.github/workflows/examples-template.yml + with: + project: llama.cpp + upstream_repo: ggml-org/llama.cpp + # Upstream's default branch is master. The engine's own default is + # main, which 404s when a release lookup falls back. + default_branch: master + target_ref: ${{ inputs.target_ref }} + # 9 supported entries, all single-card. 4 leaves room on the + # shared pool while the Qwen GGUF lock serializes the first download. + max_parallel: 4 diff --git a/examples-guards/llama.cpp/README.md b/examples-guards/llama.cpp/README.md new file mode 100644 index 000000000..314db8f55 --- /dev/null +++ b/examples-guards/llama.cpp/README.md @@ -0,0 +1,48 @@ +# llama.cpp example 看护 + +本目录是 ggml-org/llama.cpp 的 example 看护数据,给共用引擎 `.github/workflows/examples-template.yml` 用。薄触发器是 `.github/workflows/llama.cpp-examples.yml`。上游默认分支是 `master`。触发器把 `default_branch` 设成 `master`,发布查询失败时回落到这条分支。 + +被测 ref 优先取上游最新 release。清单路径按 release `v0.6.0` 的 `examples/` 核对过,列出的目录都还在。 + +## 清单里跑什么 + +`supported` 只保留不改上游源码就能跑完、并且用了昇腾的条目能在日志里看到 CANN 设备行的目录。PR 运行 38228583901 上,`examples/simple` 已经这样跑通。 + +不跑的条目: + +- `examples/idle`:Metal 空闲微基准 +- `examples/sycl`:要 Intel oneAPI +- `examples/training`:全量 FP32 微调,CANN 没有对应反向算子 +- `examples/simple-chat`:上游把低于 ERROR 的日志丢掉,CANN 设备行不会出现。要看到这行就得改上游源码。只看退出码 0 会把没落到昇腾的运行当成成功 +- `examples/retrieval`:上游读循环在输入结束后不返回,喂一行会一直挂到超时。让它返回就得改 `retrieval.cpp` +- `examples/batched`、`examples/embedding`、`examples/lookahead`、`examples/lookup`、`examples/speculative`、`examples/speculative-simple`:同一轮里 ggml-cann 在 `aclnnWeightQuantBatchMatmulV2` 报 y 不连续并中止 +- `examples/parallel`:同一轮里 `FusedInferAttentionScore` 要求非 PA 路径上的 key 连续,实际不连续并中止 +- `examples/diffusion`:运行 38249312071 里 Dream 权重已从 Hugging Face 下完并转成 Q8_0 GGUF,随后在同一个 key 不连续错误上中止 + +每条不支持的依据写在 `examples_manifest.yaml` 里该路径的上方,带源码位置或这次 CI 作业链接,不把几条合成一段注释。 + +`v0.6.0` 里另外有 `examples/deprecation-warning`、`examples/llama-eval`、`examples/test-cmake`。源仓清单没有它们,这次不调度。 + +## profile + +`scripts/setup_example.sh` 用函数 `setup_` 解释清单里的 profile。未知 profile 在编译和下载之前退出。 + +- `cann`:打开 `GGML_CANN`,只编 `exec` 指向的那一个 target。Qwen2.5 0.5B Instruct 的 Q4_0 GGUF 先从 huggingface.co 下,失败再从 ModelScope 下。量化选 Q4_0,因为上游 CANN 文档里 910B 的数据类型表写了 FP16、Q8_0、Q4_0、BF16。 +- `cpu`:cmake 不开 `GGML_CANN`。绿灯只表示这个目录在 aarch64 上能编完、能跑完。外部汇总不要把这些条目读成昇腾推理成功。 +- `cmake-pkg`:安装之后再用 `find_package` 编 `examples/simple-cmake-pkg`,确认安装包还能链上 CANN 后端。`LD_LIBRARY_PATH` 指到 `inst/lib`,因为消费方二进制的 RUNPATH 盖不住 `libggml` 再加载的 `libggml-cann`。 + +镜像是 `swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12`。编译器用镜像自带的 cmake、g++、make,不改 apt 源。 + +## 缓存 + +下载落到 runner 的 `~/.cache`,香港池会把这棵目录留到下一次运行。workflow 不写容器 volume,也不依赖源仓的预投递。 + +多条 `cann` 腿共用同一个 Qwen GGUF。脚本先下到本 job 的暂存文件,核对 GGUF 文件头之后再改名进缓存,并用 mkdir 锁避开并发追加写。 + +## 设备锚点 + +`cann`、`cmake-pkg` 的运行日志里必须出现 `CANN` 后接数字的那一行。ggml 在卡没挂上时会静默退回 CPU,进程仍然可以 0 退出。看护只在自己的运行脚本里检查这行,不改上游 example 文件。 + +## 触发 + +定时已经打开,每天一次,UTC 19:00,也就是北京时间 03:00,比 accelerate 的 example 看护晚一小时。源仓同项目的定时仍开着的话会和本仓抢 NPU,合入后由维护者关掉源仓那条。 diff --git a/examples-guards/llama.cpp/examples_manifest.yaml b/examples-guards/llama.cpp/examples_manifest.yaml new file mode 100644 index 000000000..f156281ba --- /dev/null +++ b/examples-guards/llama.cpp/examples_manifest.yaml @@ -0,0 +1,184 @@ +# llama.cpp example guard for the shared engine in examples-template.yml. +# Profiles are implemented by scripts/setup_example.sh. Timeouts include a +# cold-cache budget: this runner pool does not have the source repo's +# pre-seeded weights. Entries that crash in ggml-cann, or that cannot +# prove the CANN device without editing upstream sources, are unsupported. +version: 1 +scan: + root: examples + unit: directories + marker: CMakeLists.txt + max_depth: 1 +supported: + - path: examples/simple + profile: cann + runner: linux-aarch64-a2-1 + npu_devices: '0' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + exec: build/bin/llama-simple + overlay_args: + - -m ${LLAMA_CI_MODEL} + - -n 16 + - -ngl 99 + timeout_minutes: 90 + - path: examples/passkey + profile: cann + runner: linux-aarch64-a2-1 + npu_devices: '0' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + exec: build/bin/llama-passkey + overlay_args: + - -m ${LLAMA_CI_MODEL} + - --junk 16 + - -ngl 99 + - -v + timeout_minutes: 90 + - path: examples/debug + profile: cann + runner: linux-aarch64-a2-1 + npu_devices: '0' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + exec: build/bin/llama-debug + overlay_args: + - -m ${LLAMA_CI_MODEL} + - -p hello + - -ngl 99 + - -c 256 + - --save-logits + - --logits-output-dir ${CI_OUTPUT_DIR} + - --no-warmup + - -v + timeout_minutes: 90 + - path: examples/eval-callback + profile: cann + runner: linux-aarch64-a2-1 + npu_devices: '0' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + exec: build/bin/llama-eval-callback + overlay_args: + - -m ${LLAMA_CI_MODEL} + - -p hello + - -ngl 99 + timeout_minutes: 90 + - path: examples/convert-llama2c-to-ggml + profile: cpu + runner: linux-aarch64-a2-1 + npu_devices: '0' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + exec: build/bin/llama-convert-llama2c-to-ggml + overlay_args: + - --llama2c-model ${FIXTURE_DIR}/stories260K.bin + - --copy-vocab-from-model ${FIXTURE_DIR}/tok512.bin + - --llama2c-output-model ${CI_OUTPUT_DIR}/stories260K.gguf + timeout_minutes: 90 + - path: examples/gguf + profile: cpu + runner: linux-aarch64-a2-1 + npu_devices: '0' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + exec: build/bin/llama-gguf + timeout_minutes: 90 + - path: examples/gguf-hash + profile: cpu + runner: linux-aarch64-a2-1 + npu_devices: '0' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + exec: build/bin/llama-gguf-hash + overlay_args: + - ${FIXTURE_DIR}/ci-demo.gguf + timeout_minutes: 90 + - path: examples/gen-docs + profile: cpu + runner: linux-aarch64-a2-1 + npu_devices: '0' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + exec: build/bin/llama-gen-docs + timeout_minutes: 90 + - path: examples/simple-cmake-pkg + profile: cmake-pkg + runner: linux-aarch64-a2-1 + npu_devices: '0' + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + exec: examples/simple-cmake-pkg/build/llama-simple-cmake-pkg + overlay_args: + - -m ${LLAMA_CI_MODEL} + - -n 16 + - -ngl 99 + timeout_minutes: 120 +unsupported: + # examples/idle/README.md links PR 17766, titled + # "metal : add residency sets keep-alive heartbeat". That PR's log is + # "ggml_metal_init: found device: Apple M2 Ultra". idle.cpp:78 sleeps + # to "simulate idle GPU", then times llama_process. The example measures + # Metal residency after idle, not a CANN workload. + - examples/idle + # ls-sycl-device.cpp:8 includes ggml-sycl.h and line 13 calls + # ggml_backend_sycl_print_sycl_devices(). CMakeLists.txt builds + # llama-ls-sycl-device. README.md says to build with GGML_SYCL_TARGET + # and run `source /opt/intel/oneapi/setvars.sh`. Intel oneAPI, not CANN. + - examples/sycl + # finetune.cpp:85 calls llama_opt_epoch. Lines 33-39 force an F32 KV + # cache because OUT_PROD has no f16 support. README.md:6-7 documents + # only a CPU build with no extra backend, or CUDA with -ngl 999. + # README.md:9 says FLASH_ATTN_EXT has no backward pass. v0.6.0 + # ggml/src/ggml-cann/ggml-cann.cpp has no "backward" string, so the + # CANN backend does not implement the training backward this binary needs. + - examples/training + # simple-chat.cpp:72-77 installs llama_log_set and prints only when + # level >= GGML_LOG_LEVEL_ERROR. The CANN device line is below ERROR, + # so the guard cannot see it unless that filter is edited. Exit 0 + # would not show that the NPU was used. + - examples/simple-chat + # retrieval.cpp:265-268 is `while (true)` then getline and + # common_tokenize, with no break on an empty line or EOF. One prompt + # does not end the process. Exiting would mean editing this loop. + - examples/retrieval + # llama-batched on v0.6.0, CANN 9.1.0, Ascend910B3 aborted in + # ggml-cann.cpp:70 with Invalid_Argument(EZ0026): Parameter y of + # aclnnWeightQuantBatchMatmulV2GetWorkspaceSize is not contiguous. + # Exit 134. Fixing it means editing ggml-cann, not this example. + # https://github.com/Ascend/docs/actions/runs/38228583901/job/114742295302 + - examples/batched + # llama-embedding on run 38228583901 aborted in ggml-cann.cpp:70 with + # Invalid_Argument(EZ0026): Parameter y of + # aclnnWeightQuantBatchMatmulV2GetWorkspaceSize is not contiguous. + # Exit 134. + # https://github.com/Ascend/docs/actions/runs/38228583901/job/114742295286 + - examples/embedding + # llama-lookahead on run 38228583901 aborted in ggml-cann.cpp:70 with + # Invalid_Argument(EZ0026): Parameter y of + # aclnnWeightQuantBatchMatmulV2GetWorkspaceSize is not contiguous. + # Exit 134. + # https://github.com/Ascend/docs/actions/runs/38228583901/job/114742295384 + - examples/lookahead + # llama-lookup on run 38228583901 aborted in ggml-cann.cpp:70 with + # Invalid_Argument(EZ0026): Parameter y of + # aclnnWeightQuantBatchMatmulV2GetWorkspaceSize is not contiguous. + # Exit 134. + # https://github.com/Ascend/docs/actions/runs/38228583901/job/114742295213 + - examples/lookup + # llama-speculative on run 38228583901 aborted in ggml-cann.cpp:70 with + # Invalid_Argument(EZ0026): Parameter y of + # aclnnWeightQuantBatchMatmulV2GetWorkspaceSize is not contiguous. + # Exit 134. + # https://github.com/Ascend/docs/actions/runs/38228583901/job/114742295204 + - examples/speculative + # llama-speculative-simple on run 38228583901 aborted in ggml-cann.cpp:70 + # with Invalid_Argument(EZ0026): Parameter y of + # aclnnWeightQuantBatchMatmulV2GetWorkspaceSize is not contiguous. + # Exit 134. + # https://github.com/Ascend/docs/actions/runs/38228583901/job/114742295188 + - examples/speculative-simple + # llama-parallel on run 38228583901 aborted in ggml-cann.cpp:70 with + # Invalid_Argument(EZ0037): Parameter key of FusedInferAttentionScore + # is invalid because in non-PA scenarios the key tensor must be + # contiguous. Exit 134. + # https://github.com/Ascend/docs/actions/runs/38228583901/job/114742295315 + - examples/parallel + # llama-diffusion-cli on run 38249312071 downloaded Dream-v0-Instruct-7B + # and wrote Dream-v0-Instruct-7B.Q8_0.gguf, then aborted in + # ggml-cann.cpp:70 with Invalid_Argument(EZ0037): Parameter key of + # FusedInferAttentionScore is invalid because in non-PA scenarios the + # key tensor must be contiguous. Exit 134. + # https://github.com/Ascend/docs/actions/runs/38249312071/job/114811506183 + - examples/diffusion diff --git a/examples-guards/llama.cpp/fixtures/ci-demo.gguf b/examples-guards/llama.cpp/fixtures/ci-demo.gguf new file mode 100644 index 000000000..b775d9be4 Binary files /dev/null and b/examples-guards/llama.cpp/fixtures/ci-demo.gguf differ diff --git a/examples-guards/llama.cpp/fixtures/stories260K.bin b/examples-guards/llama.cpp/fixtures/stories260K.bin new file mode 100644 index 000000000..cfebf5a59 Binary files /dev/null and b/examples-guards/llama.cpp/fixtures/stories260K.bin differ diff --git a/examples-guards/llama.cpp/fixtures/tok512.bin b/examples-guards/llama.cpp/fixtures/tok512.bin new file mode 100644 index 000000000..be175e5ee Binary files /dev/null and b/examples-guards/llama.cpp/fixtures/tok512.bin differ diff --git a/examples-guards/llama.cpp/scripts/expand_overlay.py b/examples-guards/llama.cpp/scripts/expand_overlay.py new file mode 100755 index 000000000..6eb5334a6 --- /dev/null +++ b/examples-guards/llama.cpp/scripts/expand_overlay.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Print OVERLAY_ARGS as a shell-quoted token list. + +The workflow passes manifest.overlay_args as a JSON array. Each item is +expanded against this job's environment, so ${LLAMA_CI_MODEL} and +${CI_OUTPUT_DIR} resolve here and nowhere else. +""" +from __future__ import annotations + +import json +import os +import shlex +import sys + + +def main() -> int: + raw = os.environ.get('OVERLAY_ARGS', '').strip() + if not raw or raw in ('null', '""'): + return 0 + try: + items = json.loads(raw) + except json.JSONDecodeError as exc: + print(f'OVERLAY_ARGS is not valid JSON: {exc}', file=sys.stderr) + return 1 + if items in (None, ''): + return 0 + if not isinstance(items, list): + print(f'OVERLAY_ARGS must be a JSON array, got {type(items).__name__}', + file=sys.stderr) + return 1 + tokens: list[str] = [] + for item in items: + if not isinstance(item, str) or not item.strip(): + print('OVERLAY_ARGS items must be non-empty strings', file=sys.stderr) + return 1 + tokens.extend(shlex.split(os.path.expandvars(item), posix=True)) + print(' '.join(shlex.quote(token) for token in tokens)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/examples-guards/llama.cpp/scripts/fetch_gguf.py b/examples-guards/llama.cpp/scripts/fetch_gguf.py new file mode 100755 index 000000000..8cc812dcb --- /dev/null +++ b/examples-guards/llama.cpp/scripts/fetch_gguf.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Download one GGUF into the runner cache. + +Hugging Face first. ModelScope only if that download fails. Bytes land in +a private .part file and are renamed into the cache only after curl +finishes and the GGUF header checks out, so parallel matrix legs do not +append to one shared file. +""" +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + +from nfs_lock import download_lock, sweep_staging + +CACHE = Path(os.environ.get( + 'LLAMA_CPP_CACHE', os.path.expanduser('~/.cache/llama.cpp'))) +MAGIC = b'GGUF' + + +def gguf_ok(path: Path, min_bytes: int) -> bool: + if not path.is_file(): + return False + try: + if path.stat().st_size < min_bytes: + return False + with path.open('rb') as handle: + return handle.read(4) == MAGIC + except OSError: + return False + + +def curl_to(url: str, part: Path) -> None: + part.parent.mkdir(parents=True, exist_ok=True) + if part.is_file(): + with part.open('rb') as handle: + if handle.read(4) != MAGIC: + part.unlink() + subprocess.run( + [ + 'curl', '-fL', '--retry', '5', '--retry-delay', '5', + '--connect-timeout', '30', '-C', '-', url, '-o', str(part), + ], + check=True, + ) + + +def publish(name: str, min_bytes: int, hf_url: str, ms_url: str) -> Path: + dest = CACHE / name + if dest.exists() and not gguf_ok(dest, min_bytes): + dest.unlink() + staging = CACHE / 'staging' / f'{name}.{os.getpid()}' + staging.mkdir(parents=True, exist_ok=True) + part = staging / name + try: + print(f'[download] {name} <- HuggingFace', flush=True) + try: + curl_to(hf_url, part) + except subprocess.CalledProcessError as exc: + print(f'[fallback] HuggingFace failed ({exc}); trying ModelScope', + flush=True) + part.unlink(missing_ok=True) + try: + curl_to(ms_url, part) + except subprocess.CalledProcessError as ms_exc: + print(f'{name}: neither HuggingFace nor ModelScope delivered ' + f'the file ({ms_exc}) - download noise, not an example ' + 'failure', file=sys.stderr) + raise SystemExit(1) from ms_exc + if not gguf_ok(part, min_bytes): + print(f'{name}: downloaded file is not a complete GGUF', + file=sys.stderr) + raise SystemExit(1) + dest.parent.mkdir(parents=True, exist_ok=True) + os.replace(part, dest) + finally: + shutil.rmtree(staging, ignore_errors=True) + print(f'model ready: {dest}', flush=True) + return dest + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--name', required=True) + parser.add_argument('--min-bytes', type=int, required=True) + parser.add_argument('--hf-url', required=True) + parser.add_argument('--ms-url', required=True) + parser.add_argument('--env-var', default='LLAMA_CI_MODEL') + args = parser.parse_args() + os.environ['HF_ENDPOINT'] = 'https://huggingface.co' + sweep_staging(CACHE / 'staging') + dest = CACHE / args.name + with download_lock(CACHE / 'locks', args.name): + if gguf_ok(dest, args.min_bytes): + print(f'reusing {dest}', flush=True) + else: + dest = publish(args.name, args.min_bytes, args.hf_url, args.ms_url) + github_env = os.environ.get('GITHUB_ENV') + if not github_env: + print('GITHUB_ENV is required', file=sys.stderr) + return 1 + with open(github_env, 'a', encoding='utf-8') as handle: + handle.write(f'{args.env_var}={dest}\n') + print(f'{args.env_var}={dest}', flush=True) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/examples-guards/llama.cpp/scripts/nfs_lock.py b/examples-guards/llama.cpp/scripts/nfs_lock.py new file mode 100755 index 000000000..bb21aadd0 --- /dev/null +++ b/examples-guards/llama.cpp/scripts/nfs_lock.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""mkdir lock for the shared runner cache. + +fcntl locks do not exclude jobs on different pods of this NFS volume. +mkdir is atomic across those pods. A heartbeat refreshes the lock mtime +so a waiter can delete a lock left behind by a killed job. +""" +from __future__ import annotations + +import os +import shutil +import threading +import time +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +HEARTBEAT_SECONDS = 30 +LOCK_STALE_SECONDS = 300 +LOCK_POLL_SECONDS = 10 +LOCK_WAIT_SECONDS = 900 +STAGING_STALE_SECONDS = 24 * 3600 + + +def heartbeat(lock: Path, stop: threading.Event) -> None: + while not stop.wait(HEARTBEAT_SECONDS): + try: + os.utime(lock) + except OSError: + return + + +def break_if_stale(lock: Path) -> None: + try: + idle = time.time() - lock.stat().st_mtime + except FileNotFoundError: + return + if idle < LOCK_STALE_SECONDS: + return + print(f'[lock] {lock.name}: no heartbeat for {idle:.0f}s, breaking it', + flush=True) + stale = lock.with_name(f'{lock.name}.stale-{uuid.uuid4().hex}') + try: + os.rename(lock, stale) + except FileNotFoundError: + return + shutil.rmtree(stale, ignore_errors=True) + + +@contextmanager +def download_lock(lock_dir: Path, name: str) -> Iterator[None]: + """Hold lock_dir/name for the body. + + After LOCK_WAIT_SECONDS the body runs without the lock, so a stuck + holder cannot consume the whole job timeout. + """ + lock_dir.mkdir(parents=True, exist_ok=True) + lock = lock_dir / name + deadline = time.monotonic() + LOCK_WAIT_SECONDS + waiting = False + while True: + try: + lock.mkdir() + break + except FileExistsError: + pass + if time.monotonic() >= deadline: + print(f'[lock] {name}: still held after {LOCK_WAIT_SECONDS}s, ' + 'continuing without it', flush=True) + yield + return + if not waiting: + print(f'[lock] {name}: another job is downloading, waiting', + flush=True) + waiting = True + break_if_stale(lock) + time.sleep(LOCK_POLL_SECONDS) + stop = threading.Event() + beat = threading.Thread(target=heartbeat, args=(lock, stop), daemon=True) + beat.start() + try: + yield + finally: + stop.set() + beat.join() + shutil.rmtree(lock, ignore_errors=True) + + +def sweep_staging(staging_root: Path) -> None: + """Remove staging dirs left by killed jobs.""" + if not staging_root.is_dir(): + return + cutoff = time.time() - STAGING_STALE_SECONDS + for entry in staging_root.iterdir(): + try: + if entry.stat().st_mtime < cutoff: + shutil.rmtree(entry, ignore_errors=True) + except OSError: + continue diff --git a/examples-guards/llama.cpp/scripts/run_example.sh b/examples-guards/llama.cpp/scripts/run_example.sh new file mode 100755 index 000000000..2194b2190 --- /dev/null +++ b/examples-guards/llama.cpp/scripts/run_example.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Run one llama.cpp example from the CI working copy. +# Overlay CLI args come from OVERLAY_ARGS, a JSON array. Never +# git add, commit, or push the working copy. +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +EXAMPLE_REL="$1" +TARGET_ROOT="${TARGET_ROOT:?TARGET_ROOT is required}" +CI_OUTPUT_DIR="${CI_OUTPUT_DIR:?CI_OUTPUT_DIR is required}" +EXEC_REL="${EXEC:-}" +PROFILE="${PROFILE:-}" +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +EXAMPLE_PATH="$TARGET_ROOT/$EXAMPLE_REL" + +if [[ ! -e "$EXAMPLE_PATH" ]]; then + echo "example not found: $EXAMPLE_PATH" >&2 + exit 1 +fi +if [[ -n "$EXEC_REL" ]]; then + EXEC_PATH="$TARGET_ROOT/$EXEC_REL" + if [[ ! -f "$EXEC_PATH" ]]; then + echo "exec not found: $EXEC_PATH" >&2 + exit 1 + fi +elif [[ -d "$EXAMPLE_PATH" ]]; then + echo "path is a directory; set exec on the manifest entry: $EXAMPLE_REL" >&2 + exit 1 +fi + +mkdir -p "$CI_OUTPUT_DIR" + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +export HF_ENDPOINT=https://huggingface.co +# Weights were resolved during setup. Do not let the run step re-check +# them against the network. +export HF_HUB_OFFLINE=1 +export TRANSFORMERS_OFFLINE=1 + +if ! overlay=$(python3 "$HERE/expand_overlay.py"); then + exit 1 +fi +eval "EXTRA_ARGS=( $overlay )" + +if [[ -n "$EXEC_REL" ]]; then + echo "running $EXEC_PATH for $EXAMPLE_REL with ${#EXTRA_ARGS[@]} overlay args" +else + echo "running $EXAMPLE_PATH with ${#EXTRA_ARGS[@]} overlay args" +fi +if ((${#EXTRA_ARGS[@]})); then + printf 'overlay arg: %q\n' "${EXTRA_ARGS[@]}" +fi + +cd "$TARGET_ROOT" +export CI_OUTPUT_DIR ASCEND_RT_VISIBLE_DEVICES="${ASCEND_RT_VISIBLE_DEVICES:-0}" + +assert_cann_used() { + local run_log="$1" + # cpu builds are not an NPU result. + if [[ "$PROFILE" == "cpu" ]]; then + return 0 + fi + if ! grep -q 'CANN[0-9]' "$run_log"; then + echo "run used no CANN device; check the container card mounts" >&2 + exit 1 + fi +} + +if [[ -n "$EXEC_REL" ]]; then + RUN_LOG="$CI_OUTPUT_DIR/$(basename "$EXEC_REL").log" + case "$EXAMPLE_REL" in + examples/gguf) + "$EXEC_PATH" "$CI_OUTPUT_DIR/ci-demo.gguf" w 2>&1 | tee "$RUN_LOG" + "$EXEC_PATH" "$CI_OUTPUT_DIR/ci-demo.gguf" r 2>&1 | tee -a "$RUN_LOG" + ;; + *) + "$EXEC_PATH" "${EXTRA_ARGS[@]}" 2>&1 | tee "$RUN_LOG" + ;; + esac + assert_cann_used "$RUN_LOG" + exit 0 +fi + +if command -v python3 >/dev/null 2>&1; then + PYTHON=python3 +else + PYTHON=python +fi +case "$EXAMPLE_PATH" in + *.sh) + bash "$EXAMPLE_PATH" "${EXTRA_ARGS[@]}" + ;; + *.py) + "$PYTHON" "$EXAMPLE_PATH" "${EXTRA_ARGS[@]}" + ;; + *) + echo "unsupported example type: $EXAMPLE_PATH" >&2 + exit 1 + ;; +esac diff --git a/examples-guards/llama.cpp/scripts/setup_example.sh b/examples-guards/llama.cpp/scripts/setup_example.sh new file mode 100755 index 000000000..07d246642 --- /dev/null +++ b/examples-guards/llama.cpp/scripts/setup_example.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Prepare the CI environment for one supported llama.cpp example. +# $1 is the manifest profile. Unknown profiles fail before any install. +# EXEC and EXAMPLE_PATH come from the workflow as environment variables. +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +PROFILE="$1" +EXEC_REL="${EXEC:-}" +EXAMPLE_REL="${EXAMPLE_PATH:-}" +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# This pool reaches huggingface.co directly. Pin the endpoint so a +# leftover hf-mirror value cannot 308 a bare repo id back to the origin. +export HF_ENDPOINT=https://huggingface.co + +# Published qwen2.5-0.5b-instruct-q4_0.gguf is 428730208 bytes. The floor +# rejects a truncated download; the header check is in fetch_gguf.py. +QWEN_MIN_BYTES=300000000 +QWEN_HF_URL=https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/qwen2.5-0.5b-instruct-q4_0.gguf +QWEN_MS_URL=https://www.modelscope.cn/models/Qwen/Qwen2.5-0.5B-Instruct-GGUF/resolve/master/qwen2.5-0.5b-instruct-q4_0.gguf + +require_exec() { + if [[ -z "$EXEC_REL" ]]; then + echo "EXEC is required for profile ${PROFILE}" >&2 + exit 1 + fi +} + +source_cann() { + export PATH="/usr/local/sbin:$PATH" + source /usr/local/Ascend/ascend-toolkit/set_env.sh + if command -v npu-smi >/dev/null 2>&1; then + npu-smi info + fi +} + +assert_cann_lib() { + if ! compgen -G "$1/libggml-cann.*" >/dev/null; then + echo "build produced no CANN backend library under $1" >&2 + exit 1 + fi +} + +cmake_llama() { + local -a extra=("$@") + cmake -S "$TARGET_ROOT" -B "$TARGET_ROOT/build" \ + -DCMAKE_BUILD_TYPE=Release \ + "${extra[@]}" +} + +build_target() { + require_exec + cmake --build "$TARGET_ROOT/build" --target "$(basename "$EXEC_REL")" -j "$(nproc)" +} + +fetch_qwen() { + python3 "$HERE/fetch_gguf.py" \ + --name qwen2.5-0.5b-instruct-q4_0.gguf \ + --min-bytes "$QWEN_MIN_BYTES" \ + --hf-url "$QWEN_HF_URL" \ + --ms-url "$QWEN_MS_URL" +} + +setup_cann() { + source_cann + fetch_qwen + cmake_llama -DGGML_CANN=on + build_target + assert_cann_lib "$TARGET_ROOT/build/bin" +} + +setup_cpu() { + cmake_llama + build_target +} + +setup_cmake-pkg() { + source_cann + fetch_qwen + cmake -S "$TARGET_ROOT" -B "$TARGET_ROOT/build" \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_CANN=on \ + -DCMAKE_INSTALL_PREFIX="$TARGET_ROOT/inst" + cmake --build "$TARGET_ROOT/build" -j "$(nproc)" + cmake --install "$TARGET_ROOT/build" --prefix "$TARGET_ROOT/inst" + cmake -S "$TARGET_ROOT/examples/simple-cmake-pkg" \ + -B "$TARGET_ROOT/examples/simple-cmake-pkg/build" \ + -DCMAKE_PREFIX_PATH="$TARGET_ROOT/inst/lib/cmake" + cmake --build "$TARGET_ROOT/examples/simple-cmake-pkg/build" -j "$(nproc)" + if ! compgen -G "$TARGET_ROOT/inst/lib/libggml-cann.*" >/dev/null \ + && ! compgen -G "$TARGET_ROOT/build/bin/libggml-cann.*" >/dev/null; then + echo "build produced no CANN backend library under inst/lib or build/bin" >&2 + exit 1 + fi + local bin="$TARGET_ROOT/examples/simple-cmake-pkg/build/llama-simple-cmake-pkg" + if [[ ! -f "$bin" ]]; then + echo "cmake-pkg binary not found: $bin" >&2 + exit 1 + fi + # RUNPATH on the consumer binary does not apply to libggml's NEEDED + # backends. The run step sources CANN again and prepends the toolkit. + echo "LD_LIBRARY_PATH=$TARGET_ROOT/inst/lib" >> "$GITHUB_ENV" +} + +supported_profiles() { + declare -F | awk '/^declare -f setup_/ { sub(/^declare -f setup_/, ""); print }' | paste -sd' ' - +} + +if ! declare -F "setup_${PROFILE}" >/dev/null 2>&1; then + echo "unknown profile: ${PROFILE} (supported: $(supported_profiles))" >&2 + exit 1 +fi + +TARGET_ROOT="${TARGET_ROOT:?TARGET_ROOT is required}" +GITHUB_ENV="${GITHUB_ENV:?GITHUB_ENV is required}" + +"setup_${PROFILE}"