diff --git a/.github/workflows/tgi-examples.yml b/.github/workflows/tgi-examples.yml new file mode 100644 index 000000000..639979f3f --- /dev/null +++ b/.github/workflows/tgi-examples.yml @@ -0,0 +1,80 @@ +# tgi examples guard - project thin trigger. +# +# Calls the common engine .github/workflows/examples-template.yml; this +# file only declares what varies per project: schedule, concurrency, +# upstream repo, dispatch input forwarding, and matrix parallelism. +# The guard loop (release + workflow-files monitor / manifest-check / +# run-example matrix / validate-results / save-monitor-state) lives in +# the engine. See examples-guards/README.md for the design. +# +# Unlike most projects, TGI's examples are serving scenarios (there is +# no upstream examples/ of demo scripts in a serving stack): each +# scenario boots a real TGI service on the NPU, sends /generate (or +# streaming / concurrent) requests and asserts the replies. The +# scenarios live in the cosdt fork's examples/npu/ - the fork is the +# Ascend-adaptation upstream, and a serving example must run against +# the exact ref it was built from, so example and code stay in one +# repo (non-split mode, zero engine changes). + +name: tgi-examples + +concurrency: + # Three groups: schedule runs serialize (a run cancelled mid-way + # loses its outcome writeback, and the outcome drives the next + # cycle's retry); each PR serializes per number so a new push + # cancels the stale run instead of queueing another matrix on the + # NPU runners; each manual run stands alone. Group names are + # repository-wide, so the PR group carries the workflow name: a bare + # pr- would let two projects' runs on one PR cancel each + # other. + group: ${{ github.event_name == 'schedule' && 'tgi-examples-schedule' || (github.event_name == 'pull_request' && format('tgi-examples-pr-{0}', github.event.pull_request.number)) || format('manual-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +on: + # Daily at 03:00 Beijing time (UTC 19:00), off accelerate's 02:00 + # Beijing (UTC 18:00) slot. + schedule: + - cron: '0 19 * * *' + workflow_dispatch: + inputs: + target_ref: + description: Branch, tag, or SHA to test (empty = latest upstream release; falls back to main on API failure). During bring-up, pass the fork branch that carries examples/npu/. + required: false + default: '' + type: string + # PR runs need no merge: GitHub executes the workflow files from the + # PR merge ref, so a newly added trigger and engine run on the PR + # that adds them. Whether a fork PR waits for a maintainer's approval + # first is the repository's fork-PR policy. `pull_request`, not + # `pull_request_target`: the declared read permissions are all the + # engine needs. paths lists only this project's files, as the + # quick-start triggers do: a PR that touches only the shared engine, + # scripts or schema starts no project's matrix. + pull_request: + branches: [main] + types: [opened, synchronize, reopened] + paths: + - 'examples-guards/tgi/**' + - '.github/workflows/tgi-examples.yml' + +# Engine jobs query the GitHub Job API (validate-results); called +# workflows run with the caller's GITHUB_TOKEN permissions. +permissions: + actions: read + contents: read + +jobs: + tgi-examples: + uses: ./.github/workflows/examples-template.yml + with: + # Namespaces cache keys (examples-monitor-state-tgi_*), artifact + # names and the scheduling paths under examples-guards/tgi. + project: tgi + upstream_repo: cosdt/text-generation-inference + target_ref: ${{ inputs.target_ref }} + # 1: a serving scenario owns the whole runner (2-card TP spans + # both NPUs, and stop-tgi.sh tears down every launcher on the + # host), and parallel legs would contend on the shared cargo + # target dir. Serial legs + the persistent cargo target dir keep + # the release-opt build incremental instead of rebuilding per leg. + max_parallel: 1 diff --git a/examples-guards/tgi/constraints-npu.txt b/examples-guards/tgi/constraints-npu.txt new file mode 100644 index 000000000..4ea50ee2e --- /dev/null +++ b/examples-guards/tgi/constraints-npu.txt @@ -0,0 +1,45 @@ +# CUDA exclusion constraints for the tgi examples guard on Ascend NPU. +# The runners are aarch64 without CUDA; pip resolvers on the default +# index would happily pull nvidia-* / cuda-* metapackages (and +# flashinfer) as transitive deps, which either fail to install or are +# dead weight. Same list as the tgi quick-start test class +# (tests/tgi/test_quick_start_ascend.py) and the other projects' +# constraints files. +cuda-toolkit<0 +cuda-python<0 +cuda-bindings<0 +cuda-core<0 +cuda-pathfinder<0 +flashinfer-python<0 +nvidia-cublas<0 +nvidia-cuda-runtime<0 +nvidia-cuda-nvrtc<0 +nvidia-cuda-cupti<0 +nvidia-cudnn<0 +nvidia-cudnn-frontend<0 +nvidia-cufft<0 +nvidia-curand<0 +nvidia-cusolver<0 +nvidia-cusparse<0 +nvidia-cutlass-dsl<0 +nvidia-cutlass-dsl-libs-base<0 +nvidia-cutlass-dsl-libs-core<0 +nvidia-cutlass-dsl-libs-cu12<0 +nvidia-ml-py<0 +nvidia-nccl<0 +nvidia-nvjitlink<0 +nvidia-nvtx<0 +nvidia-cublas-cu12<0 +nvidia-cuda-nvdisasm<0 +nvidia-cuda-runtime-cu12<0 +nvidia-cuda-nvrtc-cu12<0 +nvidia-cuda-cupti-cu12<0 +nvidia-cudnn-cu12<0 +nvidia-cufft-cu12<0 +nvidia-curand-cu12<0 +nvidia-cusolver-cu12<0 +nvidia-cusparse-cu12<0 +nvidia-cusparselt-cu12<0 +nvidia-nccl-cu12<0 +nvidia-nvjitlink-cu12<0 +nvidia-nvtx-cu12<0 diff --git a/examples-guards/tgi/examples_manifest.yaml b/examples-guards/tgi/examples_manifest.yaml new file mode 100644 index 000000000..87d7f2b25 --- /dev/null +++ b/examples-guards/tgi/examples_manifest.yaml @@ -0,0 +1,55 @@ +version: 1 +scan: + root: examples + include_extensions: ['.sh'] + +supported: + # 四个场景共用 profile: tgi-serve（Rust 构建 + Python server 安装 + + # 产物冒烟），由 setup_example.sh 解释。npu_devices 必须与场景脚本 + # 的默认 --devices 一致（单卡 "0"、双卡 "0,1"），改卡时两处同步。 + # timeout 统一 180 分钟：冷缓存首腿要付 release-opt 编译（约 2 小时）， + # 热缓存腿只有几分钟的增量构建 + 服务场景本身。 + + # 单卡基线：起服务 + 一条 /generate。 + - path: examples/npu/serve_basic.sh + profile: tgi-serve + runner: linux-aarch64-a2-1 + npu_devices: "0" + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: [] + timeout_minutes: 180 + + # 双卡张量并行（--num-shard 2，HCCL 权重切分）。 + - path: examples/npu/serve_tensor_parallel.sh + profile: tgi-serve + runner: linux-aarch64-a2-2 + npu_devices: "0,1" + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: [] + timeout_minutes: 180 + + # SSE 流式（/generate_stream）：增量 token 事件 + 流式最终文本与 + # 非流式回复逐字一致（两者均为贪心解码，比对是确定性断言）。 + - path: examples/npu/serve_streaming.sh + profile: tgi-serve + runner: linux-aarch64-a2-1 + npu_devices: "0" + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: [] + timeout_minutes: 180 + + # 并发请求：NUM_REQUESTS 条 /generate 并行打同一服务（continuous + # batching 调度路径），全部须返回非空回复。 + - path: examples/npu/serve_concurrent.sh + profile: tgi-serve + runner: linux-aarch64-a2-1 + npu_devices: "0" + image: swr.cn-south-1.myhuaweicloud.com/ascendhub/cann:9.1.0-910b-ubuntu22.04-py3.12 + overlay_args: [] + timeout_minutes: 180 + +unsupported: + # 非独立入口的配套物：common.sh 是场景脚本 source 的公共助手 + # （请求 payload 构造 + 响应断言），不带 main 流程，独立执行无事可做。 + # 扫描引擎按 include_extensions 会命中，落 unsupported 是审计去重。 + - examples/npu/common.sh diff --git a/examples-guards/tgi/scripts/run_example.sh b/examples-guards/tgi/scripts/run_example.sh new file mode 100755 index 000000000..e60df929f --- /dev/null +++ b/examples-guards/tgi/scripts/run_example.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# TGI examples guard: run one serving example. +# +# $1 is the manifest entry path (the scenario script itself, relative +# to the target root). The script boots the service via the fork's +# start-tgi.sh, sends requests, asserts the replies and tears the +# service down; its exit code is the result. Never git add/commit/push. +# +# Env (provided by the engine, per the project script contract): +# TARGET_ROOT / CI_OUTPUT_DIR / OVERLAY_ARGS (JSON array, possibly []) +# ASCEND_RT_VISIBLE_DEVICES (manifest npu_devices) +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +EXAMPLE_REL="$1" +TARGET_ROOT="${TARGET_ROOT:?TARGET_ROOT is required}" +CI_OUTPUT_DIR="${CI_OUTPUT_DIR:?CI_OUTPUT_DIR is required}" + +EXAMPLE_PATH="$TARGET_ROOT/$EXAMPLE_REL" +if [[ ! -f "$EXAMPLE_PATH" ]]; then + echo "example not found: $EXAMPLE_PATH" >&2 + exit 1 +fi + +# The manifest never sets EXEC / LAUNCHER for tgi (the path is the +# entry script itself); reject them loudly instead of silently ignoring. +if [[ -n "${EXEC:-}" ]]; then + echo "EXEC is not supported by the tgi run script (path is the script)" >&2 + exit 2 +fi +if [[ -n "${LAUNCHER:-}" ]]; then + echo "LAUNCHER is not supported by the tgi run script" >&2 + exit 2 +fi + +if command -v python3 >/dev/null 2>&1; then + PYTHON=python3 +else + PYTHON=python +fi + +# CANN toolkit env: the scenario's start-tgi.sh / run-npu.sh need +# ASCEND_* and the toolkit libs on LD_LIBRARY_PATH. +for candidate in /usr/local/Ascend/ascend-toolkit/set_env.sh /usr/local/Ascend/cann/set_env.sh; do + if [[ -f "$candidate" ]]; then + source "$candidate" + break + fi +done + +mkdir -p "$CI_OUTPUT_DIR" + +# The workflow serializes manifest.overlay_args as JSON. Expand each +# item with shell quoting intact, then allow CI paths such as +# ${CI_OUTPUT_DIR} to resolve only in this job's environment (same +# expander as the accelerate run script). +expand_overlay() { + "$PYTHON" - <<'PY' +import json +import os +import shlex + +raw = os.environ.get('OVERLAY_ARGS', '').strip() +if not raw or raw in ('null', '""'): + raise SystemExit(0) +try: + items = json.loads(raw) +except json.JSONDecodeError as exc: + raise SystemExit(f'OVERLAY_ARGS is not valid JSON: {exc}') from exc +if items in (None, ''): + raise SystemExit(0) +if not isinstance(items, list): + raise SystemExit( + f'OVERLAY_ARGS must be a JSON array, got {type(items).__name__}') +tokens = [] +for item in items: + if not isinstance(item, str) or not item.strip(): + raise SystemExit('OVERLAY_ARGS items must be non-empty strings') + tokens.extend(shlex.split(os.path.expandvars(item), posix=True)) +print(' '.join(shlex.quote(token) for token in tokens)) +PY +} + +eval "EXTRA_ARGS=( $(expand_overlay) )" + +echo "running $EXAMPLE_PATH with ${#EXTRA_ARGS[@]} overlay args" +if ((${#EXTRA_ARGS[@]})); then + printf 'overlay arg: %q\n' "${EXTRA_ARGS[@]}" +fi + +cd "$TARGET_ROOT" +CI_OUTPUT_DIR="$CI_OUTPUT_DIR" bash "$EXAMPLE_PATH" "${EXTRA_ARGS[@]}" diff --git a/examples-guards/tgi/scripts/setup_example.sh b/examples-guards/tgi/scripts/setup_example.sh new file mode 100755 index 000000000..7a901edce --- /dev/null +++ b/examples-guards/tgi/scripts/setup_example.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# TGI examples guard: prepare the environment for one example job. +# +# $1 is the manifest entry's profile. Only `tgi-serve` exists today: +# build the release-opt launcher + router, install the Python server +# (doc §4 recipe, locked deps), and smoke-check the artifacts. Every +# scenario script then boots the service from the same checkout. +# +# Build economics: cargo target dir + rustup/cargo homes live under +# ~/.cache/tgi-examples, the runner's persistent volume. The first leg +# on a machine pays the full release-opt build (~2 h); later legs hit +# the incremental cache. max_parallel is 1 for tgi (see tgi-examples.yml), +# so no concurrent legs contend on the cargo target-dir lock. +# +# Env (provided by the engine, per the project script contract): +# TARGET_ROOT / GITHUB_WORKSPACE / GITHUB_ENV +set -euo pipefail + +PROFILE="${1:?usage: $0 }" + +# Unknown profile must fail before installing anything (contract). +if [[ "$PROFILE" != "tgi-serve" ]]; then + echo "unknown profile: ${PROFILE} (supported: tgi-serve)" >&2 + exit 1 +fi + +TARGET_ROOT="${TARGET_ROOT:?TARGET_ROOT is required}" +GITHUB_WORKSPACE="${GITHUB_WORKSPACE:?GITHUB_WORKSPACE is required}" +GITHUB_ENV="${GITHUB_ENV:?GITHUB_ENV is required}" + +CLUSTER_PIP_HOST=cache-service.nginx-pypi-cache.svc.cluster.local +CLUSTER_PIP_INDEX="http://${CLUSTER_PIP_HOST}/pypi/simple" +ASCEND_PIP_INDEX=https://repo.huaweicloud.com/ascend/repos/pypi +ALIYUN_PIP_INDEX=https://mirrors.aliyun.com/pypi/simple/ + +# 1) pip index: prefer the cluster cache, fall back to the Aliyun +# mirror (same selection as the other projects' setup scripts). The uv +# env below mirrors the proven combo from the quick-start engine +# (UV_INDEX_STRATEGY=unsafe-best-match + UV_INSECURE_HOST for the http +# cluster cache); uv would otherwise refuse an http index. +if python -c " +import os +import urllib.error +import urllib.request +try: + urllib.request.urlopen(os.environ['CLUSTER_PIP_INDEX'], timeout=3) +except urllib.error.HTTPError: + pass +" 2>/dev/null; then + PIP_INDEX_URL="$CLUSTER_PIP_INDEX" + PIP_TRUSTED_HOST="$CLUSTER_PIP_HOST" +else + PIP_INDEX_URL="$ALIYUN_PIP_INDEX" + unset PIP_TRUSTED_HOST +fi +export PIP_INDEX_URL +export PIP_EXTRA_INDEX_URL="$ASCEND_PIP_INDEX" +export PIP_TIMEOUT=120 +export UV_INDEX_URL="$PIP_INDEX_URL" +export UV_EXTRA_INDEX_URL="$ASCEND_PIP_INDEX" +export UV_INDEX_STRATEGY=unsafe-best-match +export UV_HTTP_TIMEOUT=120 +# Container has no venv: uv pip install must target the system Python +# (same UV_SYSTEM_PYTHON=1 the quick-start engine sets; without it uv +# errors "No virtual environment found"). UV_NO_CACHE=1 keeps uv's +# own cache off the shared persistent volume - the cluster pip cache +# is the single shared cache, uv's would be a redundant corruption +# surface (same as the quick-start engine's proven combo). +export UV_SYSTEM_PYTHON=1 +export UV_NO_CACHE=1 +# Persistent-volume root (shared across the pool's runners, same disk +# that carries the HF/model caches): the pip wheel cache lives here so +# torch_npu is downloaded once and reused by later legs and runs +# instead of re-downloading every job (the default ~/.cache/pip would +# land on the same disk, but being explicit keeps it in one place). +TGI_CACHE_ROOT="${HOME}/.cache/tgi-examples" +export PIP_CACHE_DIR="$TGI_CACHE_ROOT/pip" +mkdir -p "$PIP_CACHE_DIR" +if [[ -n "${PIP_TRUSTED_HOST:-}" ]]; then + export PIP_TRUSTED_HOST + export UV_INSECURE_HOST="$PIP_TRUSTED_HOST" +fi +echo "pip index: $PIP_INDEX_URL" + +# 2) CUDA exclusion list (same as the quick-start test class) so pip / +# uv resolvers never pull nvidia-* / cuda-* metapackages or flashinfer. +CONSTRAINTS_FILE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/constraints-npu.txt" +export PIP_CONSTRAINT="$CONSTRAINTS_FILE" +export UV_CONSTRAINT="$CONSTRAINTS_FILE" + +# 3) CANN toolkit env (ASCEND_HOME / LD_LIBRARY_PATH etc. for the build +# and the later service boot). +for candidate in /usr/local/Ascend/ascend-toolkit/set_env.sh /usr/local/Ascend/cann/set_env.sh; do + if [[ -f "$candidate" ]]; then + source "$candidate" + echo "sourced CANN env from $candidate" + break + fi +done + +# 4) System build deps (doc §4 prerequisites, same apt line as +# quick_start.md): the launcher links system OpenSSL (openssl-sys) +# which resolves through pkg-config at build time; protobuf-compiler +# provides the protoc binary PROTOC points at below. +apt-get update -qq +apt-get install -y -qq build-essential protobuf-compiler pkg-config libssl-dev curl python3-dev + +# 5) torch stack: reuse the image's preinstalled torch_npu when +# importable; otherwise install the current recommended pair from the +# Ascend source (same probe-first approach as the quick-start harness). +if python -c "import torch, torch_npu" >/dev/null 2>&1; then + python -c "import torch, torch_npu; print('reusing image torch stack:', torch.__version__, torch_npu.__version__)" +else + echo "installing torch_npu from the Ascend source" + python -m pip install torch_npu +fi + +# 6) Rust toolchain under the persistent volume. rust-toolchain.toml +# pins 1.85.1; a cargo build auto-installs the pinned channel via +# rustup when the default differs. CARGO_TARGET_DIR on the persistent +# volume makes the release-opt build incremental across legs and runs. +export RUSTUP_HOME="$TGI_CACHE_ROOT/rustup" +export CARGO_HOME="$TGI_CACHE_ROOT/cargo" +export CARGO_TARGET_DIR="$TGI_CACHE_ROOT/target" +mkdir -p "$RUSTUP_HOME" "$CARGO_HOME" "$CARGO_TARGET_DIR" +if ! command -v cargo >/dev/null 2>&1; then + echo "installing rustup (default toolchain 1.85.1)" + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --default-toolchain 1.85.1 --profile minimal +fi +export PATH="$CARGO_HOME/bin:$PATH" + +# 7) Rust build: release-opt launcher + router (doc §4 recipe). +cd "$TARGET_ROOT" +export PYO3_PYTHON="$(command -v python)" +export PROTOC="$(command -v protoc)" +echo "building release-opt launcher + router (target dir: $CARGO_TARGET_DIR)" +cargo build --profile release-opt -p text-generation-launcher -p text-generation-router-v3 + +# run-npu.sh resolves the binaries at /target/release-opt; +# point the checkout at the persistent target dir so the scenario +# scripts run unchanged. +mkdir -p "$TARGET_ROOT/target" +ln -sfn "$CARGO_TARGET_DIR/release-opt" "$TARGET_ROOT/target/release-opt" + +# 8) Python server (doc §4 recipe, locked deps). `-e server` with +# --no-deps: deps come from requirements_ascend.txt only, no drift via +# pyproject ranges. gen-server-raw does proto codegen only (gen-server +# would additionally install from the outdated requirements_gen.txt). +python -m pip install -q uv +uv pip install --no-build-isolation -r server/requirements_ascend.txt +uv pip install --no-build-isolation --no-deps -e server +uv pip install --no-build-isolation "grpcio-tools==1.84.0" "mypy-protobuf==3.6.0" +make -C server gen-server-raw + +# 9) Artifact smoke check (doc check-build equivalent). +"$TARGET_ROOT/target/release-opt/text-generation-launcher" --version +"$TARGET_ROOT/target/release-opt/text-generation-router" --version +python -c "import text_generation_server; print('server import ok')" + +# 10) Persist what the run step still needs: pip plumbing for +# start-tgi.sh's lazy `pip install modelscope` (first model use) and +# the cargo bin dir on PATH. The uv env must be persisted too: +# start-tgi.sh prefers `uv pip install modelscope` when uv is on PATH +# (step 8 installs it), and the run step is a fresh process - the +# exports above stay in the setup process. Without UV_SYSTEM_PYTHON +# that lazy uv call dies with "No virtual environment found" in the +# container (no venv), which is what the first green-setup run hit. +{ + echo "PIP_INDEX_URL=$PIP_INDEX_URL" + echo "PIP_EXTRA_INDEX_URL=$ASCEND_PIP_INDEX" + echo "PIP_CONSTRAINT=$CONSTRAINTS_FILE" + echo "UV_SYSTEM_PYTHON=1" + echo "UV_NO_CACHE=1" + echo "UV_INDEX_URL=$UV_INDEX_URL" + echo "UV_EXTRA_INDEX_URL=$ASCEND_PIP_INDEX" + echo "UV_INDEX_STRATEGY=unsafe-best-match" + echo "UV_HTTP_TIMEOUT=120" + echo "UV_CONSTRAINT=$CONSTRAINTS_FILE" + echo "PATH=$CARGO_HOME/bin:$PATH" + if [[ -n "${PIP_TRUSTED_HOST:-}" ]]; then + echo "PIP_TRUSTED_HOST=$PIP_TRUSTED_HOST" + echo "UV_INSECURE_HOST=$PIP_TRUSTED_HOST" + fi +} >> "$GITHUB_ENV" + +echo "setup tgi-serve done"