diff --git a/.dockerignore b/.dockerignore index fa56d2d..b4c6f04 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,18 @@ .git +.claude .idea __pycache__ *.pyc .venv +.benchmarks +.pytest_cache +.ruff_cache +.coverage .env -data/ +# K8s 音频冒烟镜像只需要仓库自带的少量种子音频;其他本地数据不能进入构建上下文。 +data/* +!data/audio/ +!data/audio/** *.db *.sqlite *.sqlite3 diff --git a/deploy/k8s/Dockerfile.audio b/deploy/k8s/Dockerfile.audio new file mode 100644 index 0000000..1f62a14 --- /dev/null +++ b/deploy/k8s/Dockerfile.audio @@ -0,0 +1,31 @@ +# 本地 K8s 音频链路的完整 CPU runtime。 +# 此镜像与项目 uv.lock 保持一致,包含真实 SenseVoice 所需依赖和 system codec; +# Kubernetes 资源声明不申请 GPU,CUDA 相关依赖即使存在也不会被使用。 +FROM python:3.12-slim + +COPY --from=ghcr.io/astral-sh/uv:0.9.17 /uv /uvx /bin/ + +WORKDIR /app +ENV UV_COMPILE_BYTECODE=1 \ + UV_PROJECT_ENVIRONMENT=/opt/venv \ + # PyPI 上的 Linux ARM64 Torch 依赖包含数百 MiB 的 CUDA wheel;本地网络较慢时, + # 解压阶段可能超过默认超时。保留有限下载并发,并把单个请求/解压的超时提高到 15 分钟。 + UV_HTTP_TIMEOUT=900 \ + UV_CONCURRENT_DOWNLOADS=2 \ + PATH=/opt/venv/bin:$PATH \ + CUDA_VISIBLE_DEVICES="" + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends \ + build-essential ffmpeg libsndfile1 wget curl git \ + && rm -rf /var/lib/apt/lists/* + +COPY pyproject.toml uv.lock README.md ./ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --frozen --no-install-project --no-dev +COPY . . +RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev +# 项目锁定 Ray 本体;最终 `uv sync` 会移除不在 lock 中的额外包,故必须在其之后 +# 安装 KubeRay health check 和 Ray Dashboard 需要的 default extra。 +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --python /opt/venv/bin/python "ray[default]==2.55.1" diff --git a/deploy/k8s/minio.yaml b/deploy/k8s/minio.yaml new file mode 100644 index 0000000..e9bc287 --- /dev/null +++ b/deploy/k8s/minio.yaml @@ -0,0 +1,103 @@ +# 本地专用 MinIO,不读取宿主机或现有对象存储的任何数据。 +apiVersion: v1 +kind: Secret +metadata: + name: minio-credentials + namespace: daft-ray +type: Opaque +stringData: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-data + namespace: daft-ray +spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 4Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: daft-ray +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: minio/minio:RELEASE.2025-07-23T15-54-02Z + args: [server, /data, --console-address, ":9001"] + envFrom: + - secretRef: + name: minio-credentials + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: minio-data +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: daft-ray +spec: + selector: + app: minio + ports: + - name: api + port: 9000 + targetPort: api + - name: console + port: 9001 + targetPort: console +--- +# 创建 benchmark bucket;Job 可安全重复执行,现有 bucket 不会被删除或清空。 +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-init + namespace: daft-ray +spec: + backoffLimit: 20 + template: + spec: + restartPolicy: OnFailure + containers: + - name: mc + image: minio/mc:RELEASE.2025-07-21T05-28-08Z + envFrom: + - secretRef: + name: minio-credentials + command: [/bin/sh, -ec] + args: + - >- + until mc alias set local http://minio:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"; + do sleep 2; done; + mc mb --ignore-existing local/benchmark diff --git a/deploy/k8s/mock-llm.yaml b/deploy/k8s/mock-llm.yaml new file mode 100644 index 0000000..8df7217 --- /dev/null +++ b/deploy/k8s/mock-llm.yaml @@ -0,0 +1,44 @@ +# 使用项目内 OpenAI-compatible Mock LLM,避免本地 E2E 依赖外部模型服务或密钥。 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mock-llm + namespace: daft-ray +spec: + replicas: 1 + selector: + matchLabels: + app: mock-llm + template: + metadata: + labels: + app: mock-llm + spec: + containers: + - name: mock-llm + image: multimodal-toolkit:audio-local + imagePullPolicy: IfNotPresent + command: [python, -m, benchmark.audio, serve-mock, --host, 0.0.0.0, --port, "8010", --profile, fast] + ports: + - containerPort: 8010 + name: http + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: mock-llm + namespace: daft-ray +spec: + selector: + app: mock-llm + ports: + - name: http + port: 8010 + targetPort: http diff --git a/deploy/k8s/namespace.yaml b/deploy/k8s/namespace.yaml new file mode 100644 index 0000000..4e69405 --- /dev/null +++ b/deploy/k8s/namespace.yaml @@ -0,0 +1,5 @@ +# 本地 Daft/Ray 演示环境的所有资源都隔离在此命名空间,便于一键清理。 +apiVersion: v1 +kind: Namespace +metadata: + name: daft-ray diff --git a/deploy/k8s/prometheus.yaml b/deploy/k8s/prometheus.yaml new file mode 100644 index 0000000..d2996a2 --- /dev/null +++ b/deploy/k8s/prometheus.yaml @@ -0,0 +1,113 @@ +# 轻量、单副本 Prometheus:只保存本地 Ray 集群的指标,避免本地开发机负担过重。 +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: daft-ray +data: + prometheus.yml: | + global: + scrape_interval: 5s + evaluation_interval: 5s + scrape_configs: + - job_name: ray + kubernetes_sd_configs: + # Pod discovery 配合 Ray 节点标签,避免 Endpoints 元数据在不同 + # Prometheus/Kubernetes 版本间的端口标签差异。 + - role: pod + namespaces: + names: [daft-ray] + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_ray_io_is_ray_node] + action: keep + # KubeRay 将 ray.io/is-ray-node 标签的值设为 "yes"。 + regex: "yes" + - source_labels: [__meta_kubernetes_pod_container_port_name] + action: keep + regex: metrics +--- + # Prometheus 通过 Kubernetes 服务发现读取 Service、Endpoints 和 Pod;默认账号没有 + # 这些读取权限,会导致 UI 中没有任何 active target。 +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: daft-ray +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: prometheus-discovery + namespace: daft-ray +rules: + - apiGroups: [""] + resources: ["services", "endpoints", "pods"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: prometheus-discovery + namespace: daft-ray +subjects: + - kind: ServiceAccount + name: prometheus + namespace: daft-ray +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: prometheus-discovery +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: daft-ray +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + serviceAccountName: prometheus + containers: + - name: prometheus + image: prom/prometheus:v3.5.0 + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=2d + ports: + - containerPort: 9090 + name: web + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + volumeMounts: + - name: config + mountPath: /etc/prometheus + volumes: + - name: config + configMap: + name: prometheus-config +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: daft-ray +spec: + selector: + app: prometheus + ports: + - name: web + port: 9090 + targetPort: web diff --git a/deploy/k8s/ray-metrics-service.yaml b/deploy/k8s/ray-metrics-service.yaml new file mode 100644 index 0000000..422ec15 --- /dev/null +++ b/deploy/k8s/ray-metrics-service.yaml @@ -0,0 +1,15 @@ +# Prometheus 集群内抓取所有 Ray Pod 的 OpenMetrics endpoint(head 与 worker)。 +apiVersion: v1 +kind: Service +metadata: + name: daft-ray-metrics + namespace: daft-ray + labels: + app.kubernetes.io/name: daft-ray +spec: + selector: + ray.io/cluster: daft-ray + ports: + - name: metrics + port: 8080 + targetPort: 8080 diff --git a/deploy/k8s/raycluster.yaml b/deploy/k8s/raycluster.yaml new file mode 100644 index 0000000..c77f8b1 --- /dev/null +++ b/deploy/k8s/raycluster.yaml @@ -0,0 +1,96 @@ +# KubeRay 管理的本地 Ray 集群。 +# 镜像由 scripts/k8s/build-image.sh 构建并载入 Minikube,包含完整音频 pipeline。 +apiVersion: ray.io/v1 +kind: RayCluster +metadata: + name: daft-ray + namespace: daft-ray +spec: + rayVersion: "2.55.1" + enableInTreeAutoscaling: true + autoscalerOptions: + upscalingMode: Default + headGroupSpec: + rayStartParams: + dashboard-host: "0.0.0.0" + metrics-export-port: "8080" + num-cpus: "0" + template: + spec: + containers: + - name: ray-head + image: multimodal-toolkit:audio-local + imagePullPolicy: IfNotPresent + env: + - name: MINIO_ENDPOINT + value: http://minio:9000 + - name: MINIO_REGION + value: us-east-1 + - name: NO_PROXY + value: minio,mock-llm,127.0.0.1,localhost + - name: no_proxy + value: minio,mock-llm,127.0.0.1,localhost + envFrom: + - secretRef: + name: minio-credentials + resources: + requests: + cpu: 500m + memory: 4Gi + limits: + cpu: "1" + memory: 4Gi + ports: + - containerPort: 6379 + name: gcs + - containerPort: 8265 + name: dashboard + - containerPort: 8080 + name: metrics + workerGroupSpecs: + - groupName: cpu-workers + replicas: 1 + minReplicas: 1 + maxReplicas: 2 + rayStartParams: + metrics-export-port: "8080" + template: + spec: + containers: + - name: ray-worker + image: multimodal-toolkit:audio-local + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: "2" + memory: 5Gi + limits: + cpu: "2" + memory: 5Gi + env: + - name: MINIO_ENDPOINT + value: http://minio:9000 + - name: MINIO_REGION + value: us-east-1 + - name: NO_PROXY + value: minio,mock-llm,127.0.0.1,localhost + - name: no_proxy + value: minio,mock-llm,127.0.0.1,localhost + # 同一个 worker 内的 ASR actor 复用模型;Pod 重建后会重新下载,避免依赖共享 RWX 卷。 + - name: MODELSCOPE_CACHE + value: /models + - name: CUDA_VISIBLE_DEVICES + value: "" + ports: + # 为 Prometheus 的 Pod 服务发现显式声明 Ray metrics endpoint。 + - containerPort: 8080 + name: metrics + envFrom: + - secretRef: + name: minio-credentials + volumeMounts: + - name: models + mountPath: /models + volumes: + - name: models + emptyDir: {} diff --git a/docs/k8s-ray-local.md b/docs/k8s-ray-local.md new file mode 100644 index 0000000..1fba7f0 --- /dev/null +++ b/docs/k8s-ray-local.md @@ -0,0 +1,82 @@ +# 本地 Kubernetes + Ray + Daft 完整音频环境 + +本环境在专用的 Minikube profile `daft-ray` 中运行真实音频链路,不读取宿主机已有的 +MinIO、模型服务或 S3 数据: + +```text +MinIO (cluster-local) → Daft download → SenseVoice ASR → Mock LLM → Lance Blob v2 + │ + KubeRay worker + │ :8080 + Prometheus +``` + +Ray Dashboard 由 KubeRay head 提供;Prometheus 仅抓取 Ray head 和 worker 的内置 +OpenMetrics 指标。本环境不安装 Grafana,也不暴露业务自定义指标。 + +## 前置条件 + +- Docker Desktop 或 OrbStack 已运行,并至少可分配 **4 CPU / 12 GiB 内存**;首次模型下载 + 和完整 Python 镜像还需要若干 GiB 磁盘空间。 +- 已安装 `minikube`、`kubectl`、`helm`,且能访问镜像仓库、KubeRay Helm 仓库和 PyPI。 +- 首次 audio smoke 会下载 SenseVoice/VAD 模型。模型缓存位于 worker Pod 的本地卷;Pod + 重建后会重新下载,属于开发环境的有意取舍。 + +## 启动与访问 + +```bash +scripts/k8s/up.sh +scripts/k8s/status.sh +``` + +`up.sh` 会创建专用 Minikube profile、构建 `multimodal-toolkit:audio-local`、安装固定 +版本 KubeRay operator,并部署 MinIO、Mock LLM、RayCluster 和 Prometheus。镜像完整安装 +当前 `uv.lock` 的项目依赖,但声明为 CPU-only,不请求 GPU。 + +在独立终端执行端口转发: + +```bash +kubectl --context daft-ray -n daft-ray port-forward service/daft-ray-head-svc 8265:8265 +kubectl --context daft-ray -n daft-ray port-forward service/prometheus 9090:9090 +``` + +- Ray Dashboard: +- Prometheus targets: + +所有 Ray target 应为 `UP`。可从以下 PromQL 起步;实际可用指标名以 target 的 `/metrics` +为准,因为会随 Ray 版本和组件变化: + +```promql +ray_tasks +ray_object_store_memory +ray_node_cpu_utilization +``` + +## 运行真实音频 smoke + +```bash +scripts/k8s/run-audio-smoke.sh +``` + +脚本会使用 UTC run-id,在集群内 MinIO 的 `benchmark/audio//` 下写入 4 条种子 +音频、Parquet manifest 与独立 Lance 表,随后通过 Ray Jobs API 提交: + +```text +Daft download → @daft.cls SenseVoice ASR → HTTP Mock LLM → Lance Blob v2 +``` + +结束时脚本会验证结果表行数、所有 `status=ok` 与 `audio_blob` 的 Blob v2 格式。执行期间可在 +Ray Dashboard 的 Jobs 页面查看作业和 actor;首次模型冷启动会明显比后续运行更久。 + +## 清理 + +```bash +scripts/k8s/down.sh +``` + +它只删除专用 profile 中的 `daft-ray` 命名空间及其 MinIO PVC,不会删除 Minikube 或 +KubeRay operator。若确认不再需要整个专用集群,再手动执行: + +```bash +minikube delete --profile daft-ray +``` diff --git a/examples/k8s/verify_audio_smoke.py b/examples/k8s/verify_audio_smoke.py new file mode 100644 index 0000000..112ebc3 --- /dev/null +++ b/examples/k8s/verify_audio_smoke.py @@ -0,0 +1,26 @@ +"""验证 Kubernetes 音频 smoke 的行保留、成功状态及 Lance Blob v2 写出。""" + +from __future__ import annotations + +import sys + +import daft + +from multimodal_toolkit.storage.blob import validate_blob_v2 +from multimodal_toolkit.storage.io import daft_io_config + + +def main(uri: str, expected_rows: int) -> None: + validate_blob_v2(uri, "audio_blob") + rows = daft.read_lance(uri, io_config=daft_io_config()).select("status").to_pydict()["status"] + if len(rows) != expected_rows: + raise RuntimeError(f"expected {expected_rows} rows, got {len(rows)}") + if set(rows) != {"ok"}: + raise RuntimeError(f"audio smoke contains non-success statuses: {rows}") + print(f"validated {len(rows)} rows with status=ok and Lance Blob v2") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + raise SystemExit("usage: verify_audio_smoke.py ") + main(sys.argv[1], int(sys.argv[2])) diff --git a/scripts/k8s/build-image.sh b/scripts/k8s/build-image.sh new file mode 100755 index 0000000..5071cd9 --- /dev/null +++ b/scripts/k8s/build-image.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# 构建与本地源码一致的 Ray runtime 镜像,并载入 Minikube 的节点镜像缓存。 +set -euo pipefail + +project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +image="multimodal-toolkit:audio-local" + +docker build --quiet --file "$project_root/deploy/k8s/Dockerfile.audio" --tag "$image" "$project_root" +minikube image load "$image" --profile daft-ray diff --git a/scripts/k8s/down.sh b/scripts/k8s/down.sh new file mode 100755 index 0000000..90f1562 --- /dev/null +++ b/scripts/k8s/down.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# 仅删除专用 daft-ray profile 中本项目命名空间的资源;不会删除 Minikube 或 operator。 +set -euo pipefail + +project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +kubectl --context daft-ray delete namespace daft-ray --ignore-not-found diff --git a/scripts/k8s/run-audio-smoke.sh b/scripts/k8s/run-audio-smoke.sh new file mode 100755 index 0000000..e651e20 --- /dev/null +++ b/scripts/k8s/run-audio-smoke.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# 生成隔离的测试音频,并以 Ray Jobs API 提交真实 Daft/SenseVoice/Mock-LMM/Lance 链路。 +set -euo pipefail + +kubectl_context="daft-ray" +namespace="daft-ray" +run_id="k8s-smoke-$(date -u +%Y%m%dT%H%M%SZ)" +bucket="benchmark" +prefix="audio" +head_pod="$(kubectl --context "$kubectl_context" -n "$namespace" get pod \ + -l ray.io/node-type=head -o jsonpath='{.items[0].metadata.name}')" + +if [[ -z "$head_pod" ]]; then + echo "Ray head is unavailable; run scripts/k8s/up.sh first." >&2 + exit 1 +fi + +common_env=( + "MINIO_ENDPOINT=http://minio:9000" + "MINIO_ROOT_USER=minioadmin" + "MINIO_ROOT_PASSWORD=minioadmin" + "MINIO_REGION=us-east-1" + "NO_PROXY=minio,mock-llm,127.0.0.1,localhost" + "no_proxy=minio,mock-llm,127.0.0.1,localhost" +) + +echo "[generate] run_id=$run_id" +kubectl --context "$kubectl_context" -n "$namespace" exec "$head_pod" -c ray-head -- \ + env "${common_env[@]}" python -m benchmark.audio generate \ + --run-id "$run_id" --bucket "$bucket" --prefix "$prefix" --profile smoke --count 4 \ + --source-dir /app/data/audio + +manifest="s3://$bucket/$prefix/$run_id/manifest.parquet" +output="s3://$bucket/$prefix/$run_id/output.lance" +echo "[submit] $manifest -> $output" +kubectl --context "$kubectl_context" -n "$namespace" exec "$head_pod" -c ray-head -- \ + env "${common_env[@]}" ray job submit --address http://127.0.0.1:8265 \ + --submission-id "$run_id" --working-dir /app -- \ + python -m benchmark.audio run --run-id "$run_id" --manifest "$manifest" \ + --lance-uri "$output" --mock-url http://mock-llm:8010 --ray-address auto \ + --asr-actor-cpus 2 --num-partitions 4 --max-minutes 20 + +kubectl --context "$kubectl_context" -n "$namespace" exec "$head_pod" -c ray-head -- \ + env "${common_env[@]}" python examples/k8s/verify_audio_smoke.py "$output" 4 +echo "Audio smoke completed: $output" diff --git a/scripts/k8s/status.sh b/scripts/k8s/status.sh new file mode 100755 index 0000000..38c9e9c --- /dev/null +++ b/scripts/k8s/status.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# 输出连接方式;port-forward 在前台运行,使用独立终端启动它们。 +set -euo pipefail + +kubectl --context daft-ray --namespace daft-ray get raycluster,pods,svc,jobs +cat <<'EOF' + +Ray Dashboard: + kubectl --context daft-ray -n daft-ray port-forward service/daft-ray-head-svc 8265:8265 + open http://127.0.0.1:8265 + +Prometheus: + kubectl --context daft-ray -n daft-ray port-forward service/prometheus 9090:9090 + open http://127.0.0.1:9090/targets + +MinIO Console: + kubectl --context daft-ray -n daft-ray port-forward service/minio 9001:9001 + open http://127.0.0.1:9001 + +Real audio smoke: + scripts/k8s/run-audio-smoke.sh +EOF diff --git a/scripts/k8s/up.sh b/scripts/k8s/up.sh new file mode 100755 index 0000000..a611c46 --- /dev/null +++ b/scripts/k8s/up.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# 启动专用 Minikube、完整音频 Ray 集群、MinIO、Mock LLM 与 Prometheus。 +set -euo pipefail + +project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +kubectl_context="daft-ray" + +minikube start --profile "$kubectl_context" --driver=docker --cpus=4 --memory=12288 +"$project_root/scripts/k8s/build-image.sh" + +helm repo add kuberay https://ray-project.github.io/kuberay-helm/ --force-update +helm repo update kuberay +helm upgrade --install kuberay-operator kuberay/kuberay-operator \ + --kube-context "$kubectl_context" \ + --namespace kuberay-system --create-namespace --version 1.4.1 + +kubectl --context "$kubectl_context" apply -f "$project_root/deploy/k8s/namespace.yaml" +kubectl --context "$kubectl_context" apply -f "$project_root/deploy/k8s/minio.yaml" +kubectl --context "$kubectl_context" apply -f "$project_root/deploy/k8s/prometheus.yaml" +kubectl --context "$kubectl_context" apply -f "$project_root/deploy/k8s/mock-llm.yaml" + +kubectl --context "$kubectl_context" wait --namespace kuberay-system \ + --for=condition=available deployment/kuberay-operator --timeout=5m +kubectl --context "$kubectl_context" rollout status --namespace daft-ray deployment/minio --timeout=5m +kubectl --context "$kubectl_context" wait --namespace daft-ray \ + --for=condition=complete job/minio-init --timeout=5m + +kubectl --context "$kubectl_context" apply -f "$project_root/deploy/k8s/raycluster.yaml" +kubectl --context "$kubectl_context" apply -f "$project_root/deploy/k8s/ray-metrics-service.yaml" + +kubectl wait --context "$kubectl_context" --namespace daft-ray \ + --for=condition=ready pod -l ray.io/node-type=head --timeout=5m +kubectl wait --context "$kubectl_context" --namespace daft-ray \ + --for=condition=ready pod -l ray.io/node-type=worker --timeout=5m +kubectl rollout status --context "$kubectl_context" --namespace daft-ray deployment/prometheus --timeout=5m +kubectl rollout status --context "$kubectl_context" --namespace daft-ray deployment/mock-llm --timeout=5m + +echo "Environment ready. Run scripts/k8s/run-audio-smoke.sh for the real audio pipeline."