run-eval.sh

#!/bin/bash
# QKV-Bench 一键评测入口。
#
#   bash run-eval.sh                          # 四家 × 六组，全部串行
#   bash run-eval.sh fable5 qwen3.8max        # 只评指定实现
#   bash run-eval.sh --rounds 3               # 每组连跑 3 轮（取中位数更稳）
#   QKV_IMAGE=my-img bash run-eval.sh sol     # 复用已有镜像，跳过 build
#
# 流程：1) 确保评测镜像存在（没有就用 Dockerfile 构建）
#       2) 逐个 job 拉起一对全新容器，容器内断网编译 + 压测
#       3) 汇总出六组对比表
set -uo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
IMAGE="${QKV_IMAGE:-qkv-bench:1}"
PY="$(command -v python3 || command -v python)"
export MSYS_NO_PATHCONV=1

winpath() { command -v cygpath >/dev/null 2>&1 && cygpath -m "$1" || printf '%s' "$1"; }

# ---- 解析参数 ------------------------------------------------------------
ROUNDS=1
AGENTS=()
while [ $# -gt 0 ]; do
  case "$1" in
    --rounds) ROUNDS="$2"; shift 2 ;;
    -h|--help) sed -n '2,10p' "$0"; exit 0 ;;
    *) AGENTS+=("$1"); shift ;;
  esac
done
[ ${#AGENTS[@]} -eq 0 ] && AGENTS=(fable5 qwen3.8max sol opus4.8)
GROUPS=(G1 G2 G3 G4 G5 G6)

# ---- 前置检查 ------------------------------------------------------------
command -v docker >/dev/null 2>&1 || { echo "!! 未找到 docker，请先安装并启动 Docker"; exit 1; }
docker info >/dev/null 2>&1 || { echo "!! docker 守护进程未运行"; exit 1; }
[ -n "$PY" ] || { echo "!! 未找到 python3/python（出表需要）"; exit 1; }

for a in "${AGENTS[@]}"; do
  [ -d "$ROOT/arena/agents/$a" ] || { echo "!! 未知实现: $a（arena/agents/ 下没有该目录）"; exit 1; }
done

# ---- 1. 确保镜像存在 -----------------------------------------------------
if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then
  echo "==> 镜像 $IMAGE 不存在，用 Dockerfile 构建（首次约几分钟）..."
  docker build -t "$IMAGE" "$(winpath "$ROOT")" || {
    echo "!! 镜像构建失败。若你的环境无法访问 Docker Hub，请自备一个含 Rust 1.83 + python3"
    echo "   的镜像，再用  QKV_IMAGE=<你的镜像> bash run-eval.sh  运行。"; exit 1; }
fi

# ---- 2. 串行跑所有 job ---------------------------------------------------
t0=$(date +%s); fail=0
for r in $(seq 1 "$ROUNDS"); do
  [ "$ROUNDS" -gt 1 ] && echo "########## 第 $r/$ROUNDS 轮 ##########"
  dest="$ROOT/results"; [ "$ROUNDS" -gt 1 ] && dest="$ROOT/results-r$r"
  for g in "${GROUPS[@]}"; do
    for a in "${AGENTS[@]}"; do
      s=$(date +%s)
      echo "===== $a / $g （第 $r 轮）====="
      if QKV_IMAGE="$IMAGE" QKV_OUT="$dest" bash "$ROOT/job.sh" "$a" "$g" 2>&1 \
           | grep -E "peak |达成|target|-> results|!!"; then :; else
        echo "!! $a/$g 失败"; fail=$((fail+1))
      fi
      echo "   用时 $(( $(date +%s) - s ))s"
    done
  done
done

# ---- 3. 出对比表 ---------------------------------------------------------
echo; echo "===== 汇总 ====="
if [ "$ROUNDS" -gt 1 ]; then
  "$PY" "$ROOT/harness/score/median.py" "$ROOT"/results-r* || true
else
  "$PY" "$ROOT/harness/score/refresh.py" --results "$ROOT/results" || true
fi

echo; echo "耗时 $(( ($(date +%s) - t0) / 60 )) 分钟，失败 $fail 个 job"
echo "原始结果: $ROOT/results*/<agent>/G*.json"