diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f72a70f5..f125b54a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,18 @@ jobs: node-version: "22" cache: pnpm + - name: Cache static tool outputs + uses: actions/cache@v4 + with: + path: | + .cache/eslint + .cache/tsc + .cache/vite + .cache/vitest-unit + key: static-tools-${{ runner.os }}-node-22-${{ hashFiles('pnpm-lock.yaml', 'package.json', 'tsconfig.json', 'eslint.config.mjs', 'vite.config.ts', 'vitest.unit.config.ts') }} + restore-keys: | + static-tools-${{ runner.os }}-node-22- + - name: Install Node dependencies run: pnpm install --frozen-lockfile @@ -48,22 +60,29 @@ jobs: KERIPY_REF: cbbf700fa8091587b96b5475c5f50d1d8bf3ca40 KERIA_BRANCH: main KERIA_REF: aba457cab3813078bfedb65a7d819f48d86974b8 + VLEI_BRANCH: main + VLEI_REF: f514b9431c5f965b5f7f64a8693e19df2f181564 PUPPETEER_CACHE_DIR: /home/runner/.cache/puppeteer + PYTHON_VERSION: "3.12.8" VITE_OPERATION_TIMEOUT_MS: "45000" steps: - name: Checkout uses: actions/checkout@v4 + - name: Cache apt packages + uses: actions/cache@v4 + with: + path: .ci/apt + key: apt-${{ runner.os }}-${{ runner.arch }}-libsodium-dev-v1 + - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libsodium-dev + run: scripts/ci/install-system-deps.sh - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.12.8" + python-version: ${{ env.PYTHON_VERSION }} cache: pip cache-dependency-path: .github/ci/keria-runtime-requirements.txt @@ -83,7 +102,23 @@ jobs: uses: actions/cache@v4 with: path: .ci/deps - key: keri-repos-${{ runner.os }}-${{ env.KERIPY_REF }}-${{ env.KERIA_REF }} + key: keri-repos-${{ runner.os }}-${{ env.KERIPY_REF }}-${{ env.KERIA_REF }}-${{ env.VLEI_REF }} + + - name: Cache pinned KERI wheels + uses: actions/cache@v4 + with: + path: .ci/wheels + key: keri-wheels-${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-${{ hashFiles('.github/ci/keria-runtime-requirements.txt', 'scripts/ci/install-keri-stack.sh') }}-${{ env.KERIPY_REF }}-${{ env.KERIA_REF }}-${{ env.VLEI_REF }} + + - name: Cache live tool outputs + uses: actions/cache@v4 + with: + path: | + .cache/vite + .cache/vitest + key: live-tools-${{ runner.os }}-node-22-${{ hashFiles('pnpm-lock.yaml', 'package.json', 'vite.config.ts', 'vitest.config.ts') }} + restore-keys: | + live-tools-${{ runner.os }}-node-22- - name: Cache Puppeteer browser uses: actions/cache@v4 @@ -99,7 +134,7 @@ jobs: - name: Install Puppeteer browser run: pnpm exec puppeteer browsers install chrome - - name: Install pinned KERIpy and KERIA + - name: Install pinned KERIpy, KERIA, and vLEI run: scripts/ci/install-keri-stack.sh - name: Start KERIA and demo witnesses diff --git a/docs/ci.md b/docs/ci.md index 7b1417dc..71b6ab2d 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -28,7 +28,7 @@ The live KERIA job: 3. sets up Node.js and pnpm, 4. installs Node dependencies from `pnpm-lock.yaml`, 5. installs a Puppeteer browser, -6. installs pinned KERIpy and KERIA from GitHub commits, +6. installs pinned KERIpy, KERIA, and vLEI from GitHub commits, 7. starts local KERI demo witnesses, 8. starts local KERIA, 9. runs `pnpm ci:live`, @@ -78,10 +78,11 @@ immutable SHAs, not floating branches. CI installs these exact repositories: -| Project | Branch | Commit | Version | -| --- | --- | --- | --- | -| KERIpy | `v1.2.13` | `cbbf700fa8091587b96b5475c5f50d1d8bf3ca40` | `keri==1.2.13` | -| KERIA | `main` | `aba457cab3813078bfedb65a7d819f48d86974b8` | `keria==0.4.0` | +| Project | Branch | Commit | Version | +|---------|-----------|--------------------------------------------|----------------| +| KERIpy | `v1.2.13` | `cbbf700fa8091587b96b5475c5f50d1d8bf3ca40` | `keri==1.2.13` | +| KERIA | `main` | `aba457cab3813078bfedb65a7d819f48d86974b8` | `keria==0.4.0` | +| vLEI | `main` | `f514b9431c5f965b5f7f64a8693e19df2f181564` | `vlei==1.0.2` | The install script is `scripts/ci/install-keri-stack.sh`. @@ -90,10 +91,12 @@ requested KERIpy `1.2.13` commit. To make that explicit, the script: 1. installs KERIA runtime dependencies from `.github/ci/keria-runtime-requirements.txt`, -2. installs KERIpy from the pinned commit, -3. installs KERIA from the pinned commit with `--no-deps`, -4. verifies `keri.__version__ == "1.2.13"` and - `keria.__version__ == "0.4.0"`. +2. restores or builds local wheels for the pinned KERIpy, KERIA, and vLEI + commits, +3. installs KERIpy from the pinned wheel, +4. installs KERIA and vLEI from pinned wheels with `--no-deps`, +5. verifies `keri.__version__ == "1.2.13"`, + `keria.__version__ == "0.4.0"`, and `vlei.__version__ == "1.0.2"`. Do not replace this with an unconstrained `pip install keria`; that would allow the resolver to choose a different KERIpy version. @@ -133,14 +136,14 @@ WITNESS_LOGLEVEL=DEBUG KERIA_LOGLEVEL=DEBUG scripts/ci/start-keri-stack.sh Expected ports: -| Service | Port | -| --- | --- | +| Service | Port | +|------------------|--------| | Wan witness HTTP | `5642` | | Wil witness HTTP | `5643` | | Wes witness HTTP | `5644` | -| KERIA admin API | `3901` | +| KERIA admin API | `3901` | | KERIA router API | `3902` | -| KERIA boot API | `3903` | +| KERIA boot API | `3903` | The start script waits for all ports before returning. Logs are written under `${RUNNER_TEMP}/keri-stack/logs` in GitHub Actions and uploaded as the @@ -152,17 +155,24 @@ CI uses these caches: - pnpm dependency cache via `actions/setup-node`. - pip cache via `actions/setup-python`. +- apt package cache for `libsodium-dev` under `.ci/apt`. +- static tool caches under `.cache/eslint`, `.cache/tsc`, `.cache/vite`, and + `.cache/vitest-unit`. +- live tool caches under `.cache/vite` and `.cache/vitest`. - pinned KERI repository clones under `.ci/deps`. +- pinned KERI wheels under `.ci/wheels`. - Puppeteer browser cache under `~/.cache/puppeteer`. -The KERI repository cache key includes the pinned KERIpy and KERIA commits, so -changing either commit creates a fresh cache. +The KERI repository and wheel cache keys include the pinned KERIpy, KERIA, and +vLEI commits, so changing any pinned commit creates a fresh cache. The wheel key +also includes Python version, the KERIA runtime requirements file, and +`scripts/ci/install-keri-stack.sh`. ## Updating Pinned KERI Versions -To update KERIpy or KERIA: +To update KERIpy, KERIA, or vLEI: -1. update `KERIPY_BRANCH` / `KERIPY_REF` or `KERIA_BRANCH` / `KERIA_REF` in +1. update the corresponding branch/ref environment variables in `.github/workflows/ci.yml`, 2. update the version assertions in `scripts/ci/install-keri-stack.sh`, 3. update this document, diff --git a/package.json b/package.json index defb673e..8764e06d 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "scripts": { "dev": "vite", "build": "npx tsc && vite build", - "lint": "eslint src tests scripts eslint.config.mjs vite.config.ts vitest.config.ts vitest.unit.config.ts --max-warnings 0", + "lint": "eslint --cache --cache-location .cache/eslint/eslintcache --cache-strategy content src tests scripts eslint.config.mjs vite.config.ts vitest.config.ts vitest.unit.config.ts --max-warnings 0", "preview": "vite preview", "keria:smoke": "tsx scripts/keria-smoke.ts", "contact:ui-smoke": "tsx tests/contact-oobi-smoke.ts", diff --git a/scripts/ci/install-keri-stack.sh b/scripts/ci/install-keri-stack.sh index 2b50acd7..096a1601 100755 --- a/scripts/ci/install-keri-stack.sh +++ b/scripts/ci/install-keri-stack.sh @@ -18,6 +18,7 @@ KERIA_REF="${KERIA_REF:-aba457cab3813078bfedb65a7d819f48d86974b8}" VLEI_REPO="${VLEI_REPO:-https://github.com/WebOfTrust/vLEI.git}" VLEI_BRANCH="${VLEI_BRANCH:-main}" VLEI_REF="${VLEI_REF:-f514b9431c5f965b5f7f64a8693e19df2f181564}" +WHEEL_DIR="${WHEEL_DIR:-${ROOT_DIR}/.ci/wheels/${KERIPY_REF}-${KERIA_REF}-${VLEI_REF}}" sync_repo() { local dir="$1" @@ -30,6 +31,17 @@ sync_repo() { if [[ ! -d "${dir}/.git" ]]; then rm -rf "$dir" git clone --no-checkout "$repo" "$dir" + else + local actual + local remote + actual="$(git -C "$dir" rev-parse HEAD 2>/dev/null || true)" + remote="$(git -C "$dir" remote get-url origin 2>/dev/null || true)" + if [[ "$actual" == "$ref" && "$remote" == "$repo" ]]; then + git -C "$dir" reset --hard "$ref" >/dev/null + git -C "$dir" clean -xdf >/dev/null + echo "Using cached ${dir} at ${ref}" + return + fi fi git -C "$dir" remote set-url origin "$repo" @@ -50,16 +62,50 @@ sync_repo() { fi } +build_wheel() { + local dir="$1" + local distribution="$2" + local wheel + + if wheel="$(find_wheel "$distribution" 2>/dev/null)"; then + echo "Using cached wheel ${wheel}" + return + fi + + python -m pip wheel --no-deps --wheel-dir "$WHEEL_DIR" "$dir" +} + +find_wheel() { + local distribution="$1" + + python - "$WHEEL_DIR" "$distribution" <<'PY' +from pathlib import Path +import sys + +wheel_dir = Path(sys.argv[1]) +distribution = sys.argv[2].replace("-", "_").lower() +wheels = sorted(wheel_dir.glob(f"{distribution}-*.whl")) +if not wheels: + raise SystemExit(f"No wheel found for {distribution} in {wheel_dir}") +print(wheels[-1]) +PY +} + python -m pip install --upgrade pip wheel setuptools sync_repo "$KERIPY_DIR" "$KERIPY_REPO" "$KERIPY_BRANCH" "$KERIPY_REF" sync_repo "$KERIA_DIR" "$KERIA_REPO" "$KERIA_BRANCH" "$KERIA_REF" sync_repo "$VLEI_DIR" "$VLEI_REPO" "$VLEI_BRANCH" "$VLEI_REF" +mkdir -p "$WHEEL_DIR" +build_wheel "$KERIPY_DIR" keri +build_wheel "$KERIA_DIR" keria +build_wheel "$VLEI_DIR" vlei + python -m pip install -r "${ROOT_DIR}/.github/ci/keria-runtime-requirements.txt" -python -m pip install "$KERIPY_DIR" -python -m pip install --no-deps "$KERIA_DIR" -python -m pip install --no-deps "$VLEI_DIR" +python -m pip install "$(find_wheel keri)" +python -m pip install --no-deps "$(find_wheel keria)" +python -m pip install --no-deps "$(find_wheel vlei)" python - <<'PY' import keri diff --git a/scripts/ci/install-system-deps.sh b/scripts/ci/install-system-deps.sh new file mode 100755 index 00000000..dce27b4c --- /dev/null +++ b/scripts/ci/install-system-deps.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +APT_CACHE_DIR="${APT_CACHE_DIR:-${ROOT_DIR}/.ci/apt}" +APT_ARCHIVES_DIR="${APT_CACHE_DIR}/archives" + +mkdir -p "${APT_ARCHIVES_DIR}/partial" + +install_cached_debs() { + shopt -s nullglob + local debs=("${APT_ARCHIVES_DIR}"/*.deb) + shopt -u nullglob + + if (( ${#debs[@]} == 0 )); then + return 1 + fi + + sudo apt-get install -y "${debs[@]}" +} + +if install_cached_debs; then + exit 0 +fi + +sudo apt-get update +sudo apt-get -o Dir::Cache="${APT_CACHE_DIR}" install -y libsodium-dev diff --git a/tsconfig.json b/tsconfig.json index 7d844120..f64baa0a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,6 +12,8 @@ "isolatedModules": true, "strict": true, "noEmit": true, + "incremental": true, + "tsBuildInfoFile": ".cache/tsc/tsconfig.tsbuildinfo", "jsx": "react-jsx" }, "include": ["src"] diff --git a/vite.config.ts b/vite.config.ts index 2e85f147..a0f10d60 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -5,6 +5,7 @@ import wasm from 'vite-plugin-wasm'; // https://vitejs.dev/config/ export default defineConfig({ plugins: [react(), wasm()], + cacheDir: '.cache/vite', server: { host: "127.0.0.1", port: 5173, diff --git a/vitest.config.ts b/vitest.config.ts index 6ac25375..195e6329 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,6 +3,7 @@ import wasm from 'vite-plugin-wasm'; export default defineConfig({ plugins: [wasm()], + cacheDir: '.cache/vitest', test: { environment: 'node', fileParallelism: false, diff --git a/vitest.unit.config.ts b/vitest.unit.config.ts index a6c3e92a..e27b8a4f 100644 --- a/vitest.unit.config.ts +++ b/vitest.unit.config.ts @@ -3,10 +3,11 @@ import wasm from 'vite-plugin-wasm'; export default defineConfig({ plugins: [wasm()], + cacheDir: '.cache/vitest-unit', test: { environment: 'node', fileParallelism: true, - hookTimeout: 180_000, - testTimeout: 180_000, + hookTimeout: 10_000, + testTimeout: 15_000, }, });