diff --git a/.github/actions/setup-apt-mirrors/action.yml b/.github/actions/setup-apt-mirrors/action.yml index 5beff201d188e..ec1df94c0c04c 100644 --- a/.github/actions/setup-apt-mirrors/action.yml +++ b/.github/actions/setup-apt-mirrors/action.yml @@ -5,7 +5,7 @@ runs: steps: - shell: bash run: | - # Replace the existing Blacksmith mirror list as well as direct sources. + # Replace GitHub and Blacksmith mirror lists as well as direct sources. printf '%s\tpriority:%s\n' \ https://archive.ubuntu.com/ubuntu 1 \ https://mirrors.edge.kernel.org/ubuntu 2 \ @@ -15,7 +15,7 @@ runs: # APT's mirror transport retries each file against the next server. sudo find /etc/apt -maxdepth 2 -type f \( -name '*.list' -o -name '*.sources' \) \ -exec sed -i -E \ - 's#https?://(([^/]+\.)?archive|security)\.ubuntu\.com/ubuntu/?#mirror+file:/etc/apt/blacksmith-ubuntu-mirrors.txt#g' {} + + 's#https?://(([^/]+\.)?archive|security)\.ubuntu\.com/ubuntu/?|mirror\+file:/etc/apt/apt-mirrors\.txt#mirror+file:/etc/apt/blacksmith-ubuntu-mirrors.txt#g' {} + # Move on to a fallback before an unreachable server exhausts the job. printf '%s\n' 'Acquire::http::Timeout "15";' 'Acquire::https::Timeout "15";' \ diff --git a/.github/scripts/fork-dependencies.test.cjs b/.github/scripts/fork-dependencies.test.cjs new file mode 100644 index 0000000000000..a73c3bf74514f --- /dev/null +++ b/.github/scripts/fork-dependencies.test.cjs @@ -0,0 +1,132 @@ +const assert = require("node:assert/strict"); +const { spawnSync } = require("node:child_process"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const test = require("node:test"); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "t3-fork-dependencies-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return root; +} + +function run(script, env) { + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-e", "-o", "pipefail", "-c", script], + { + encoding: "utf8", + env: { ...process.env, ...env }, + }, + ); + assert.ifError(result.error); + return result; +} + +test("APT failover covers GitHub mirror lists, direct Ubuntu sources, and existing Blacksmith sources", (t) => { + const root = fixture(t); + fs.mkdirSync(path.join(root, "sources.list.d")); + fs.mkdirSync(path.join(root, "apt.conf.d")); + const sources = { + "sources.list": [ + "deb http://azure.archive.ubuntu.com/ubuntu/ noble main", + "deb https://archive.ubuntu.com/ubuntu noble-updates main", + "deb http://security.ubuntu.com/ubuntu/ noble-security main", + "deb https://packages.example.invalid/ubuntu noble main", + ].join("\n"), + "sources.list.d/ubuntu.sources": + "Types: deb\nURIs: mirror+file:/etc/apt/apt-mirrors.txt\nSuites: noble noble-updates\nComponents: main universe\n", + "sources.list.d/blacksmith.sources": + "Types: deb\nURIs: mirror+file:/etc/apt/blacksmith-ubuntu-mirrors.txt\nSuites: noble\nComponents: main\n", + }; + for (const [file, contents] of Object.entries(sources)) { + fs.writeFileSync(path.join(root, file), contents); + } + const action = fs.readFileSync( + path.join(__dirname, "../actions/setup-apt-mirrors/action.yml"), + "utf8", + ); + const step = action.match(/ run: \|\n([\s\S]+)/); + assert.ok(step); + const script = + `sudo() { + local arg + local args=() + for arg in "$@"; do + if [[ "$arg" == /etc/apt* ]]; then + args+=("$FIXTURE_APT\${arg#/etc/apt}") + else + args+=("$arg") + fi + done + "\${args[@]}" + }\n` + step[1].replace(/^ /gm, ""); + const env = { FIXTURE_APT: root.replaceAll("\\", "/") }; + const result = run(script, env); + assert.equal(result.status, 0, result.stderr); + const mirror = "mirror+file:/etc/apt/blacksmith-ubuntu-mirrors.txt"; + assert.equal( + fs.readFileSync(path.join(root, "sources.list"), "utf8"), + [ + `deb ${mirror} noble main`, + `deb ${mirror} noble-updates main`, + `deb ${mirror} noble-security main`, + "deb https://packages.example.invalid/ubuntu noble main", + ].join("\n"), + ); + assert.equal( + fs.readFileSync(path.join(root, "sources.list.d/ubuntu.sources"), "utf8"), + sources["sources.list.d/ubuntu.sources"].replace( + "mirror+file:/etc/apt/apt-mirrors.txt", + mirror, + ), + ); + assert.equal( + fs.readFileSync(path.join(root, "sources.list.d/blacksmith.sources"), "utf8"), + sources["sources.list.d/blacksmith.sources"], + ); + assert.equal(run(script, env).status, 0); +}); + +test("fork browser dependency checks avoid unnecessary APT installs without masking failures", (t) => { + const scripts = ["fork-nightly.yml", "fork-pr.yml"].map((file) => { + const workflow = fs.readFileSync(path.join(__dirname, "../workflows", file), "utf8"); + const step = workflow.match( + /- name: Install missing Chromium system dependencies\n run: \|\n((?: .*\n|\n)+)/, + ); + assert.ok(step, `${file} must check cached Chromium dependencies`); + return step[1].replace(/^ /gm, ""); + }); + assert.equal(scripts[0], scripts[1]); + for (const { browser = true, libraries = "", lddExit = 0, installExit = 0, status, installs } of [ + { status: 0, installs: false }, + { libraries: "libnss3.so => not found", status: 0, installs: true }, + { browser: false, status: 1, installs: false }, + { lddExit: 2, status: 2, installs: false }, + { libraries: "libnss3.so => not found", installExit: 100, status: 100, installs: true }, + ]) { + const root = fixture(t); + const cache = path.join(root, ".cache", "ms-playwright", "chromium_headless_shell-1234"); + fs.mkdirSync(cache, { recursive: true }); + if (browser) fs.writeFileSync(path.join(cache, "chrome-headless-shell"), ""); + const calls = path.join(root, "calls"); + const result = run( + `ldd() { printf '%s\\n' "$FIXTURE_LIBRARIES"; return "$FIXTURE_LDD_EXIT"; } + node() { printf '%s\\n' "$*" >> "$FIXTURE_CALLS"; return "$FIXTURE_INSTALL_EXIT"; }\n` + + scripts[0], + { + HOME: root.replaceAll("\\", "/"), + FIXTURE_CALLS: calls.replaceAll("\\", "/"), + FIXTURE_LIBRARIES: libraries, + FIXTURE_LDD_EXIT: String(lddExit), + FIXTURE_INSTALL_EXIT: String(installExit), + }, + ); + assert.equal(result.status, status, result.stderr); + assert.equal( + fs.existsSync(calls) ? fs.readFileSync(calls, "utf8") : "", + installs ? "apps/web/node_modules/playwright/cli.js install-deps chromium\n" : "", + ); + } +}); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f75c92562d03..b8717f97770f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: # open: if the diff cannot be resolved, everything runs. changes: name: Changes - runs-on: blacksmith-2vcpu-ubuntu-2404 + runs-on: ${{ github.repository == 'amv146/t3code' && 'ubuntu-24.04' || 'blacksmith-2vcpu-ubuntu-2404' }} timeout-minutes: 5 outputs: typecheck: ${{ steps.detect.outputs.typecheck }} @@ -64,7 +64,7 @@ jobs: contents: read id-token: write name: Lint - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ${{ github.repository == 'amv146/t3code' && 'ubuntu-24.04' || 'blacksmith-8vcpu-ubuntu-2404' }} timeout-minutes: 10 steps: - name: Checkout @@ -120,7 +120,7 @@ jobs: name: Build and Typecheck needs: changes if: ${{ !cancelled() && (needs.changes.outputs.build != 'false' || needs.changes.outputs.typecheck != 'false') }} - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ${{ github.repository == 'amv146/t3code' && 'ubuntu-24.04' || 'blacksmith-8vcpu-ubuntu-2404' }} timeout-minutes: 10 steps: - name: Checkout @@ -275,7 +275,7 @@ jobs: name: Test needs: changes if: ${{ !cancelled() && needs.changes.outputs.test != 'false' }} - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ${{ github.repository == 'amv146/t3code' && 'ubuntu-24.04' || 'blacksmith-8vcpu-ubuntu-2404' }} timeout-minutes: 10 steps: - name: Checkout @@ -365,7 +365,7 @@ jobs: name: Test Web ${{ matrix.shard }} needs: changes if: ${{ !cancelled() && needs.changes.outputs.test_web != 'false' }} - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ${{ github.repository == 'amv146/t3code' && 'ubuntu-24.04' || 'blacksmith-8vcpu-ubuntu-2404' }} timeout-minutes: 10 strategy: fail-fast: false @@ -403,6 +403,9 @@ jobs: if: steps.chromium_cache.outputs.cache-hit != 'true' run: node apps/web/node_modules/playwright/cli.js install --only-shell chromium + - uses: ./.github/actions/setup-apt-mirrors + if: github.repository == 'amv146/t3code' + - name: Install missing Chromium system dependencies run: | browser=$(find "$HOME/.cache/ms-playwright" -name chrome-headless-shell -type f -print -quit) @@ -427,7 +430,7 @@ jobs: contents: read id-token: write name: Test Server ${{ matrix.shard }} - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ github.repository == 'amv146/t3code' && 'ubuntu-24.04' || 'blacksmith-4vcpu-ubuntu-2404' }} # Shards take 2-5 minutes. When the shared task cache throttles reads, each # failed fetch stalls vp for 15-90s, which pushed shards past 10 minutes. timeout-minutes: 20 @@ -478,6 +481,9 @@ jobs: if: steps.changes.outputs.test_server != 'false' && steps.chromium_cache.outputs.cache-hit != 'true' run: node apps/server/node_modules/playwright-core/cli.js install --only-shell chromium + - uses: ./.github/actions/setup-apt-mirrors + if: github.repository == 'amv146/t3code' && steps.changes.outputs.test_server != 'false' + - name: Install missing Chromium system dependencies if: steps.changes.outputs.test_server != 'false' run: | @@ -553,7 +559,7 @@ jobs: name: Rust needs: changes if: ${{ !cancelled() && needs.changes.outputs.rust != 'false' }} - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ${{ github.repository == 'amv146/t3code' && 'ubuntu-24.04' || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 10 steps: - name: Checkout @@ -586,7 +592,7 @@ jobs: name: Mobile Native Static Analysis needs: changes if: ${{ !cancelled() && needs.changes.outputs.mobile_native != 'false' }} - runs-on: blacksmith-6vcpu-macos-26 + runs-on: ${{ github.repository == 'amv146/t3code' && 'macos-26' || 'blacksmith-6vcpu-macos-26' }} timeout-minutes: 10 steps: - name: Checkout @@ -619,7 +625,7 @@ jobs: name: Release Smoke needs: changes if: ${{ !cancelled() && needs.changes.outputs.release_smoke != 'false' }} - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ${{ github.repository == 'amv146/t3code' && 'ubuntu-24.04' || 'blacksmith-8vcpu-ubuntu-2404' }} timeout-minutes: 15 steps: - name: Checkout @@ -659,7 +665,7 @@ jobs: mobile_native_static_analysis, release_smoke, ] - runs-on: blacksmith-2vcpu-ubuntu-2404 + runs-on: ${{ github.repository == 'amv146/t3code' && 'ubuntu-24.04' || 'blacksmith-2vcpu-ubuntu-2404' }} timeout-minutes: 5 steps: - name: Require every job to pass diff --git a/.github/workflows/fork-nightly.yml b/.github/workflows/fork-nightly.yml index 065f97342d887..043abd6af77cd 100644 --- a/.github/workflows/fork-nightly.yml +++ b/.github/workflows/fork-nightly.yml @@ -42,7 +42,7 @@ jobs: fetch-depth: 0 token: ${{ secrets.FORK_SYNC_TOKEN }} - name: Check sync logic - run: node --test .github/scripts/fork-nightly.test.cjs + run: node --test .github/scripts/fork-nightly.test.cjs .github/scripts/fork-dependencies.test.cjs - name: Merge the latest published upstream nightly id: prepare uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 @@ -71,13 +71,32 @@ jobs: !/.repos/ sparse-checkout-cone-mode: false - uses: ./.github/actions/setup-ci-deps + - uses: ./.github/actions/setup-apt-mirrors + - name: Resolve Playwright version + id: playwright_version + run: echo "version=$(node -p "require('./apps/web/node_modules/playwright/package.json').version")" >> "$GITHUB_OUTPUT" + - name: Cache Chromium + id: chromium_cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/ms-playwright + key: playwright-chromium-shell-${{ runner.os }}-${{ runner.arch }}-${{ steps.playwright_version.outputs.version }} - name: Install Chromium for changed browser tests - run: node apps/web/node_modules/playwright/cli.js install --with-deps --only-shell chromium + if: steps.chromium_cache.outputs.cache-hit != 'true' + run: node apps/web/node_modules/playwright/cli.js install --only-shell chromium + - name: Install missing Chromium system dependencies + run: | + browser=$(find "$HOME/.cache/ms-playwright" -name chrome-headless-shell -type f -print -quit) + test -n "$browser" + libraries=$(ldd "$browser") + if grep '=> not found' <<< "$libraries"; then + node apps/web/node_modules/playwright/cli.js install-deps chromium + fi - name: Check fork changes against the upstream nightly env: UPSTREAM_SHA: ${{ needs.sync.outputs.upstream_sha }} run: | - node --test .github/scripts/fork-nightly.test.cjs + node --test .github/scripts/fork-nightly.test.cjs .github/scripts/fork-dependencies.test.cjs vp run check:changed "$UPSTREAM_SHA" - name: Load public T3 Connect configuration id: config @@ -148,6 +167,7 @@ jobs: cargo build --locked --release --manifest-path native/resource-monitor/Cargo.toml --target x86_64-unknown-linux-gnu mkdir -p "$RUNNER_TEMP/cli-resource-monitor/linux-x64" cp native/resource-monitor/target/x86_64-unknown-linux-gnu/release/t3-resource-monitor "$RUNNER_TEMP/cli-resource-monitor/linux-x64/" + - uses: ./.github/actions/setup-apt-mirrors - name: Install archive build libraries run: sudo apt-get update && sudo apt-get install -y patchelf libsecret-1-dev pkg-config - name: Build matching WSL runtime diff --git a/.github/workflows/fork-pr.yml b/.github/workflows/fork-pr.yml index 86cfa0095fdd5..4e07f2ccd88fa 100644 --- a/.github/workflows/fork-pr.yml +++ b/.github/workflows/fork-pr.yml @@ -34,8 +34,27 @@ jobs: !/.repos/ sparse-checkout-cone-mode: false - uses: ./.github/actions/setup-ci-deps + - uses: ./.github/actions/setup-apt-mirrors + - name: Resolve Playwright version + id: playwright_version + run: echo "version=$(node -p "require('./apps/web/node_modules/playwright/package.json').version")" >> "$GITHUB_OUTPUT" + - name: Cache Chromium + id: chromium_cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/ms-playwright + key: playwright-chromium-shell-${{ runner.os }}-${{ runner.arch }}-${{ steps.playwright_version.outputs.version }} - name: Install Chromium for changed browser tests - run: node apps/web/node_modules/playwright/cli.js install --with-deps --only-shell chromium + if: steps.chromium_cache.outputs.cache-hit != 'true' + run: node apps/web/node_modules/playwright/cli.js install --only-shell chromium + - name: Install missing Chromium system dependencies + run: | + browser=$(find "$HOME/.cache/ms-playwright" -name chrome-headless-shell -type f -print -quit) + test -n "$browser" + libraries=$(ldd "$browser") + if grep '=> not found' <<< "$libraries"; then + node apps/web/node_modules/playwright/cli.js install-deps chromium + fi - name: Check changes against the target branch env: BASE: ${{ github.event.pull_request.base.sha || format('origin/{0}', inputs.base) }} diff --git a/.github/workflows/mobile-fingerprint-check.yml b/.github/workflows/mobile-fingerprint-check.yml index 9fabb3a5f2ad2..62e905d45656c 100644 --- a/.github/workflows/mobile-fingerprint-check.yml +++ b/.github/workflows/mobile-fingerprint-check.yml @@ -25,7 +25,7 @@ concurrency: jobs: fingerprint: name: Native fingerprint diff - runs-on: blacksmith-8vcpu-ubuntu-2404 + runs-on: ${{ github.repository == 'amv146/t3code' && 'ubuntu-24.04' || 'blacksmith-8vcpu-ubuntu-2404' }} permissions: contents: read env: