diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41a3f04..762634e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -340,16 +340,47 @@ jobs: if-no-files-found: warn retention-days: 7 + # Build actual shipping Docker targets and recover disposable synthetic data. + # No registry publication, deployment credentials or production environment. + container-rehearsal: + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + env: + ONEERP_CONTAINER_REHEARSAL: '1' + steps: + - name: Checkout exact source revision + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Build, start and recover isolated application containers + run: node scripts/ci/container-rehearsal.mjs + - name: Upload sanitized container rehearsal evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: container-rehearsal-report + path: ${{ runner.temp }}/container-rehearsal-report.json + if-no-files-found: error + retention-days: 7 + # Retain the existing required check name and require ALL independent jobs. validate: if: always() - needs: [security-audit, validation-tests, http-journey, browser-workspace] + needs: [security-audit, validation-tests, http-journey, browser-workspace, container-rehearsal] runs-on: ubuntu-latest steps: - - name: Require security, validation, HTTP and browser acceptance success + - name: Require security, validation, HTTP, browser and container recovery success env: SECURITY_RESULT: ${{ needs.security-audit.result }} VALIDATION_RESULT: ${{ needs.validation-tests.result }} HTTP_RESULT: ${{ needs.http-journey.result }} BROWSER_RESULT: ${{ needs.browser-workspace.result }} - run: test "$SECURITY_RESULT" = success && test "$VALIDATION_RESULT" = success && test "$HTTP_RESULT" = success && test "$BROWSER_RESULT" = success + CONTAINER_RESULT: ${{ needs.container-rehearsal.result }} + run: test "$SECURITY_RESULT" = success && test "$VALIDATION_RESULT" = success && test "$HTTP_RESULT" = success && test "$BROWSER_RESULT" = success && test "$CONTAINER_RESULT" = success diff --git a/docs/CI_CONTAINER_REHEARSAL.md b/docs/CI_CONTAINER_REHEARSAL.md new file mode 100644 index 0000000..9e277a1 --- /dev/null +++ b/docs/CI_CONTAINER_REHEARSAL.md @@ -0,0 +1,32 @@ +# 当前提交容器交付演练 / Current-head container rehearsal + +`scripts/ci/container-rehearsal.mjs` 在 GitHub Actions 的一次性 Linux runner 上，使用当前干净 checkout 的现有 Dockerfile 构建 API `production`、API `builder`（migration/init）及 Web `production` 镜像。它不推送镜像、不调用 SSH、不部署生产、不合并分支。 + +运行入口（只在一次性 GitHub Actions runner 执行）： + +```bash +CI=true GITHUB_ACTIONS=true ONEERP_CONTAINER_REHEARSAL=1 \ + CONTAINER_REHEARSAL_REPORT="$RUNNER_TEMP/container-rehearsal-report.json" \ + node scripts/ci/container-rehearsal.mjs +``` + +工作流应 checkout PR 的精确 head，授予 `contents: read`，设置 60 分钟 job 超时，并把该 job 纳入最终 `validate` 必须成功的依赖。脚本不需要主机 npm 安装；HTTP 验收脚本只使用 Node 22 内置 API。 + +## 实际验证边界 + +1. 记录 `git rev-parse HEAD`、tree 和实际本地 image ID；开始与结束都要求 source checkout 干净。 +2. 顺序构建三个应用镜像以及现有 `scripts/ci/minio.Dockerfile` 的 CI 专用固定源码 MinIO 镜像。MinIO **不是生产镜像替代方案，未解决 #35**。 +3. 用随机 `oneerp-ci-<24 hex>` 项目创建 PostgreSQL、Redis、MinIO 私有网络和卷。仅 API/Web 使用随机回环端口。依赖 readiness 通过后，实际执行生产 migration/init 镜像。 +4. API/Web 使用生产镜像启动；验证运行镜像 ID，读取 Web `/login` 及其引用的静态资源，验证 Web API proxy health 和登录。库存贸易 13 步与制造 12 步通过真实 HTTP 容器边界执行。 +5. 调用 `recovery-rehearsal.mjs`：备份合成 PostgreSQL 和上传对象，使用同一组实际运行 image ID 在独立空卷中恢复；比较业务行、文件身份和下载字节。恢复不重新迁移或初始化。 +6. 默认删除本次随机项目的容器、网络和卷，删除临时凭据/原始验收报告；仅输出允许字段构成的 JSON 证据。清理失败也使门禁失败。 + +脚本拒绝缺失显式 CI 开关、非默认 Docker host/context、非本地 runner socket、脏 checkout 及非回环发布端口。随机密码通过 GitHub mask 注册，临时 `.env` 权限为 `0600`。不要上传临时目录、Docker config/inspect 全文、容器日志、SQL、归档、signed URL 或 token。工作流只上传 `container-rehearsal-report.json`。 + +## 资源与证据解释 + +标准 Ubuntu runner 顺序构建，运行时每个 stack 内存上限合计约 3 GiB（一次性 migration 上限 1 GiB；恢复阶段最多两个 stack）。构建、镜像和 BuildKit 缓存需要额外内存/磁盘；预留约 10–15 GB 磁盘作为初始预算，实际消耗以 runner 为准。每个命令有超时，job 的 60 分钟限额是最终上界。没有全局 prune、共享卷删除或镜像发布；runner 销毁负责最终镜像/缓存回收。 + +本地无 Docker daemon 时，`node --test scripts/ci/container-rehearsal.test.mjs` 只证明命令编排、防误用、清理和证据脱敏；它不证明镜像能构建或服务能启动。只有精确提交的 GitHub 容器 job 通过后，才能报告该提交的真实构建、25 步 HTTP 与合成恢复已通过。证据 JSON 中的本地 image ID 是本次构建身份，不是可供生产拉取的 registry digest。 + +该结果仍不代替目标生产环境的 MinIO 分发治理、TLS/唯一凭据、异地备份、生产 RPO/RTO、真实数据与角色验收、库存/财务/运维签字。生产批准继续使用 `PRODUCTION_READINESS.md`、`HA_LITE_RUNBOOK.md` 与 `GO_LIVE_CHECKLIST.md`。 diff --git a/docs/CONTROLLED_TRIAL_DELIVERY.md b/docs/CONTROLLED_TRIAL_DELIVERY.md new file mode 100644 index 0000000..658dc78 --- /dev/null +++ b/docs/CONTROLLED_TRIAL_DELIVERY.md @@ -0,0 +1,30 @@ +# 受控试运行交付 + +本轮把当前源代码的容器构建、业务验收与隔离恢复接到同一个 CI 门禁。交付状态以对应 PR 精确 head 的运行记录为准；本文件不是生产上线批准。 + +## 验收入口 + +GitHub Actions 的 `CI / container-rehearsal` 从 PR head 构建现有 API production、API builder（迁移）和 Web production Docker target。所有服务及恢复副本都使用随机的 `oneerp-ci-*` 项目和临时凭据；端口只绑定 runner 的 loopback。 + +执行顺序：构建镜像 → 初始化空数据库 → 启动 API/Web → 检查登录页、静态资源和 API 代理 → 13 步库存贸易与 12 步制造 HTTP 链路 → 备份 PostgreSQL/MinIO → 在新卷中恢复并验证。恢复复用正在运行的镜像 ID，不重新执行管理员初始化，不覆盖源数据。 + +下载该次运行的 `container-rehearsal-report` artifact，可核对源提交、源码树、镜像 ID、各步骤状态、恢复表记录数和文件校验结果。任何失败都阻断 aggregate validate。模拟 Docker 的脚本测试不等于真实容器运行通过。 + +此演练仅允许 GitHub Actions 的一次性环境，不能把 CI 标志加到生产服务器后运行。临时数据库、归档、凭据和完整容器日志不作为 artifact 发布。 + +恢复组件独立执行 `pg_dump`、`tar` 和 `psql`，不等于已经验收运维用的 `backup.*`、`restore.*` 脚本。它在合成业务写入结束、事件队列排空后归档；不证明并发写入中的跨存储一致性。恢复 Redis 使用新卷，原有会话与缓存连续性不在本次验收范围。 + +## 集成和部署边界 + +- 保留现有审查栈的提交历史；通过精确 head 验证后，再固定独立的 `release/controlled-trial-*` 分支作为试运行基线。 +- 主分支推送可能触发镜像发布和服务器部署。创建试运行分支与 Draft PR 不代表已合并或已部署。 +- CI 的 MinIO 源码构建仅用于测试；生产分发问题 #35 仍需可访问、经过版本治理的生产镜像。不能把这份旧版 CI 镜像直接发布为生产方案。 +- 当前恢复演练使用合成数据，证明的是当前镜像及数据库/对象文件恢复链路；真实目标的 TLS、异地备份、恢复时长与业务签字仍须独立验证。 +- 已有数据库的切换必须先停止写入并排空旧事件消费者，尤其是 #41 会计事件来源字段变更。不能在旧消费者运行时直接迁移，再用新版本重放历史事件。 +- 回退应同时固定应用镜像、Compose/脚本版本、数据库兼容性与备份。仅切换旧 image tag 不能证明数据库可回退。 + +## 实际环境接入所需信息 + +目标环境地址与现有访问方式、部署开关及环境保护状态、生产对象存储镜像/服务选择，以及负责验收的业务角色。未能通过当前工具读取 GitHub Secrets/Variables，不等于这些配置不存在；不得为了验证其存在而打印凭据或触发部署。 + +接入之后应先在隔离试运行目标验证当前固定版本，再按业务清单验收采购、销售、库存、制造、工程文档和财务边界。生产切换需保留停写、备份、恢复与回退记录。 diff --git a/docs/PRODUCTION_READINESS.md b/docs/PRODUCTION_READINESS.md index 6770406..0d9496b 100644 --- a/docs/PRODUCTION_READINESS.md +++ b/docs/PRODUCTION_READINESS.md @@ -74,7 +74,9 @@ Deploy workflow 会校验部署变量格式：`DEPLOY_PATH` 必须是无空格 远端部署目录必须保留生产 `.env`；workflow 会在部署前同步当前 commit 的 `docker-compose.prod.yml`、`scripts/audit-prod-config.sh` 和 `scripts/deploy-check.sh`，并在拉取镜像前审计远端 `.env`。部署完成后会自动执行远端 `scripts/deploy-check.sh`；`scripts/prod-smoke.*` 和 `scripts/business-acceptance.*` 仍需在受控试运行前单独执行并留存报告。 -`scripts/deploy-check.*` 不只检查 Docker Compose 命令可执行，还会把 `exited`、`unhealthy`、`restarting`、`dead` 等服务状态视为失败。 +`scripts/deploy-check.*` 会从所选 Compose 配置确认 `api`、`web`、`db`、`redis`、`minio` 五个服务已定义，并逐容器检查它们都在运行；配置了 healthcheck 的容器必须为 `healthy`。检查会等待最多 60 秒（可用 `DEPLOY_CHECK_TIMEOUT_SECONDS` 调整，范围 0–900 秒），`starting` 和尚未创建的容器会在期限内重试；超时、缺失服务、已退出或 unhealthy 容器会失败。若所选配置定义了 `migrate`，还要求其容器以退出码 0 完成；未定义 migrate 时跳过此项。Compose 文件中的 profile-only 服务（例如 `with-nginx` 下的 `nginx`）不会成为必需服务。 + +Shell 脚本使用 `COMPOSE_FILE`、`ENV_FILE` 环境变量；PowerShell 脚本支持同名环境变量和 `-ComposeFile`、`-EnvFile` 参数。健康探测使用 `docker compose ps --all -q ` 和仅包含容器状态、health 状态、退出码的 `docker inspect --format` 输出，不打印渲染后的 Compose 环境值。 ## P1：强烈建议 / Strongly Recommended diff --git a/package.json b/package.json index ddba18b..d14246b 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "commitlint": "commitlint --config commitlint.config.cjs --from=HEAD~1 --to=HEAD", "typecheck": "npm --prefix apps/api run typecheck && npm --prefix apps/web run typecheck", "lint": "npm --prefix apps/api run lint && npm --prefix apps/web run lint", - "test": "node --test scripts/*.test.mjs && npm --prefix apps/api run test -- --passWithNoTests && npm --prefix apps/web run test", + "test": "node --test scripts/*.test.mjs scripts/ci/*.test.mjs && npm --prefix apps/api run test -- --passWithNoTests && npm --prefix apps/web run test", "build": "npm --prefix apps/api run build && npm --prefix apps/web run build", "validate": "node scripts/prisma-static-check.mjs && npm run typecheck && npm run lint && npm run test && npm run build", "audit:security": "npm audit --audit-level=high && npm --prefix apps/api audit --audit-level=high && npm --prefix apps/web audit --audit-level=high && npm --prefix apps/mobile audit --audit-level=high", diff --git a/scripts/ci/compose.rehearsal.yml b/scripts/ci/compose.rehearsal.yml new file mode 100644 index 0000000..733363e --- /dev/null +++ b/scripts/ci/compose.rehearsal.yml @@ -0,0 +1,134 @@ +# Disposable GitHub CI only. The production Compose file and MinIO policy are +# deliberately unchanged; this uses the existing verified-source CI MinIO build. +# All images are built locally by container-rehearsal.mjs. Never publish this stack. +x-runtime: &runtime + restart: "no" + logging: + driver: json-file + options: + max-size: "5m" + max-file: "2" + +services: + db: + <<: *runtime + image: postgres:15-alpine + mem_limit: 512m + environment: + POSTGRES_USER: ${POSTGRES_USER:?required} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?required} + POSTGRES_DB: ${POSTGRES_DB:?required} + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 3s + timeout: 3s + retries: 30 + + redis: + <<: *runtime + image: redis:7-alpine + mem_limit: 192m + command: redis-server --appendonly yes --save 60 1000 + volumes: + - redisdata:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 30 + + minio: + <<: *runtime + image: ${REHEARSAL_MINIO_IMAGE:?required} + pull_policy: never + mem_limit: 768m + environment: + MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:?required} + MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:?required} + command: server /data + volumes: + - miniodata:/data + # The scratch CI image intentionally has no shell/curl/mc. The orchestrator + # probes its HTTP readiness from the API image on this private network. + + migrate: + <<: *runtime + image: ${REHEARSAL_MIGRATE_IMAGE:?required} + pull_policy: never + mem_limit: 1g + depends_on: + db: + condition: service_healthy + environment: + DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB} + INIT_ADMIN_EMAIL: ${INIT_ADMIN_EMAIL:?required} + INIT_ADMIN_PASSWORD: ${INIT_ADMIN_PASSWORD:?required} + INIT_ADMIN_NAME: Container Rehearsal + INIT_COMPANY_NAME: ${INIT_COMPANY_NAME:?required} + command: ["sh", "-c", "npx --no-install prisma migrate deploy --schema=./prisma/schema.prisma && npm run init:prod"] + + api: + <<: *runtime + image: ${REHEARSAL_API_IMAGE:?required} + pull_policy: never + mem_limit: 1g + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + minio: + condition: service_started + environment: + NODE_ENV: production + DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + JWT_SECRET: ${JWT_SECRET:?required} + CORS_ORIGINS: http://127.0.0.1 + REDIS_HOST: redis + REDIS_PORT: "6379" + MINIO_ENDPOINT: minio + MINIO_PORT: "9000" + MINIO_USE_SSL: "false" + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + AI_WRITE_ENABLED: "false" + PORT: "8000" + ports: + - "127.0.0.1::8000" + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:8000/api/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"] + interval: 3s + timeout: 3s + retries: 40 + start_period: 15s + + web: + <<: *runtime + image: ${REHEARSAL_WEB_IMAGE:?required} + pull_policy: never + mem_limit: 512m + depends_on: + api: + condition: service_healthy + environment: + NODE_ENV: production + PORT: "3000" + HOSTNAME: 0.0.0.0 + NEXT_PUBLIC_API_BASE_URL: /api/proxy + API_BASE_URL: http://api:8000 + ports: + - "127.0.0.1::3000" + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3000/login').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"] + interval: 3s + timeout: 3s + retries: 40 + start_period: 10s + +volumes: + pgdata: + redisdata: + miniodata: diff --git a/scripts/ci/container-rehearsal.mjs b/scripts/ci/container-rehearsal.mjs new file mode 100644 index 0000000..22ca26b --- /dev/null +++ b/scripts/ci/container-rehearsal.mjs @@ -0,0 +1,306 @@ +#!/usr/bin/env node +import { randomBytes } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const PROJECT_PATTERN = /^oneerp-ci-[a-f0-9]{24}(?:-restore)?$/; +const IMAGE_ID_PATTERN = /^sha256:[a-f0-9]{64}$/; +const assert = (condition, message) => { if (!condition) throw new Error(message); }; + +export function requireCi(env) { + assert(env.CI === 'true' && env.GITHUB_ACTIONS === 'true' && env.ONEERP_CONTAINER_REHEARSAL === '1', + 'Disposable rehearsal requires CI=true, GITHUB_ACTIONS=true and ONEERP_CONTAINER_REHEARSAL=1'); + assert(!env.DOCKER_HOST && (!env.DOCKER_CONTEXT || env.DOCKER_CONTEXT === 'default'), + 'Disposable rehearsal refuses an overridden Docker host or context'); +} + +export function composeArgs({ root, composeFile, envFile, project }, args) { + assert(PROJECT_PATTERN.test(project), 'Refusing a non-rehearsal Compose project'); + return ['compose', '--project-directory', root, '--env-file', envFile, + '--file', composeFile, '--project-name', project, ...args]; +} + +export function loopbackPort(value) { + const match = /^127\.0\.0\.1:(\d+)$/.exec(value.trim()); + const port = Number(match?.[1]); + assert(match && port >= 1 && port <= 65535, 'Compose did not publish exactly one loopback port'); + return `http://127.0.0.1:${port}`; +} + +export function summarizeJourney(report, expectedSteps) { + assert(report?.passed === true && Array.isArray(report.steps) + && report.steps.length === expectedSteps && report.steps.every((step) => step.passed === true), + 'HTTP journey did not pass its complete expected step set'); + return { passed: true, passedSteps: report.steps.length, expectedSteps }; +} + +function summarizeRecovery(result) { + return { status: result.status, + checks: result.checks?.map(({ name, status }) => ({ name, status })), + verifiedFileCount: result.verifiedFileCount, + objectEvidenceDigest: result.objectEvidenceDigest, + businessTables: result.businessTables?.map(({ table, count }) => ({ table, count })), + }; +} + +// No shell, secret-bearing command text, or unbounded process output. Diagnostics +// stay in memory and are never included in the public evidence artifact. +export function runCommand(command, args, { cwd = ROOT, env = process.env, + timeoutMs = 120_000, signal, onFailureOutput } = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], signal }); + let output = ''; + let diagnostics = ''; + let timedOut = false; + let killTimer; + const timer = setTimeout(() => { + timedOut = true; + child.kill('SIGTERM'); + killTimer = setTimeout(() => child.kill('SIGKILL'), 5_000); + }, timeoutMs); + const collect = (chunk) => { + output = `${output}${chunk}`.slice(-131_072); + diagnostics = `${diagnostics}${chunk}`.slice(-131_072); + }; + child.stdout.on('data', collect); + // Do not mix stderr with machine-readable stdout. + child.stderr.on('data', (chunk) => { diagnostics = `${diagnostics}${chunk}`.slice(-131_072); }); + child.once('error', (error) => { clearTimeout(timer); clearTimeout(killTimer); reject(error); }); + child.once('close', (code) => { + clearTimeout(timer); + clearTimeout(killTimer); + if (timedOut || code !== 0) { + onFailureOutput?.(diagnostics); + reject(new Error(timedOut ? 'Command timed out' : `Command exited ${code}`)); + } else resolve(output.trim()); + }); + }); +} + +export async function checkWeb(webBaseUrl, credentials, companyName, fetchImpl = fetch) { + const request = async (route, options = {}) => { + const response = await fetchImpl(`${webBaseUrl}${route}`, { + ...options, redirect: 'error', signal: AbortSignal.timeout(15_000), + }); + assert(response.ok, 'Web or its API proxy returned a non-success response'); + return response; + }; + const page = await request('/login'); + assert(page.headers.get('content-type')?.includes('text/html'), 'Web login did not return HTML'); + const html = await page.text(); + const assets = [...new Set([...html.matchAll(/(?:src|href)=["'](\/_next\/static\/[^"']+)["']/g)] + .map((match) => match[1]))]; + assert(assets.length > 0 && assets.length <= 64 && assets.some((asset) => /\.js(?:\?|$)/.test(asset)), + 'Web login did not reference a bounded set of production JavaScript assets'); + for (const asset of assets) { + const url = new URL(asset, webBaseUrl); + assert(url.origin === webBaseUrl && url.pathname.startsWith('/_next/static/'), 'Invalid Web static asset path'); + const response = await request(`${url.pathname}${url.search}`); + assert(!response.headers.get('content-type')?.includes('text/html') + && (await response.arrayBuffer()).byteLength > 0, 'Web static asset was missing or returned HTML'); + } + const health = await (await request('/api/proxy/health')).json(); + assert(health.status === 'ok', 'Web proxy health did not reach the API'); + const login = await (await request('/api/proxy/auth/login', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify(credentials), + })).json(); + assert(typeof login.accessToken === 'string' && login.accessToken.length > 0 + && login.companies?.filter((company) => company.name === companyName).length === 1, + 'Web proxy login did not select the synthetic company'); + return { loginPage: 'passed', staticAssets: assets.length, apiHealthProxy: 'passed', loginProxy: 'passed' }; +} + +export async function runRehearsal({ root = ROOT, env = process.env, runner = runCommand, + fetchImpl = fetch, recover, log = console.log, signal } = {}) { + requireCi(env); + const reportPath = path.resolve(env.CONTAINER_REHEARSAL_REPORT + || path.join(env.RUNNER_TEMP || tmpdir(), 'container-rehearsal-report.json')); + const tempDir = await mkdtemp(path.join(env.RUNNER_TEMP || tmpdir(), 'oneerp-container-')); + const nonce = randomBytes(12).toString('hex'); + const sourceProject = `oneerp-ci-${nonce}`; + const recoveryProject = `${sourceProject}-restore`; + const composeFile = path.join(root, 'scripts/ci/compose.rehearsal.yml'); + const envFile = path.join(tempDir, 'rehearsal.env'); + const imageTags = Object.fromEntries(['api', 'migrate', 'web', 'minio'].map((name) => [name, `${sourceProject}-${name}:local`])); + const generated = { + POSTGRES_USER: 'oneerp_ci', POSTGRES_DB: 'oneerp_ci', POSTGRES_PASSWORD: randomBytes(32).toString('hex'), + JWT_SECRET: randomBytes(48).toString('hex'), MINIO_ACCESS_KEY: `ci${randomBytes(12).toString('hex')}`, + MINIO_SECRET_KEY: randomBytes(32).toString('hex'), INIT_ADMIN_EMAIL: `${sourceProject}@example.invalid`, + INIT_ADMIN_PASSWORD: `Ci!${randomBytes(32).toString('hex')}`, + INIT_COMPANY_NAME: `OneERP Synthetic HTTP Journey ${nonce}`, + REHEARSAL_API_IMAGE: imageTags.api, REHEARSAL_MIGRATE_IMAGE: imageTags.migrate, + REHEARSAL_WEB_IMAGE: imageTags.web, REHEARSAL_MINIO_IMAGE: imageTags.minio, + }; + for (const key of ['POSTGRES_PASSWORD', 'JWT_SECRET', 'MINIO_ACCESS_KEY', 'MINIO_SECRET_KEY', 'INIT_ADMIN_PASSWORD']) { + log(`::add-mask::${generated[key]}`); + } + const dockerEnv = { ...env, ...generated, COMPOSE_DISABLE_ENV_FILE: 'true', DOCKER_BUILDKIT: '1' }; + for (const key of ['COMPOSE_FILE', 'COMPOSE_PROJECT_NAME', 'COMPOSE_PROFILES', 'COMPOSE_ENV_FILES']) delete dockerEnv[key]; + const report = { schemaVersion: 1, scope: 'disposable-current-head-container-rehearsal', + status: 'failed', startedAt: new Date().toISOString(), commit: null, tree: null, + images: {}, checks: [], limitations: ['CI-only MinIO source image; production distribution issue #35 remains open', + 'Synthetic data; no production deployment, TLS, offsite backup, RPO certification or business sign-off'] }; + const call = (command, args, options = {}) => runner(command, args, { cwd: root, env: dockerEnv, signal, ...options }); + const compose = (args, options = {}, project = sourceProject) => call('docker', + composeArgs({ root, composeFile, envFile, project }, args), options); + const step = async (name, fn) => { + log(`[START] ${name}`); + try { + const result = await fn(); + report.checks.push({ name, status: 'passed' }); + log(`[PASS] ${name}`); + return result; + } catch { + report.checks.push({ name, status: 'failed' }); + throw new Error(`Container rehearsal failed at ${name}`); + } + }; + let failure; + let mayHaveContainers = false; + try { + await writeFile(envFile, Object.entries(generated).map(([key, value]) => `${key}=${JSON.stringify(value)}\n`).join(''), { mode: 0o600 }); + await step('clean-current-head', async () => { + report.commit = await call('git', ['rev-parse', 'HEAD']); + report.tree = await call('git', ['rev-parse', 'HEAD^{tree}']); + assert(/^[a-f0-9]{40}$/.test(report.commit) && /^[a-f0-9]{40}$/.test(report.tree), 'Invalid Git identity'); + assert(!(await call('git', ['status', '--porcelain', '--untracked-files=all'])), 'Build requires a clean source checkout'); + }); + await step('local-docker-and-compose', async () => { + const endpoint = await call('docker', ['context', 'inspect', '--format', '{{.Endpoints.docker.Host}}']); + assert(endpoint === 'unix:///var/run/docker.sock', 'Only the local GitHub runner Docker socket is allowed'); + await call('docker', ['info', '--format', '{{.ServerVersion}}']); + await call('docker', ['compose', 'version']); + await compose(['config', '--quiet']); + }); + for (const [name, dockerfile, target, context, minutes] of [ + ['migrate', 'apps/api/Dockerfile', 'builder', '.', 12], + ['api', 'apps/api/Dockerfile', 'production', '.', 10], + ['web', 'apps/web/Dockerfile', 'production', '.', 15], + ['minio', 'scripts/ci/minio.Dockerfile', null, 'scripts/ci', 20], + ]) { + await step(`build-${name}-image`, async () => { + const args = ['build', '--progress=plain', '--file', dockerfile, '--tag', imageTags[name]]; + if (target) args.push('--target', target, '--label', `org.opencontainers.image.revision=${report.commit}`); + args.push(context); + await call('docker', args, { timeoutMs: minutes * 60_000, onFailureOutput: (output) => { + // Builds receive no secret build args. Still redact generated values and + // prefix each line so untrusted build output cannot issue Actions commands. + let safe = output; + for (const value of Object.values(generated)) safe = safe.split(value).join('[REDACTED]'); + safe = safe.replace(/[\x00-\x08\x0b-\x1f\x7f]/g, '').slice(-16_384); + log(safe.split('\n').slice(-80).map((line) => `[BUILD ${name}] ${line}`).join('\n')); + } }); + const imageId = await call('docker', ['image', 'inspect', '--format', '{{.Id}}', imageTags[name]]); + assert(IMAGE_ID_PATTERN.test(imageId), 'Invalid built image ID'); + report.images[name] = { id: imageId, target: target || 'CI-only verified source' }; + }); + } + await step('start-isolated-dependencies', async () => { + // Set before up: a partially failed up can still have created owned volumes. + mayHaveContainers = true; + await compose(['up', '--detach', '--wait', '--wait-timeout', '120', 'db', 'redis', 'minio'], { timeoutMs: 180_000 }); + for (const name of ['db', 'redis', 'minio']) { + const container = await compose(['ps', '-q', name]); + assert(/^[a-f0-9]{12,64}$/.test(container), 'Expected exactly one dependency container'); + const imageId = await call('docker', ['inspect', '--format', '{{.Image}}', container]); + assert(IMAGE_ID_PATTERN.test(imageId), 'Invalid dependency image ID'); + if (name === 'minio') assert(imageId === report.images.minio.id, 'MinIO image changed after build'); + else report.images[name] = { id: imageId }; + } + await compose(['run', '--rm', '--no-deps', '--entrypoint', 'node', 'api', '-e', + "(async()=>{for(let i=0;i<60;i++){try{const r=await fetch('http://minio:9000/minio/health/ready',{signal:AbortSignal.timeout(2000)});if(r.ok)return}catch{}await new Promise(r=>setTimeout(r,1000))}process.exit(1)})()"], + { timeoutMs: 200_000 }); + }); + await step('migrate-and-initialize-production-schema', () => compose(['run', '--rm', '--no-deps', 'migrate'], { timeoutMs: 180_000 })); + await step('start-production-api-and-web', async () => { + await compose(['up', '--detach', '--no-deps', '--wait', '--wait-timeout', '180', 'api', 'web'], { timeoutMs: 240_000 }); + for (const name of ['api', 'web']) { + const container = await compose(['ps', '-q', name]); + assert(/^[a-f0-9]{12,64}$/.test(container), 'Expected exactly one application container'); + const imageId = await call('docker', ['inspect', '--format', '{{.Image}}', container]); + assert(imageId === report.images[name].id, 'Running application image differs from built image'); + } + }); + const apiBaseUrl = `${loopbackPort(await compose(['port', 'api', '8000']))}/api`; + const webBaseUrl = loopbackPort(await compose(['port', 'web', '3000'])); + const credentials = { email: generated.INIT_ADMIN_EMAIL, password: generated.INIT_ADMIN_PASSWORD }; + report.web = await step('web-login-and-api-proxy', () => checkWeb(webBaseUrl, credentials, generated.INIT_COMPANY_NAME, fetchImpl)); + const journeyEnv = { ...dockerEnv, GITHUB_SHA: report.commit, API_BASE_URL: apiBaseUrl, + STOCKED_TRADE_ACCEPTANCE: '1', MANUFACTURING_ACCEPTANCE: '1', + BUSINESS_ACCEPTANCE_ADMIN_EMAIL: credentials.email, BUSINESS_ACCEPTANCE_ADMIN_PASSWORD: credentials.password, + BUSINESS_ACCEPTANCE_COMPANY_NAME: generated.INIT_COMPANY_NAME }; + const journeyReports = {}; + report.journeys = {}; + for (const [name, file, expectedSteps] of [ + ['stockedTrade', 'stocked-trade-acceptance.mjs', 13], ['manufacturing', 'manufacturing-acceptance.mjs', 12], + ]) { + const journeyPath = path.join(tempDir, `${name}.json`); + report.journeys[name] = await step(`http-${name}`, async () => { + await call(process.execPath, [path.join(root, 'scripts', file), '--report', journeyPath], { env: journeyEnv, timeoutMs: 300_000 }); + journeyReports[name] = JSON.parse(await readFile(journeyPath, 'utf8')); + return summarizeJourney(journeyReports[name], expectedSteps); + }); + } + report.recovery = await step('backup-and-fresh-volume-recovery', async () => { + const runRecovery = recover || (await import('./recovery-rehearsal.mjs')).runRecovery; + const recoveryReportPath = path.join(tempDir, 'recovery-report.json'); + let result; + try { + result = await runRecovery({ root, composeFile, envFile, sourceProject, recoveryProject, + tempDir, reportPath: recoveryReportPath, commit: report.commit, + credentials, apiBaseUrl, webBaseUrl, journeyReports, dockerEnv }); + } catch (error) { + try { report.recovery = summarizeRecovery(JSON.parse(await readFile(recoveryReportPath, 'utf8'))); } + catch { report.recovery = { status: 'failed' }; } + throw error; + } + assert(result?.status === 'passed', 'Recovery did not pass'); + // Explicit allowlist: never copy generic context, failure output, or credentials. + return summarizeRecovery(result); + }); + await step('source-remains-current-head', async () => { + assert(await call('git', ['rev-parse', 'HEAD']) === report.commit, 'Checkout changed during rehearsal'); + assert(await call('git', ['rev-parse', 'HEAD^{tree}']) === report.tree, 'Source tree changed during rehearsal'); + assert(!(await call('git', ['status', '--porcelain', '--untracked-files=all'])), 'Source changed during rehearsal'); + }); + } catch (error) { + failure = error; + } finally { + if (mayHaveContainers) { + let clean = true; + for (const project of [recoveryProject, sourceProject]) { + try { + // Cleanup must remain available after a cancellation signal. + await compose(['down', '--volumes', '--remove-orphans', '--timeout', '10'], + { signal: undefined, timeoutMs: 60_000 }, project); + } catch { clean = false; } + } + report.checks.push({ name: 'cleanup-owned-projects-and-volumes', status: clean ? 'passed' : 'failed' }); + if (!clean) failure ||= new Error('Container rehearsal cleanup failed'); + } + await rm(tempDir, { recursive: true, force: true }); + report.status = failure ? 'failed' : 'passed'; + report.endedAt = new Date().toISOString(); + await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 }); + } + if (failure) throw failure; + return report; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { + const controller = new AbortController(); + for (const event of ['SIGINT', 'SIGTERM']) process.once(event, () => controller.abort()); + try { + const report = await runRehearsal({ signal: controller.signal }); + console.log(`Container rehearsal ${report.status}: ${report.commit}`); + } catch (error) { + // Stage names are safe; never print child output, credentials, or raw config. + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/scripts/ci/container-rehearsal.test.mjs b/scripts/ci/container-rehearsal.test.mjs new file mode 100644 index 0000000..22b425b --- /dev/null +++ b/scripts/ci/container-rehearsal.test.mjs @@ -0,0 +1,153 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, readFile, rm, writeFile, access } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { checkWeb, composeArgs, loopbackPort, requireCi, runRehearsal, summarizeJourney } from './container-rehearsal.mjs'; + +const ci = { CI: 'true', GITHUB_ACTIONS: 'true', ONEERP_CONTAINER_REHEARSAL: '1' }; +const imageId = `sha256:${'a'.repeat(64)}`; +const sha = 'b'.repeat(40); +const tree = 'c'.repeat(40); +const containerId = 'd'.repeat(64); + +test('opt-in and local Docker guards fail closed', () => { + for (const env of [{}, { CI: 'true' }, { ...ci, ONEERP_CONTAINER_REHEARSAL: '0' }, + { ...ci, DOCKER_HOST: 'ssh://production' }, { ...ci, DOCKER_CONTEXT: 'production' }]) { + assert.throws(() => requireCi(env)); + } + assert.doesNotThrow(() => requireCi(ci)); + assert.throws(() => composeArgs({ project: 'oneerp-production' }, ['down', '--volumes'])); + assert.throws(() => loopbackPort('0.0.0.0:8000')); + assert.throws(() => loopbackPort('127.0.0.1:8000\n127.0.0.1:8001')); + assert.equal(loopbackPort('127.0.0.1:32123\n'), 'http://127.0.0.1:32123'); +}); + +test('complete journey count is required and report does not propagate context', () => { + const report = { passed: true, context: { token: 'do-not-publish' }, steps: Array.from({ length: 13 }, () => ({ passed: true })) }; + assert.deepEqual(summarizeJourney(report, 13), { passed: true, passedSteps: 13, expectedSteps: 13 }); + assert.throws(() => summarizeJourney(report, 12)); + assert.throws(() => summarizeJourney({ ...report, steps: report.steps.map(() => ({ passed: false })) }, 13)); +}); + +function webFetch(company = 'OneERP Synthetic HTTP Journey test', missingAsset = false) { + return async (url) => { + const route = new URL(url).pathname; + if (route === '/login') return new Response('', { headers: { 'content-type': 'text/html' } }); + if (route.startsWith('/_next/static/')) return new Response(missingAsset ? 'missing' : 'content', { headers: { 'content-type': missingAsset ? 'text/html' : 'text/plain' } }); + if (route.endsWith('/health')) return Response.json({ status: 'ok' }); + if (route.endsWith('/auth/login')) return Response.json({ accessToken: 'private-token', companies: [{ name: company }] }); + throw new Error('Unexpected HTTP request'); + }; +} + +test('Web check requires real static assets and API proxy login', async () => { + const company = 'OneERP Synthetic HTTP Journey test'; + const result = await checkWeb('http://127.0.0.1:3000', { email: 'test', password: 'secret' }, company, webFetch(company)); + assert.equal(result.staticAssets, 2); + await assert.rejects(checkWeb('http://127.0.0.1:3000', {}, company, webFetch(company, true)), /static asset/); + await assert.rejects(checkWeb('http://127.0.0.1:3000', {}, 'wrong company', webFetch(company)), /synthetic company/); +}); + +async function harness(t, { failStage, dirty = false, endpoint = 'unix:///var/run/docker.sock' } = {}) { + const dir = await mkdtemp(path.join(tmpdir(), 'rehearsal-test-')); + t.after(() => rm(dir, { recursive: true, force: true })); + const reportPath = path.join(dir, 'report.json'); + const commands = []; + let generated; + let envFile; + const runner = async (command, args, options) => { + commands.push({ command, args }); + generated = options.env; + if (command === 'git') { + if (args[0] === 'status') return dirty ? ' M apps/api/src/main.ts' : ''; + return args[1] === 'HEAD' ? sha : tree; + } + if (command === process.execPath) { + const count = args[0].endsWith('stocked-trade-acceptance.mjs') ? 13 : 12; + await writeFile(args[2], JSON.stringify({ passed: true, context: { token: 'private-token' }, steps: Array.from({ length: count }, () => ({ passed: true })) })); + return ''; + } + assert.equal(command, 'docker'); + if (args[0] === 'context') return endpoint; + if (args[0] === 'image' || args[0] === 'inspect') return imageId; + if (args[0] === 'compose' && args.includes('--env-file')) { + envFile = args[args.indexOf('--env-file') + 1]; + const action = args.slice(args.indexOf('--project-name') + 2); + if (failStage === 'migrate' && action[0] === 'run' && action.at(-1) === 'migrate') throw new Error(generated.INIT_ADMIN_PASSWORD); + if (action[0] === 'ps') return containerId; + if (action[0] === 'port') return action[1] === 'api' ? '127.0.0.1:31000' : '127.0.0.1:31001'; + } + return ''; + }; + const options = { root: '/reviewed/source', env: { ...ci, RUNNER_TEMP: dir, CONTAINER_REHEARSAL_REPORT: reportPath }, runner, + log: () => {}, fetchImpl: async (url, options) => webFetch(generated.INIT_COMPANY_NAME)(url, options), + recover: async (options) => { + assert.match(options.sourceProject, /^oneerp-ci-[a-f0-9]{24}$/); + assert.equal(options.recoveryProject, `${options.sourceProject}-restore`); + assert.equal(options.credentials.password, generated.INIT_ADMIN_PASSWORD); + assert.equal(options.journeyReports.manufacturing.steps.length, 12); + return { status: 'passed', checks: [{ name: 'restored-files', status: 'passed' }], verifiedFileCount: 1, + objectEvidenceDigest: 'e'.repeat(64), businessTables: [{ table: 'FileRecord', count: 1 }], credentials: 'never publish' }; + } }; + return { options, commands, reportPath, getGenerated: () => generated, getEnvFile: () => envFile }; +} + +test('dirty checkout and remote Docker daemon stop before build or cleanup', async (t) => { + for (const scenario of [{ dirty: true }, { endpoint: 'ssh://production' }]) { + const h = await harness(t, scenario); + await assert.rejects(runRehearsal(h.options)); + assert.equal(h.commands.filter(({ args }) => args[0] === 'build' || args.includes('down')).length, 0); + assert.equal(JSON.parse(await readFile(h.reportPath, 'utf8')).status, 'failed'); + } +}); + +test('migration failure cleans only allocated random projects and writes no secrets', async (t) => { + const h = await harness(t, { failStage: 'migrate' }); + await assert.rejects(runRehearsal(h.options), /migrate-and-initialize/); + const cleanups = h.commands.filter(({ args }) => args.includes('down')); + assert.equal(cleanups.length, 2); + for (const { args } of cleanups) { + assert.match(args[args.indexOf('--project-name') + 1], /^oneerp-ci-[a-f0-9]{24}(?:-restore)?$/); + assert.ok(args.includes('--volumes')); + } + assert.equal(h.commands.some(({ command }) => command === process.execPath), false); + const raw = await readFile(h.reportPath, 'utf8'); + assert.equal(raw.includes(h.getGenerated().INIT_ADMIN_PASSWORD), false); + assert.equal(JSON.parse(raw).status, 'failed'); + await assert.rejects(access(h.getEnvFile())); +}); + +test('successful mocked rehearsal records actual source, image IDs, 25 steps and sanitized recovery', async (t) => { + const h = await harness(t); + const result = await runRehearsal(h.options); + assert.equal(result.status, 'passed'); + assert.equal(result.commit, sha); + assert.equal(result.tree, tree); + assert.equal(result.images.api.id, imageId); + assert.equal(result.journeys.stockedTrade.passedSteps + result.journeys.manufacturing.passedSteps, 25); + assert.equal(result.recovery.verifiedFileCount, 1); + assert.equal(h.commands.filter(({ args }) => args[0] === 'build').length, 4); + const raw = await readFile(h.reportPath, 'utf8'); + for (const secret of [h.getGenerated().POSTGRES_PASSWORD, h.getGenerated().JWT_SECRET, + h.getGenerated().MINIO_SECRET_KEY, h.getGenerated().INIT_ADMIN_PASSWORD, 'private-token', 'never publish']) { + assert.equal(raw.includes(secret), false); + } + await assert.rejects(access(h.getEnvFile())); +}); + +test('recovery failure retains only safe failed stage evidence and still cleans up', async (t) => { + const h = await harness(t); + h.options.recover = async ({ reportPath }) => { + await writeFile(reportPath, JSON.stringify({ status: 'failed', + checks: [{ name: 'restored-row-verification', status: 'failed', reason: 'secret details' }], + password: 'never-publish' })); + throw new Error('never-publish'); + }; + await assert.rejects(runRehearsal(h.options), /backup-and-fresh-volume-recovery/); + const raw = await readFile(h.reportPath, 'utf8'); + assert.deepEqual(JSON.parse(raw).recovery, { status: 'failed', checks: [{ name: 'restored-row-verification', status: 'failed' }] }); + assert.equal(raw.includes('never-publish'), false); + assert.equal(raw.includes('secret details'), false); + assert.equal(h.commands.filter(({ args }) => args.includes('down')).length, 2); +}); diff --git a/scripts/ci/recovery-rehearsal.md b/scripts/ci/recovery-rehearsal.md new file mode 100644 index 0000000..15a9187 --- /dev/null +++ b/scripts/ci/recovery-rehearsal.md @@ -0,0 +1,40 @@ +# Synthetic container recovery component + +`runRecovery` is invoked by the container rehearsal after HTTP journeys finish. It +requires `CI=true`, `GITHUB_ACTIONS=true`, `ONEERP_CONTAINER_REHEARSAL=1`, a +`oneerp-ci-*` source project and its exact `-restore` counterpart. Never invoke it +against production or use its result as real-data recovery acceptance. + +Inputs: `root`, `composeFile`, `envFile`, `sourceProject`, `recoveryProject`, +`tempDir`, `reportPath`, `commit`, `credentials: {email,password}`, and `dockerEnv`. +The caller must have finished synthetic writes and background event processing. +It must upload only the explicit sanitized reports, never the temporary directory. + +The component reuses the existing backup scripts' PostgreSQL logical dump and +MinIO volume archive approach. It does not invoke `restore-drill.sh`, because +that script rebuilds images and can initialize a database through dependencies. +The bounded component instead: + +- rejects preexisting restore containers or volumes; +- resolves source running image IDs and starts the same five immutable images; +- removes published ports and migration dependencies, recreates project-scoped + volumes, and rejects external volumes and bind mounts; +- uses `psql -X -v ON_ERROR_STOP=1`; SQL errors prevent subsequent API startup; +- extracts MinIO bytes before starting restored MinIO; +- compares every public table's count and ordered complete-row fingerprint, + including business record IDs; requires a journey engineering revision; +- logs in through the real API and downloads every retained upload through its + API-issued signed URL, checking SHA-256 against the database and source; +- verifies source rows remain unchanged and destroys only the restore project. + +The logical dump and archive stay in process memory. Resolved Compose credentials +are private (0700 directory / 0600 file) and removed in `finally`. Reports contain +only stage status, table counts and an aggregate file-evidence hash, never SQL, +archives, environment values, credentials, file paths, tokens or signed URLs. + +Validation: `node --test scripts/ci/recovery-rehearsal.test.mjs` exercises scope, +volume/image isolation and a mocked SQL failure with restore-only cleanup. These +are orchestration tests, not evidence that a real container recovery succeeded. +The GitHub container rehearsal must supply that evidence for its exact commit. +Graphify graph data and executable were absent in this checkout; no graph refresh +is claimed. diff --git a/scripts/ci/recovery-rehearsal.mjs b/scripts/ci/recovery-rehearsal.mjs new file mode 100644 index 0000000..394f788 --- /dev/null +++ b/scripts/ci/recovery-rehearsal.mjs @@ -0,0 +1,164 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import path from 'node:path'; + +const services = ['db', 'redis', 'minio', 'api', 'web']; +export function assertRecoveryScope({ sourceProject, recoveryProject }, env = process.env) { + if (env.CI !== 'true' || env.GITHUB_ACTIONS !== 'true' || env.ONEERP_CONTAINER_REHEARSAL !== '1') throw new Error('Recovery requires disposable GitHub Actions CI'); + if (!/^oneerp-ci-[a-z0-9-]+$/.test(sourceProject) || recoveryProject !== `${sourceProject}-restore`) throw new Error('Recovery project must be the isolated CI restore project'); +} +export function isolatedConfig(source, imageIds) { + const result = { services: {}, volumes: {} }; + for (const name of services) { + const original = source.services[name]; + if (!original || !/^sha256:[a-f0-9]{64}$/.test(imageIds[name])) throw new Error(`Missing immutable image for ${name}`); + const service = structuredClone(original); + delete service.build; delete service.container_name; delete service.ports; delete service.depends_on; + delete service.networks; delete service.profiles; + service.image = imageIds[name]; service.pull_policy = 'never'; service.restart = 'no'; + for (const volume of service.volumes || []) { + if (volume.type !== 'volume' || !volume.source || source.volumes?.[volume.source]?.external) throw new Error('Recovery permits only project-owned named volumes'); + result.volumes[volume.source] = {}; + } + result.services[name] = service; + } + return result; +} + +// Read-only fingerprint includes every business row and its identity, not just counts. +const fingerprintSql = `SELECT format('SELECT %L AS table_name, count(*) AS row_count, md5(coalesce(string_agg(row_to_json(t)::text, %L ORDER BY row_to_json(t)::text), %L)) AS fingerprint FROM %I.%I t;', tablename, E'\\n', '', schemaname, tablename) FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename;`; +export const recoveryProbeCode = ` +const crypto = require('node:crypto'); +let input=''; process.stdin.on('data', b=>input+=b); process.stdin.on('end', async()=>{ +try { + const {email,password,companyName}=JSON.parse(input); + const credentials={email,password}; + const base='http://127.0.0.1:8000/api'; + async function req(p, options={}) { const r=await fetch(base+p, options); if(!r.ok) throw Error('HTTP '+r.status); const j=await r.json(); return j.data ?? j; } + const login=await req('/auth/login',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(credentials)}); + const token=login.access_token ?? login.accessToken; if(!token) throw Error('Missing login token'); + const companies=(login.companies || []).filter(c=>c.name===companyName); + if(!companyName || companies.length!==1 || !companies[0].id) throw Error('Missing or ambiguous synthetic company'); + const files=await req('/files',{headers:{Authorization:'Bearer '+token,'x-company-id':companies[0].id}}); + if(!Array.isArray(files) || files.length===0) throw Error('No journey upload to verify'); + const result=[]; + for(const f of files) { + if(!f.id || !f.checksumSha256 || !f.downloadUrl) throw Error('Incomplete file evidence'); + const u=new URL(f.downloadUrl); if(u.hostname!=='minio' || u.port!=='9000') throw Error('Unexpected object origin'); + const r=await fetch(u); if(!r.ok) throw Error('Object download failed'); + const checksum=crypto.createHash('sha256').update(Buffer.from(await r.arrayBuffer())).digest('hex'); + if(checksum!==f.checksumSha256) throw Error('Object bytes checksum mismatch'); + result.push({id:f.id,checksumSha256:checksum}); + } + result.sort((a,b)=>a.id.localeCompare(b.id)); + process.stdout.write(JSON.stringify({files:result})); +} catch { process.stderr.write('API recovery verification failed'); process.exitCode=1; } +});`; + +export async function runRecovery(options) { + const { root, composeFile, envFile, sourceProject, recoveryProject, tempDir, reportPath, commit, credentials, dockerEnv = process.env } = options; + assertRecoveryScope(options); + const checks = []; + const report = { schemaVersion: 1, commit, scope: 'synthetic-ci-only', status: 'failed', checks }; + const recoveryDir = path.join(tempDir, 'recovery-private'); + mkdirSync(recoveryDir, { recursive: true, mode: 0o700 }); + const restoreFile = path.join(recoveryDir, 'compose.json'); + let allocated = false; + let stage = 'isolation-guards'; + function docker(args, input) { + const result = spawnSync('docker', args, { cwd: root, env: dockerEnv, input, encoding: input instanceof Buffer ? null : 'utf8', maxBuffer: 128 * 1024 * 1024, timeout: 300000 }); + if (result.error || result.status !== 0) throw new Error('Recovery Docker operation failed'); + return result.stdout; + } + const source = (...args) => docker(['compose', '-p', sourceProject, '--env-file', envFile, '-f', composeFile, ...args]); + const restore = (...args) => docker(['compose', '-p', recoveryProject, '-f', restoreFile, ...args]); + function psql(project, sql) { + const file = project === sourceProject ? composeFile : restoreFile; + const envArgs = project === sourceProject ? ['--env-file', envFile] : []; + return docker(['compose', '-p', project, ...envArgs, '-f', file, 'exec', '-T', 'db', 'sh', '-c', 'psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At'], sql); + } + function snapshot(project) { return psql(project, psql(project, fingerprintSql)).trim(); } + function probe(project) { + const file = project === sourceProject ? composeFile : restoreFile; + const envArgs = project === sourceProject ? ['--env-file', envFile] : []; + return JSON.parse(docker(['compose', '-p', project, ...envArgs, '-f', file, 'exec', '-T', 'api', 'node', '-e', recoveryProbeCode], JSON.stringify({ ...credentials, companyName: dockerEnv.INIT_COMPANY_NAME }))); + } + try { + // Refuse even stopped containers or leftover volumes: never restore into existing data. + for (const kind of ['container', 'volume']) { + if (docker([kind, 'ls', ...(kind === 'container' ? ['-a'] : []), '-q', '--filter', `label=com.docker.compose.project=${recoveryProject}`]).trim()) throw new Error('Recovery project is not fresh'); + } + const config = JSON.parse(source('config', '--format', 'json')); + const imageIds = {}; + const containers = {}; + for (const name of services) { + containers[name] = source('ps', '-q', name).trim(); + if (!/^[a-f0-9]{12,64}$/.test(containers[name])) throw new Error(`Missing source service ${name}`); + imageIds[name] = docker(['inspect', '--format', '{{.Image}}', containers[name]]).trim(); + } + stage = 'source-api-and-backup'; + const sourceFiles = probe(sourceProject); + // The event runner retries every 30s. Await a drained queue without altering it. + let drained = false; + for (let attempt = 0; attempt < 45; attempt++) { + const pending = psql(sourceProject, `SELECT count(*) FROM "EventDlq" WHERE status IN ('PENDING', 'RETRYING');`).trim(); + if (pending === '0') { drained = true; break; } + if (!/^\d+$/.test(pending)) throw new Error('Invalid pending event count'); + await new Promise(resolve => setTimeout(resolve, 2000)); + } + if (!drained) throw new Error('Synthetic event queue did not drain'); + const before = snapshot(sourceProject); + if (!before.split('\n').some(line => /^EngineeringDocumentRevision\|[1-9][0-9]*\|/.test(line))) throw new Error('Missing engineering journey revision'); + const sql = source('exec', '-T', 'db', 'sh', '-c', 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --clean --if-exists --no-owner --no-privileges'); + // The same PostgreSQL alpine image supplies tar; no additional mutable helper image. + const archive = docker(['run', '--rm', '--network', 'none', '--volumes-from', `${containers.minio}:ro`, '--entrypoint', 'tar', imageIds.db, 'czf', '-', '-C', '/data', '.'], Buffer.alloc(0)); + if (!sql || archive.length === 0 || before !== snapshot(sourceProject)) throw new Error('Source changed during backup'); + docker(['run', '--rm', '-i', '--network', 'none', '--entrypoint', 'tar', imageIds.db, 'tzf', '-'], archive); + checks.push({ name: 'backup-consistent', status: 'passed' }); + writeFileSync(restoreFile, JSON.stringify(isolatedConfig(config, imageIds)), { mode: 0o600 }); + allocated = true; + stage = 'postgres-and-object-restore'; + restore('up', '-d', '--wait', '--wait-timeout', '180', 'db', 'redis'); + // Create MinIO without starting it so extraction precedes any object-store writes. + restore('create', 'minio'); + const minio = restore('ps', '-a', '-q', 'minio').trim(); + if (!/^[a-f0-9]{12,64}$/.test(minio)) throw new Error('Missing restore object store'); + psql(recoveryProject, sql); + docker(['run', '--rm', '-i', '--network', 'none', '--volumes-from', minio, '--entrypoint', 'tar', imageIds.db, 'xzf', '-', '-C', '/data'], archive); + stage = 'restored-row-verification'; + const after = snapshot(recoveryProject); + if (before !== after) throw new Error('Restored business row identities or counts differ'); + report.businessTables = before.split('\n').map(line => { const [table, count] = line.split('|'); return { table, count: Number(count) }; }); + checks.push({ name: 'business-row-counts-and-identities', status: 'passed' }); + stage = 'restored-api-and-image-verification'; + restore('up', '-d', 'minio'); + // Scratch MinIO has no healthcheck executable. Probe using the exact API image + // before application initialization can attempt to create its bucket. + restore('run', '--rm', '--no-deps', '--entrypoint', 'node', 'api', '-e', + "(async()=>{for(let i=0;i<60;i++){try{const r=await fetch('http://minio:9000/minio/health/ready',{signal:AbortSignal.timeout(2000)});if(r.ok)return}catch{}await new Promise(r=>setTimeout(r,1000))}process.exit(1)})()"); + restore('up', '-d', '--wait', '--wait-timeout', '180', 'api', 'web'); + for (const name of services) { + const id = restore('ps', '-q', name).trim(); + if (docker(['inspect', '--format', '{{.Image}}', id]).trim() !== imageIds[name]) throw new Error('Restore image differs'); + } + checks.push({ name: 'exact-running-image-reuse', status: 'passed' }); + const recoveredFiles = probe(recoveryProject); + if (JSON.stringify(sourceFiles) !== JSON.stringify(recoveredFiles)) throw new Error('Recovered upload identity differs'); + report.verifiedFileCount = recoveredFiles.files.length; + report.objectEvidenceDigest = createHash('sha256').update(JSON.stringify(recoveredFiles)).digest('hex'); + checks.push({ name: 'restored-api-login-and-upload-bytes', status: 'passed' }); + if (before !== snapshot(sourceProject)) throw new Error('Source business rows changed'); + checks.push({ name: 'source-preserved', status: 'passed' }); + report.status = 'passed'; + } catch (error) { + // No raw Docker output, SQL, environment, signed URLs, tokens or archive content. + checks.push({ name: stage, status: 'failed', reason: 'Recovery verification failed; inspect the failed stage without publishing private data.' }); + throw error; + } finally { + try { if (allocated) restore('down', '-v', '--remove-orphans'); } + catch (error) { report.status = 'failed'; checks.push({ name: 'restore-cleanup', status: 'failed' }); throw error; } + finally { rmSync(recoveryDir, { recursive: true, force: true }); writeFileSync(reportPath, JSON.stringify(report, null, 2)); } + } + return report; +} diff --git a/scripts/ci/recovery-rehearsal.test.mjs b/scripts/ci/recovery-rehearsal.test.mjs new file mode 100644 index 0000000..f3edfdc --- /dev/null +++ b/scripts/ci/recovery-rehearsal.test.mjs @@ -0,0 +1,83 @@ +import test from 'node:test'; +import { spawnSync } from 'node:child_process'; +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { assertRecoveryScope, isolatedConfig, runRecovery, recoveryProbeCode } from './recovery-rehearsal.mjs'; +const scope = { sourceProject: 'oneerp-ci-abc', recoveryProject: 'oneerp-ci-abc-restore' }; +const env = { CI: 'true', GITHUB_ACTIONS: 'true', ONEERP_CONTAINER_REHEARSAL: '1' }; +const digest = `sha256:${'a'.repeat(64)}`; +const names = ['db', 'redis', 'minio', 'api', 'web']; +const images = Object.fromEntries(names.map(n => [n, digest])); +const config = { services: Object.fromEntries(names.map(n => [n, { image: 'mutable:tag', ports: ['8000:8000'], depends_on: { migrate: {} }, volumes: n === 'db' ? [{ type: 'volume', source: 'data', target: '/data' }] : [] }])), volumes: { data: { name: 'source_data' } } }; +test('scope fails closed outside explicit isolated GitHub CI', () => { + assert.doesNotThrow(() => assertRecoveryScope(scope, env)); + for (const key of Object.keys(env)) assert.throws(() => assertRecoveryScope(scope, { ...env, [key]: '' })); + assert.throws(() => assertRecoveryScope({ ...scope, recoveryProject: scope.sourceProject }, env)); + assert.throws(() => assertRecoveryScope({ sourceProject: 'production', recoveryProject: 'production-restore' }, env)); +}); +test('derived stack pins exact images, fresh volumes, no migration or exposed ports', () => { + const isolated = isolatedConfig(config, images); + assert.deepEqual(isolated.volumes, { data: {} }); + for (const service of Object.values(isolated.services)) { + assert.equal(service.image, digest); assert.equal(service.pull_policy, 'never'); + assert.equal(service.ports, undefined); assert.equal(service.depends_on, undefined); + } + assert.equal(isolated.services.migrate, undefined); + const unsafe = structuredClone(config); unsafe.volumes.data.external = true; + assert.throws(() => isolatedConfig(unsafe, images)); + unsafe.volumes.data.external = false; unsafe.services.db.volumes[0].type = 'bind'; + assert.throws(() => isolatedConfig(unsafe, images)); + assert.throws(() => isolatedConfig(config, { ...images, api: 'mutable:tag' })); +}); +test('SQL failure halts before API start and cleans only restore project', async () => { + const dir = mkdtempSync(path.join(os.tmpdir(), 'recovery-mock-')); + const old = Object.fromEntries(Object.keys(env).map(k => [k, process.env[k]])); + Object.assign(process.env, env); + const log = path.join(dir, 'calls.jsonl'); + const report = path.join(dir, 'report.json'); + const mock = `#!/usr/bin/env -S node -- +const fs=require('node:fs'); const a=process.argv.slice(2); const input=fs.readFileSync(0,'utf8'); +fs.appendFileSync(process.env.MOCK_LOG,JSON.stringify({a,input})+'\\n'); +if(a[0]==='container'||a[0]==='volume') process.exit(0); +if(a.includes('config')) process.stdout.write(${JSON.stringify(JSON.stringify(config))}); +else if(a[0]==='inspect') console.log(${JSON.stringify(digest)}); +else if(a.includes('ps')) console.log('aaaaaaaaaaaa'); +else if(a.includes('node')) console.log(JSON.stringify({files:[{id:'file',checksumSha256:'checksum'}]})); +else if(a.some(x=>x.includes('pg_dump'))) console.log('SQL PRIVATE'); +else if(a.includes('czf')) process.stdout.write('archive'); +else if(a.some(x=>x.includes('psql'))) { + if(input.includes('SELECT count(*) FROM')) { console.log('0'); process.exit(0); } + if(a.includes('oneerp-ci-abc-restore') && input.includes('SQL PRIVATE')) process.exit(3); + console.log('EngineeringDocumentRevision|1|fingerprint'); +} +`; + writeFileSync(path.join(dir, 'docker'), mock, { mode: 0o700 }); + try { + await assert.rejects(runRecovery({ ...scope, root: dir, composeFile: '/mock/source.json', envFile: '/mock/ci.env', tempDir: dir, reportPath: report, credentials: { email: 'ci@example.test', password: 'secret-password-sentinel' }, dockerEnv: { ...process.env, PATH: `${dir}:${process.env.PATH}`, MOCK_LOG: log } })); + const calls = readFileSync(log, 'utf8').trim().split('\n').map(JSON.parse); + const sqlRestore = calls.find(c => c.a.includes(scope.recoveryProject) && c.input.includes('SQL PRIVATE')); + assert.ok(sqlRestore.a.some(x => x.includes('ON_ERROR_STOP=1'))); + assert.ok(calls.some(c => c.a.includes(scope.recoveryProject) && c.a.includes('down'))); + assert.ok(!calls.some(c => c.a.includes(scope.sourceProject) && c.a.includes('down'))); + assert.ok(!calls.some(c => c.a.includes(scope.recoveryProject) && c.a.includes('up') && c.a.includes('api'))); + const saved = readFileSync(report, 'utf8'); assert.equal(JSON.parse(saved).status, 'failed'); + assert.ok(!saved.includes('SQL PRIVATE')); assert.ok(!saved.includes('secret-password-sentinel')); + } finally { for(const [k,v] of Object.entries(old)) { if(v===undefined) delete process.env[k]; else process.env[k]=v; } rmSync(dir,{recursive:true,force:true}); } +}); + +test('real API probe selects exact company and supplies tenant header', () => { + const setup = `global.fetch=async(url,options)=>{ url=String(url); + if(url.endsWith('/auth/login')) return {ok:true,json:async()=>({accessToken:'token',companies:[{id:'synthetic-id',name:'CI Company'}]})}; + if(url.endsWith('/files')) { + if(options.headers['x-company-id']!=='synthetic-id') throw Error('Missing tenant'); + return {ok:true,json:async()=>[{id:'file-id',checksumSha256:require('node:crypto').createHash('sha256').update('bytes').digest('hex'),downloadUrl:'http://minio:9000/object'}]}; + } + return {ok:true,arrayBuffer:async()=>Buffer.from('bytes')}; + };`; + const run = companyName => spawnSync(process.execPath, ['-e', setup + recoveryProbeCode], { input: JSON.stringify({email:'ci@example.test',password:'sentinel',companyName}), encoding:'utf8' }); + const passed = run('CI Company'); assert.equal(passed.status, 0, passed.stderr); + assert.equal(JSON.parse(passed.stdout).files[0].id, 'file-id'); + assert.notEqual(run('Wrong Company').status, 0); +}); diff --git a/scripts/deploy-check.ps1 b/scripts/deploy-check.ps1 index f547ae0..f049718 100644 --- a/scripts/deploy-check.ps1 +++ b/scripts/deploy-check.ps1 @@ -1,13 +1,36 @@ param( - [string]$ComposeFile = "docker-compose.ha-lite.yml", - [string]$EnvFile = ".env", + [string]$ComposeFile = "", + [string]$EnvFile = "", [string]$ApiUrl = "", - [string]$WebUrl = "" + [string]$WebUrl = "", + [int]$TimeoutSeconds = -1, + [int]$PollIntervalSeconds = -1 ) $ErrorActionPreference = "Stop" $root = Split-Path -Parent $PSScriptRoot $results = New-Object System.Collections.Generic.List[object] +$script:ComposeServices = @() +$script:InspectFormat = '{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}|{{.State.ExitCode}}' + +if ($ComposeFile -eq "") { $ComposeFile = if (-not [string]::IsNullOrEmpty($env:COMPOSE_FILE)) { $env:COMPOSE_FILE } else { "docker-compose.ha-lite.yml" } } +if ($EnvFile -eq "") { $EnvFile = if (-not [string]::IsNullOrEmpty($env:ENV_FILE)) { $env:ENV_FILE } else { ".env" } } +function ConvertTo-BoundedSeconds { + param([string]$Value, [int]$Default, [int]$Maximum) + if ($Value -notmatch '^\d+$') { return $Default } + $normalized = $Value.TrimStart('0') + if ($normalized -eq "") { return 0 } + if ($normalized.Length -gt 3) { return $Maximum } + $parsed = [int]$normalized + if ($parsed -gt $Maximum) { return $Maximum } + return $parsed +} + +if ($TimeoutSeconds -lt 0) { $TimeoutSeconds = ConvertTo-BoundedSeconds $env:DEPLOY_CHECK_TIMEOUT_SECONDS 60 900 } +if ($PollIntervalSeconds -lt 0) { $PollIntervalSeconds = ConvertTo-BoundedSeconds $env:DEPLOY_CHECK_POLL_SECONDS 3 60 } +if ($TimeoutSeconds -gt 900) { $TimeoutSeconds = 900 } +if ($PollIntervalSeconds -gt 60) { $PollIntervalSeconds = 60 } +if ($PollIntervalSeconds -lt 1) { $PollIntervalSeconds = 1 } function Add-Result { param([string]$Name, [bool]$Passed, [string]$Detail) @@ -29,31 +52,132 @@ function Read-Env { return $map } +function Invoke-Compose { + param([string[]]$ComposeArgs) + $baseArgs = @() + if ($EnvFile -ne "" -and (Test-Path $envPath)) { $baseArgs += @("--env-file", $envPath) } + $baseArgs += @("-f", $composePath) + $output = @(& docker compose @baseArgs @ComposeArgs 2>$null) + $exitCode = $LASTEXITCODE + return [pscustomobject]@{ ExitCode = $exitCode; Output = ($output -join [Environment]::NewLine) } +} + +function Invoke-Docker { + param([string[]]$DockerArgs) + $output = @(& docker @DockerArgs 2>$null) + $exitCode = $LASTEXITCODE + return [pscustomobject]@{ ExitCode = $exitCode; Output = ($output -join [Environment]::NewLine) } +} + function Test-Http { param([string]$Url) try { - $response = Invoke-WebRequest -Uri $Url -UseBasicParsing -TimeoutSec 10 + $response = Invoke-WebRequest -Uri $Url -UseBasicParsing -TimeoutSec 15 return $response.StatusCode -ge 200 -and $response.StatusCode -lt 500 - } catch { - return $false + } catch { return $false } +} + +function Test-ComposeConfig { + $config = Invoke-Compose -ComposeArgs @("config") + $services = Invoke-Compose -ComposeArgs @("config", "--services") + if ($services.ExitCode -eq 0) { + $script:ComposeServices = @($services.Output -split '\r?\n' | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" }) + } + return [pscustomobject]@{ + ConfigPassed = ($config.ExitCode -eq 0) + ServicesPassed = ($services.ExitCode -eq 0 -and $script:ComposeServices.Count -gt 0) } } -function Invoke-Compose { - param([string[]]$ComposeArgs) - $baseArgs = @() - if ($EnvFile -ne "" -and (Test-Path $envPath)) { - $baseArgs += @("--env-file", $envPath) +function Test-ComposeService { + param([string]$Service, [string]$Mode) + if ($script:ComposeServices -notcontains $Service) { + return [pscustomobject]@{ State = "failed"; Detail = "service is not defined in selected Compose configuration" } } - $baseArgs += @("-f", $composePath) - docker compose @baseArgs @ComposeArgs + + $psResult = Invoke-Compose -ComposeArgs @("ps", "--all", "-q", $Service) + if ($psResult.ExitCode -ne 0) { return [pscustomobject]@{ State = "failed"; Detail = "docker compose ps failed" } } + $ids = @($psResult.Output -split '\r?\n' | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" }) + if ($ids.Count -eq 0) { return [pscustomobject]@{ State = "pending"; Detail = "container is not present yet" } } + + $state = "ready" + $detail = "running and healthy" + foreach ($id in $ids) { + $inspect = Invoke-Docker -DockerArgs @("inspect", "--format", $script:InspectFormat, $id) + if ($inspect.ExitCode -ne 0) { return [pscustomobject]@{ State = "failed"; Detail = "container state inspection failed" } } + $parts = @($inspect.Output.Trim() -split '\|', 3) + if ($parts.Count -ne 3) { return [pscustomobject]@{ State = "failed"; Detail = "container state inspection returned invalid fields" } } + $status = $parts[0] + $health = $parts[1] + $exitCode = $parts[2] + + if ($Mode -eq "migration") { + if ($status -eq "exited" -and $exitCode -eq "0") { continue } + if ($status -eq "running" -or $status -eq "created") { + if ($state -ne "failed") { $state = "pending" } + $detail = "migration status=$status" + continue + } + return [pscustomobject]@{ State = "failed"; Detail = "migration status=$status exitCode=$exitCode" } + } + + if ($status -eq "running") { + if ($health -eq "healthy" -or $health -eq "none") { continue } + if ($health -eq "starting") { + if ($state -ne "failed") { $state = "pending" } + $detail = "container health is starting" + continue + } + if ($health -eq "unhealthy") { return [pscustomobject]@{ State = "failed"; Detail = "container health is unhealthy" } } + return [pscustomobject]@{ State = "failed"; Detail = "container health state is $health" } + } + if ($status -eq "created") { + if ($state -ne "failed") { $state = "pending" } + $detail = "container has not started yet" + continue + } + return [pscustomobject]@{ State = "failed"; Detail = "container status=$status" } + } + return [pscustomobject]@{ State = $state; Detail = $detail } } -function Test-ComposeServicesHealthy { - param([string]$PsOutput) - if ($PsOutput.Trim() -eq "") { return $false } - if ($PsOutput -match '(?i)(exited|unhealthy|restarting|dead)') { return $false } - return $true +function Test-ComposeRuntime { + $services = @("api", "web", "db", "redis", "minio") + if ($script:ComposeServices -contains "migrate") { $services += "migrate" } + $modeByService = @{ migrate = "migration" } + $deadline = [DateTime]::UtcNow.AddSeconds($TimeoutSeconds) + $latest = @{} + $timedOut = $false + + while ($true) { + $pending = $false + $failed = $false + foreach ($service in $services) { + $mode = if ($modeByService.ContainsKey($service)) { $modeByService[$service] } else { "runtime" } + $probe = Test-ComposeService -Service $service -Mode $mode + $latest[$service] = $probe + if ($probe.State -eq "pending") { $pending = $true } + if ($probe.State -eq "failed") { $failed = $true } + } + if ($failed -or -not $pending) { break } + if ([DateTime]::UtcNow -ge $deadline) { $timedOut = $true; break } + Start-Sleep -Seconds $PollIntervalSeconds + } + + $passed = $true + foreach ($service in $services) { + $probe = $latest[$service] + if ($probe.State -eq "ready") { + Add-Result "runtime-$service" $true $probe.Detail + } else { + $passed = $false + $detail = $probe.Detail + if ($probe.State -eq "pending" -and $timedOut) { $detail = "timed out waiting: $detail" } + elseif ($probe.State -eq "pending") { $detail = "not ready: $detail" } + Add-Result "runtime-$service" $false $detail + } + } + return $passed } Push-Location $root @@ -65,22 +189,20 @@ try { if ($apiPort -eq "") { $apiPort = "8000" } $webPort = [string]$envMap["WEB_PORT"] if ($webPort -eq "") { $webPort = "3000" } - if ($ApiUrl -eq "" -and $env:API_BASE_URL -ne "") { + if ($ApiUrl -eq "" -and -not [string]::IsNullOrEmpty($env:API_BASE_URL)) { $ApiUrl = $env:API_BASE_URL if ($ApiUrl -notmatch '/api/health$') { $ApiUrl = "$($ApiUrl.TrimEnd('/'))/health" } } - if ($WebUrl -eq "" -and $env:WEB_BASE_URL -ne "") { - $WebUrl = $env:WEB_BASE_URL - } + if ($WebUrl -eq "" -and -not [string]::IsNullOrEmpty($env:WEB_BASE_URL)) { $WebUrl = $env:WEB_BASE_URL } if ($ApiUrl -eq "") { $ApiUrl = "http://localhost:$apiPort/api/health" } if ($WebUrl -eq "") { $WebUrl = "http://localhost:$webPort/" } - docker version | Out-Null - Add-Result "docker" ($LASTEXITCODE -eq 0) "Docker CLI is available" - - Invoke-Compose -ComposeArgs @("config") | Out-Null - Add-Result "compose-config" ($LASTEXITCODE -eq 0) "$ComposeFile is valid" + $docker = Invoke-Docker -DockerArgs @("version") + Add-Result "docker" ($docker.ExitCode -eq 0) "Docker CLI is available" + $config = Test-ComposeConfig + Add-Result "compose-config" $config.ConfigPassed "$ComposeFile is valid" + Add-Result "compose-services" $config.ServicesPassed "Compose service names are available" Add-Result "env-file" (Test-Path $envPath) "$EnvFile exists" foreach ($name in @("POSTGRES_PASSWORD", "JWT_SECRET", "MINIO_SECRET_KEY", "INIT_ADMIN_PASSWORD")) { @@ -89,9 +211,7 @@ try { Add-Result "secret-$name" $strong "length=$($value.Length)" } - $ps = Invoke-Compose -ComposeArgs @("ps") - Add-Result "compose-ps" (($LASTEXITCODE -eq 0) -and (Test-ComposeServicesHealthy -PsOutput "$ps")) "services must be running and not unhealthy/exited" - + Add-Result "compose-ps" (Test-ComposeRuntime) "required services are present and ready; migration exits successfully when declared" Add-Result "api-health" (Test-Http -Url $ApiUrl) $ApiUrl Add-Result "web-root" (Test-Http -Url $WebUrl) $WebUrl @@ -104,9 +224,7 @@ try { results = $results } $report | ConvertTo-Json -Depth 5 | Set-Content -Path (Join-Path $root "deploy-check-report.json") -Encoding UTF8 - if ($failed.Count -gt 0) { - throw "$($failed.Count) deploy check(s) failed. See deploy-check-report.json" - } + if ($failed.Count -gt 0) { throw "$($failed.Count) deploy check(s) failed. See deploy-check-report.json" } } finally { Pop-Location } diff --git a/scripts/deploy-check.sh b/scripts/deploy-check.sh index d2c560b..b6bc486 100644 --- a/scripts/deploy-check.sh +++ b/scripts/deploy-check.sh @@ -1,5 +1,6 @@ #!/usr/bin/env sh set -eu +set -f ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.ha-lite.yml}" @@ -11,7 +12,6 @@ case "$ENV_FILE" in /*) ENV_PATH="$ENV_FILE" ;; *) ENV_PATH="$ROOT/$ENV_FILE" ;; esac - case "$COMPOSE_FILE" in /*) COMPOSE_PATH="$COMPOSE_FILE" ;; *) COMPOSE_PATH="$ROOT/$COMPOSE_FILE" ;; @@ -30,12 +30,7 @@ env_value() { check() { name="$1" shift - if "$@"; then - echo "[PASS] $name" - else - echo "[FAIL] $name" - FAILED=$((FAILED + 1)) - fi + if "$@"; then echo "[PASS] $name"; else echo "[FAIL] $name"; FAILED=$((FAILED + 1)); fi } strong_secret() { @@ -52,15 +47,170 @@ compose() { fi } -compose_services_healthy() { - output="$(compose ps 2>&1)" || return 1 - [ "$output" != "" ] || return 1 - printf '%s\n' "$output" | grep -Eiq '(exited|unhealthy|restarting|dead)' && return 1 - return 0 +compose_config_valid() { compose config >/dev/null 2>&1; } + +load_compose_services() { + CONFIG_SERVICES="$(compose config --services 2>/dev/null)" || return 1 + [ -n "$CONFIG_SERVICES" ] +} + +compose_has_service() { + [ -n "$CONFIG_SERVICES" ] && printf '%s\n' "$CONFIG_SERVICES" | grep -Fxq "$1" +} + +INSPECT_FORMAT='{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}|{{.State.ExitCode}}' +PROBE_STATE='' +PROBE_DETAIL='' + +probe_service() { + probe_name="$1" + probe_mode="$2" + ids="$(compose ps --all -q "$probe_name" 2>/dev/null)" || { + PROBE_STATE=failed + PROBE_DETAIL='docker compose ps failed' + return + } + if [ -z "$ids" ]; then + PROBE_STATE=pending + PROBE_DETAIL='container is not present yet' + return + fi + + PROBE_STATE=ready + PROBE_DETAIL='running and healthy' + for container_id in $ids; do + [ -n "$container_id" ] || continue + state_line="$(docker inspect --format "$INSPECT_FORMAT" "$container_id" 2>/dev/null)" || { + PROBE_STATE=failed + PROBE_DETAIL='container state inspection failed' + break + } + status="${state_line%%|*}" + rest="${state_line#*|}" + health="${rest%%|*}" + exit_code="${rest#*|}" + if [ "$status" = "$state_line" ] || [ "$health" = "$rest" ] || [ "$exit_code" = "$rest" ]; then + PROBE_STATE=failed + PROBE_DETAIL='container state inspection returned invalid fields' + break + fi + + if [ "$probe_mode" = migration ]; then + if [ "$status" = exited ] && [ "$exit_code" = 0 ]; then + : + elif [ "$status" = running ] || [ "$status" = created ]; then + [ "$PROBE_STATE" = failed ] || PROBE_STATE=pending + PROBE_DETAIL="migration status=$status" + else + PROBE_STATE=failed + PROBE_DETAIL="migration status=$status exitCode=$exit_code" + break + fi + elif [ "$status" = running ]; then + case "$health" in + healthy|none) : ;; + starting) + [ "$PROBE_STATE" = failed ] || PROBE_STATE=pending + PROBE_DETAIL='container health is starting' + ;; + unhealthy) + PROBE_STATE=failed + PROBE_DETAIL='container health is unhealthy' + break + ;; + *) + PROBE_STATE=failed + PROBE_DETAIL="container health state is $health" + break + ;; + esac + elif [ "$status" = created ]; then + [ "$PROBE_STATE" = failed ] || PROBE_STATE=pending + PROBE_DETAIL='container has not started yet' + else + PROBE_STATE=failed + PROBE_DETAIL="container status=$status" + break + fi + done } -compose_config_valid() { - compose config >/dev/null +DEPLOY_CHECK_TIMEOUT_SECONDS="${DEPLOY_CHECK_TIMEOUT_SECONDS:-60}" +DEPLOY_CHECK_POLL_SECONDS="${DEPLOY_CHECK_POLL_SECONDS:-3}" +case "$DEPLOY_CHECK_TIMEOUT_SECONDS" in ''|*[!0-9]*) DEPLOY_CHECK_TIMEOUT_SECONDS=60 ;; esac +case "$DEPLOY_CHECK_POLL_SECONDS" in ''|*[!0-9]*) DEPLOY_CHECK_POLL_SECONDS=3 ;; esac +DEPLOY_CHECK_TIMEOUT_SECONDS="$(printf '%s' "$DEPLOY_CHECK_TIMEOUT_SECONDS" | sed 's/^0*//')" +DEPLOY_CHECK_POLL_SECONDS="$(printf '%s' "$DEPLOY_CHECK_POLL_SECONDS" | sed 's/^0*//')" +[ -n "$DEPLOY_CHECK_TIMEOUT_SECONDS" ] || DEPLOY_CHECK_TIMEOUT_SECONDS=0 +[ -n "$DEPLOY_CHECK_POLL_SECONDS" ] || DEPLOY_CHECK_POLL_SECONDS=0 +[ "${#DEPLOY_CHECK_TIMEOUT_SECONDS}" -le 3 ] || DEPLOY_CHECK_TIMEOUT_SECONDS=900 +[ "${#DEPLOY_CHECK_POLL_SECONDS}" -le 2 ] || DEPLOY_CHECK_POLL_SECONDS=60 +[ "$DEPLOY_CHECK_TIMEOUT_SECONDS" -le 900 ] || DEPLOY_CHECK_TIMEOUT_SECONDS=900 +[ "$DEPLOY_CHECK_POLL_SECONDS" -le 60 ] || DEPLOY_CHECK_POLL_SECONDS=60 +[ "$DEPLOY_CHECK_POLL_SECONDS" -gt 0 ] || DEPLOY_CHECK_POLL_SECONDS=1 + +wait_for_compose_runtime() { + required_services='api web db redis minio' + migration_declared=0 + compose_has_service migrate && migration_declared=1 + runtime_report='' + runtime_pending=0 + runtime_failure=0 + timed_out=0 + deadline=$(( $(date +%s) + DEPLOY_CHECK_TIMEOUT_SECONDS )) + + while :; do + runtime_report='' + runtime_pending=0 + runtime_failure=0 + for runtime_service in $required_services; do + if compose_has_service "$runtime_service"; then + probe_service "$runtime_service" runtime + else + PROBE_STATE=failed + PROBE_DETAIL='service is not defined in selected Compose configuration' + fi + runtime_report="${runtime_report}${runtime_service}|${PROBE_STATE}|${PROBE_DETAIL} +" + [ "$PROBE_STATE" = pending ] && runtime_pending=1 + [ "$PROBE_STATE" = failed ] && runtime_failure=1 + done + + if [ "$migration_declared" -eq 1 ]; then + probe_service migrate migration + runtime_report="${runtime_report}migrate|${PROBE_STATE}|${PROBE_DETAIL} +" + [ "$PROBE_STATE" = pending ] && runtime_pending=1 + [ "$PROBE_STATE" = failed ] && runtime_failure=1 + fi + + [ "$runtime_failure" -eq 1 ] && break + [ "$runtime_pending" -eq 0 ] && break + if [ "$(date +%s)" -ge "$deadline" ]; then + timed_out=1 + break + fi + sleep "$DEPLOY_CHECK_POLL_SECONDS" + done + + runtime_passed=1 + while IFS='|' read -r runtime_service outcome detail; do + [ -n "$runtime_service" ] || continue + if [ "$outcome" = ready ]; then + echo "[PASS] runtime-$runtime_service" + else + runtime_passed=0 + if [ "$outcome" = pending ] && [ "$timed_out" -eq 1 ]; then + detail="timed out waiting: $detail" + elif [ "$outcome" = pending ]; then + detail="not ready: $detail" + fi + echo "[FAIL] runtime-$runtime_service - $detail" + fi + done < "$RESULTS" < pathEntries.map((entry) => path.join(entry, command)).find((candidate) => existsSync(candidate)); +const powershell = process.env.PWSH_PATH || findOnPath(process.platform === 'win32' ? 'pwsh.exe' : 'pwsh') || + (process.platform === 'win32' ? findOnPath('powershell.exe') : undefined); +const powershellProbe = powershell ? spawnSync(powershell, ['-NoProfile', '-Command', 'exit 0']) : undefined; +const hasPowerShell = Boolean(powershell && !powershellProbe.error && powershellProbe.status === 0); + +const fakeDocker = String.raw`#!/usr/bin/env node +import fs from 'node:fs'; +const args = process.argv.slice(2); +const scenario = process.env.FAKE_DOCKER_CASE || 'passing'; +if (process.env.FAKE_DOCKER_LOG) fs.appendFileSync(process.env.FAKE_DOCKER_LOG, JSON.stringify(args) + '\n'); +if (args[0] === 'version') process.exit(0); +if (args[0] === 'inspect') { + const id = args[args.length - 1]; + const service = id.replace(/^id-/, ''); + let status = 'running'; + let health = 'healthy'; + let code = '0'; + if (scenario === 'exited' && service === 'api') { status = 'exited'; health = 'none'; code = '1'; } + if (scenario === 'starting' && service === 'api') health = 'starting'; + if (scenario === 'starting-then-healthy' && service === 'api') { + const log = process.env.FAKE_DOCKER_LOG; + const priorCalls = log && fs.existsSync(log) + ? fs.readFileSync(log, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line)) + .filter((call) => call[0] === 'inspect' && call[call.length - 1] === 'id-api').length + : 0; + if (priorCalls === 1) health = 'starting'; + } + if (scenario === 'unhealthy' && service === 'db') health = 'unhealthy'; + if (service === 'migrate') { + status = scenario === 'migration-running' ? 'running' : 'exited'; + health = 'none'; + code = scenario === 'migration-failed' ? '7' : '0'; + } + process.stdout.write([status, health, code].join('|') + '\n'); + process.exit(0); +} +if (args[0] === 'compose') { + const composeArgs = args.slice(1); + if (composeArgs.includes('config')) { + if (composeArgs.includes('--services')) { + let services = (process.env.FAKE_COMPOSE_SERVICES || 'api,web,db,redis,minio,migrate,nginx').split(','); + if (scenario === 'missing-config-service') services = services.filter((name) => name !== 'api'); + process.stdout.write(services.join('\n') + '\n'); + } + process.exit(0); + } + if (composeArgs.includes('ps')) { + const q = composeArgs.indexOf('-q'); + const service = q >= 0 ? composeArgs[q + 1] : ''; + if (((scenario === 'missing-service' && service === 'api') || (scenario === 'migration-missing' && service === 'migrate')) || !service || service === 'nginx') process.exit(0); + process.stdout.write('id-' + service + '\n'); + process.exit(0); + } +} +process.stderr.write('unexpected fake docker arguments\n'); +process.exit(2); +`; + +function fixture() { + const root = mkdtempSync(path.join(tmpdir(), 'oneerp-deploy-check-')); + const scripts = path.join(root, 'scripts'); + const bin = path.join(root, 'bin'); + mkdirSync(scripts); + mkdirSync(bin); + copyFileSync(path.join(repoRoot, 'scripts', 'deploy-check.sh'), path.join(scripts, 'deploy-check.sh')); + copyFileSync(path.join(repoRoot, 'scripts', 'deploy-check.ps1'), path.join(scripts, 'deploy-check.ps1')); + writeFileSync(path.join(root, 'selected-compose.yml'), 'services: {}\n'); + writeFileSync(path.join(root, 'docker-compose.ha-lite.yml'), 'services: {}\n'); + const envFile = path.join(root, 'selected.env'); + const secretValues = [ + 'pg-secret-value-for-test-0001', + 'jwt-secret-value-for-test-0002', + 'minio-secret-value-for-test-003', + 'admin-secret-value-for-test-004', + ]; + writeFileSync(envFile, [ + `POSTGRES_PASSWORD=${secretValues[0]}`, + `JWT_SECRET=${secretValues[1]}`, + `MINIO_SECRET_KEY=${secretValues[2]}`, + `INIT_ADMIN_PASSWORD=${secretValues[3]}`, + 'API_PORT=8000', + 'WEB_PORT=3000', + ].join('\n') + '\n'); + copyFileSync(envFile, path.join(root, '.env')); + const fakeDockerPath = path.join(bin, 'docker.mjs'); + writeFileSync(fakeDockerPath, fakeDocker); + if (process.platform === 'win32') { + writeFileSync(path.join(bin, 'docker.cmd'), '@echo off\r\nnode "%~dp0docker.mjs" %*\r\n'); + } else { + const dockerPath = path.join(bin, 'docker'); + writeFileSync(dockerPath, '#!/bin/sh\nexec node "$(dirname "$0")/docker.mjs" "$@"\n'); + chmodSync(dockerPath, 0o755); + const curlPath = path.join(bin, 'curl'); + writeFileSync(curlPath, '#!/bin/sh\nexit 0\n'); + chmodSync(curlPath, 0o755); + } + return { root, bin, envFile, composeFile: path.join(root, 'selected-compose.yml'), secretValues }; +} + +function runShell(fx, scenario, timeoutSeconds = 0) { + const log = path.join(fx.root, 'docker-args.jsonl'); + const result = spawnSync('sh', [path.join(fx.root, 'scripts', 'deploy-check.sh')], { + cwd: fx.root, + encoding: 'utf8', + env: { + ...process.env, + PATH: `${fx.bin}${path.delimiter}${process.env.PATH || ''}`, + COMPOSE_FILE: 'selected-compose.yml', + ENV_FILE: 'selected.env', + DEPLOY_CHECK_TIMEOUT_SECONDS: String(timeoutSeconds), + DEPLOY_CHECK_POLL_SECONDS: '1', + FAKE_DOCKER_CASE: scenario, + FAKE_DOCKER_LOG: log, + }, + }); + return { ...result, log, report: readFileSync(path.join(fx.root, 'deploy-check-report.json'), 'utf8') }; +} + +function runPowerShell(fx, scenario, timeoutSeconds = 0, selectedFiles = true) { + const log = path.join(fx.root, 'docker-args.jsonl'); + const script = path.join(fx.root, 'scripts', 'deploy-check.ps1').replaceAll("'", "''"); + const command = `function Invoke-WebRequest { param($Uri, [switch]$UseBasicParsing, $TimeoutSec) [pscustomobject]@{ StatusCode = 200 } }; & '${script}' -TimeoutSeconds ${timeoutSeconds}`; + const env = { + ...process.env, + PATH: `${fx.bin}${path.delimiter}${process.env.PATH || ''}`, + COMPOSE_FILE: 'selected-compose.yml', + ENV_FILE: 'selected.env', + FAKE_DOCKER_CASE: scenario, + FAKE_DOCKER_LOG: log, + }; + if (!selectedFiles) { + delete env.COMPOSE_FILE; + delete env.ENV_FILE; + delete env.API_BASE_URL; + delete env.WEB_BASE_URL; + } + const result = spawnSync(powershell, ['-NoProfile', '-Command', command], { + cwd: fx.root, + encoding: 'utf8', + env, + }); + return { ...result, log, report: readFileSync(path.join(fx.root, 'deploy-check-report.json'), 'utf8') }; +} + +function assertNoSecretLeak(result, secretValues) { + const combined = `${result.stdout || ''}\n${result.stderr || ''}\n${result.report}`; + for (const value of secretValues) assert.equal(combined.includes(value), false, 'secret value was emitted'); +} + +const scenarios = [ + { name: 'missing service container', scenario: 'missing-service', output: 'runtime-api', detail: 'not present' }, + { name: 'service exited', scenario: 'exited', output: 'runtime-api', detail: 'status=exited' }, + { name: 'health still starting at the deadline', scenario: 'starting', output: 'runtime-api', detail: 'timed out' }, + { name: 'service unhealthy', scenario: 'unhealthy', output: 'runtime-db', detail: 'unhealthy' }, + { name: 'migration exit code is nonzero', scenario: 'migration-failed', output: 'runtime-migrate', detail: 'exitCode=7' }, + { name: 'declared migration container is missing', scenario: 'migration-missing', output: 'runtime-migrate', detail: 'not present' }, + { name: 'declared migration is still running at the deadline', scenario: 'migration-running', output: 'runtime-migrate', detail: 'status=running' }, + { name: 'required service missing from selected config', scenario: 'missing-config-service', output: 'runtime-api', detail: 'not defined' }, +]; + +for (const { name, scenario, output, detail } of scenarios) { + test(`deploy-check.sh fails when ${name}`, (t) => { + const fx = fixture(); + t.after(() => rmSync(fx.root, { recursive: true, force: true })); + const result = runShell(fx, scenario); + assert.notEqual(result.status, 0, result.stdout); + assert.match(result.stdout, new RegExp(`\\[FAIL\\] ${output}`)); + assert.match(result.stdout, new RegExp(detail)); + assertNoSecretLeak(result, fx.secretValues); + }); + + test(`deploy-check.ps1 fails when ${name}`, { skip: !hasPowerShell }, (t) => { + const fx = fixture(); + t.after(() => rmSync(fx.root, { recursive: true, force: true })); + const result = runPowerShell(fx, scenario); + assert.notEqual(result.status, 0, result.stdout); + assert.match(result.stdout, new RegExp(`\\[FAIL\\] runtime-${output.replace('runtime-', '')}`)); + assert.match(result.stdout, new RegExp(detail)); + assertNoSecretLeak(result, fx.secretValues); + }); +} + +test('deploy-check.sh passes for healthy runtime services and a successful one-shot migration', (t) => { + const fx = fixture(); + t.after(() => rmSync(fx.root, { recursive: true, force: true })); + const result = runShell(fx, 'passing'); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.match(result.stdout, /\[PASS\] runtime-api/); + assert.match(result.stdout, /\[PASS\] runtime-migrate/); + assert.match(result.report, /"failed": 0/); + const calls = readFileSync(result.log, 'utf8').trim().split('\n').map((line) => JSON.parse(line)); + assert.ok(calls.some((args) => args.includes('--env-file') && args.includes(fx.envFile))); + assert.ok(calls.some((args) => args.includes('-f') && args.includes(fx.composeFile))); + assert.ok(calls.some((args) => args.includes('--all') && args.includes('-q'))); + assert.equal(calls.some((args) => args.includes('ps') && args.includes('nginx')), false, 'profile-only nginx must not be required'); + assertNoSecretLeak(result, fx.secretValues); +}); + +test('deploy-check.ps1 passes for healthy runtime services and a successful one-shot migration', { skip: !hasPowerShell }, (t) => { + const fx = fixture(); + t.after(() => rmSync(fx.root, { recursive: true, force: true })); + const result = runPowerShell(fx, 'passing'); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.match(result.stdout, /\[PASS\] runtime-api/); + assert.match(result.stdout, /\[PASS\] runtime-migrate/); + const calls = readFileSync(result.log, 'utf8').trim().split('\n').map((line) => JSON.parse(line)); + assert.ok(calls.some((args) => args.includes('--env-file') && args.includes(fx.envFile))); + assert.ok(calls.some((args) => args.includes('-f') && args.includes(fx.composeFile))); + assert.ok(calls.some((args) => args.includes('--all') && args.includes('-q'))); + assert.equal(calls.some((args) => args.includes('ps') && args.includes('nginx')), false, 'profile-only nginx must not be required'); + assertNoSecretLeak(result, fx.secretValues); +}); + +test('deploy-check.sh waits for health starting to become healthy', (t) => { + const fx = fixture(); + t.after(() => rmSync(fx.root, { recursive: true, force: true })); + const result = runShell(fx, 'starting-then-healthy', 5); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.match(result.stdout, /\[PASS\] runtime-api/); +}); + +test('deploy-check.ps1 waits for health starting to become healthy', { skip: !hasPowerShell }, (t) => { + const fx = fixture(); + t.after(() => rmSync(fx.root, { recursive: true, force: true })); + const result = runPowerShell(fx, 'starting-then-healthy', 5); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.match(result.stdout, /\[PASS\] runtime-api/); +}); + +test('deploy-check.ps1 uses HA-lite and .env defaults when Compose env vars are unset', { skip: !hasPowerShell }, (t) => { + const fx = fixture(); + t.after(() => rmSync(fx.root, { recursive: true, force: true })); + const result = runPowerShell(fx, 'passing', 0, false); + assert.equal(result.status, 0, result.stdout + result.stderr); + const calls = readFileSync(result.log, 'utf8').trim().split('\n').map((line) => JSON.parse(line)); + assert.ok(calls.some((args) => args.includes('-f') && args.includes(path.join(fx.root, 'docker-compose.ha-lite.yml')))); + assert.ok(calls.some((args) => args.includes('--env-file') && args.includes(path.join(fx.root, '.env')))); +});