From b127e6fff030cdab79c388161618426345d78400 Mon Sep 17 00:00:00 2001
From: R <53855466+cb8010d6@users.noreply.github.com>
Date: Sun, 27 Sep 2026 11:08:29 +0800
Subject: [PATCH 1/3] ci: verify shipping containers and isolated data recovery
---
.github/workflows/ci.yml | 38 ++-
docs/CI_CONTAINER_REHEARSAL.md | 32 +++
docs/CONTROLLED_TRIAL_DELIVERY.md | 28 +++
package.json | 2 +-
scripts/ci/compose.rehearsal.yml | 134 +++++++++++
scripts/ci/container-rehearsal.mjs | 306 ++++++++++++++++++++++++
scripts/ci/container-rehearsal.test.mjs | 153 ++++++++++++
scripts/ci/recovery-rehearsal.md | 40 ++++
scripts/ci/recovery-rehearsal.mjs | 164 +++++++++++++
scripts/ci/recovery-rehearsal.test.mjs | 83 +++++++
10 files changed, 976 insertions(+), 4 deletions(-)
create mode 100644 docs/CI_CONTAINER_REHEARSAL.md
create mode 100644 docs/CONTROLLED_TRIAL_DELIVERY.md
create mode 100644 scripts/ci/compose.rehearsal.yml
create mode 100644 scripts/ci/container-rehearsal.mjs
create mode 100644 scripts/ci/container-rehearsal.test.mjs
create mode 100644 scripts/ci/recovery-rehearsal.md
create mode 100644 scripts/ci/recovery-rehearsal.mjs
create mode 100644 scripts/ci/recovery-rehearsal.test.mjs
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 41a3f04..f4effb0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -340,16 +340,48 @@ jobs:
if-no-files-found: warn
retention-days: 7
+ # Build actual shipping Docker targets and recover disposable synthetic data.
+ # No registry publication, deployment credentials or production environment.
+ container-rehearsal:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ permissions:
+ contents: read
+ env:
+ ONEERP_CONTAINER_REHEARSAL: '1'
+ CONTAINER_REHEARSAL_REPORT: ${{ runner.temp }}/container-rehearsal-report.json
+ steps:
+ - name: Checkout exact source revision
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.sha }}
+ persist-credentials: false
+ - name: Setup Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: 22
+ - name: Build, start and recover isolated application containers
+ run: node scripts/ci/container-rehearsal.mjs
+ - name: Upload sanitized container rehearsal evidence
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: container-rehearsal-report
+ path: ${{ runner.temp }}/container-rehearsal-report.json
+ if-no-files-found: error
+ retention-days: 7
+
# Retain the existing required check name and require ALL independent jobs.
validate:
if: always()
- needs: [security-audit, validation-tests, http-journey, browser-workspace]
+ needs: [security-audit, validation-tests, http-journey, browser-workspace, container-rehearsal]
runs-on: ubuntu-latest
steps:
- - name: Require security, validation, HTTP and browser acceptance success
+ - name: Require security, validation, HTTP, browser and container recovery success
env:
SECURITY_RESULT: ${{ needs.security-audit.result }}
VALIDATION_RESULT: ${{ needs.validation-tests.result }}
HTTP_RESULT: ${{ needs.http-journey.result }}
BROWSER_RESULT: ${{ needs.browser-workspace.result }}
- run: test "$SECURITY_RESULT" = success && test "$VALIDATION_RESULT" = success && test "$HTTP_RESULT" = success && test "$BROWSER_RESULT" = success
+ CONTAINER_RESULT: ${{ needs.container-rehearsal.result }}
+ run: test "$SECURITY_RESULT" = success && test "$VALIDATION_RESULT" = success && test "$HTTP_RESULT" = success && test "$BROWSER_RESULT" = success && test "$CONTAINER_RESULT" = success
diff --git a/docs/CI_CONTAINER_REHEARSAL.md b/docs/CI_CONTAINER_REHEARSAL.md
new file mode 100644
index 0000000..9e277a1
--- /dev/null
+++ b/docs/CI_CONTAINER_REHEARSAL.md
@@ -0,0 +1,32 @@
+# 当前提交容器交付演练 / Current-head container rehearsal
+
+`scripts/ci/container-rehearsal.mjs` 在 GitHub Actions 的一次性 Linux runner 上，使用当前干净 checkout 的现有 Dockerfile 构建 API `production`、API `builder`（migration/init）及 Web `production` 镜像。它不推送镜像、不调用 SSH、不部署生产、不合并分支。
+
+运行入口（只在一次性 GitHub Actions runner 执行）：
+
+```bash
+CI=true GITHUB_ACTIONS=true ONEERP_CONTAINER_REHEARSAL=1 \
+ CONTAINER_REHEARSAL_REPORT="$RUNNER_TEMP/container-rehearsal-report.json" \
+ node scripts/ci/container-rehearsal.mjs
+```
+
+工作流应 checkout PR 的精确 head，授予 `contents: read`，设置 60 分钟 job 超时，并把该 job 纳入最终 `validate` 必须成功的依赖。脚本不需要主机 npm 安装；HTTP 验收脚本只使用 Node 22 内置 API。
+
+## 实际验证边界
+
+1. 记录 `git rev-parse HEAD`、tree 和实际本地 image ID；开始与结束都要求 source checkout 干净。
+2. 顺序构建三个应用镜像以及现有 `scripts/ci/minio.Dockerfile` 的 CI 专用固定源码 MinIO 镜像。MinIO **不是生产镜像替代方案，未解决 #35**。
+3. 用随机 `oneerp-ci-<24 hex>` 项目创建 PostgreSQL、Redis、MinIO 私有网络和卷。仅 API/Web 使用随机回环端口。依赖 readiness 通过后，实际执行生产 migration/init 镜像。
+4. API/Web 使用生产镜像启动；验证运行镜像 ID，读取 Web `/login` 及其引用的静态资源，验证 Web API proxy health 和登录。库存贸易 13 步与制造 12 步通过真实 HTTP 容器边界执行。
+5. 调用 `recovery-rehearsal.mjs`：备份合成 PostgreSQL 和上传对象，使用同一组实际运行 image ID 在独立空卷中恢复；比较业务行、文件身份和下载字节。恢复不重新迁移或初始化。
+6. 默认删除本次随机项目的容器、网络和卷，删除临时凭据/原始验收报告；仅输出允许字段构成的 JSON 证据。清理失败也使门禁失败。
+
+脚本拒绝缺失显式 CI 开关、非默认 Docker host/context、非本地 runner socket、脏 checkout 及非回环发布端口。随机密码通过 GitHub mask 注册，临时 `.env` 权限为 `0600`。不要上传临时目录、Docker config/inspect 全文、容器日志、SQL、归档、signed URL 或 token。工作流只上传 `container-rehearsal-report.json`。
+
+## 资源与证据解释
+
+标准 Ubuntu runner 顺序构建，运行时每个 stack 内存上限合计约 3 GiB（一次性 migration 上限 1 GiB；恢复阶段最多两个 stack）。构建、镜像和 BuildKit 缓存需要额外内存/磁盘；预留约 10–15 GB 磁盘作为初始预算，实际消耗以 runner 为准。每个命令有超时，job 的 60 分钟限额是最终上界。没有全局 prune、共享卷删除或镜像发布；runner 销毁负责最终镜像/缓存回收。
+
+本地无 Docker daemon 时，`node --test scripts/ci/container-rehearsal.test.mjs` 只证明命令编排、防误用、清理和证据脱敏；它不证明镜像能构建或服务能启动。只有精确提交的 GitHub 容器 job 通过后，才能报告该提交的真实构建、25 步 HTTP 与合成恢复已通过。证据 JSON 中的本地 image ID 是本次构建身份，不是可供生产拉取的 registry digest。
+
+该结果仍不代替目标生产环境的 MinIO 分发治理、TLS/唯一凭据、异地备份、生产 RPO/RTO、真实数据与角色验收、库存/财务/运维签字。生产批准继续使用 `PRODUCTION_READINESS.md`、`HA_LITE_RUNBOOK.md` 与 `GO_LIVE_CHECKLIST.md`。
diff --git a/docs/CONTROLLED_TRIAL_DELIVERY.md b/docs/CONTROLLED_TRIAL_DELIVERY.md
new file mode 100644
index 0000000..3dc9e4b
--- /dev/null
+++ b/docs/CONTROLLED_TRIAL_DELIVERY.md
@@ -0,0 +1,28 @@
+# 受控试运行交付
+
+本轮把当前源代码的容器构建、业务验收与隔离恢复接到同一个 CI 门禁。交付状态以对应 PR 精确 head 的运行记录为准；本文件不是生产上线批准。
+
+## 验收入口
+
+GitHub Actions 的 `CI / container-rehearsal` 从 PR head 构建现有 API production、API builder（迁移）和 Web production Docker target。所有服务及恢复副本都使用随机的 `oneerp-ci-*` 项目和临时凭据；端口只绑定 runner 的 loopback。
+
+执行顺序：构建镜像 → 初始化空数据库 → 启动 API/Web → 检查登录页、静态资源和 API 代理 → 13 步库存贸易与 12 步制造 HTTP 链路 → 备份 PostgreSQL/MinIO → 在新卷中恢复并验证。恢复复用正在运行的镜像 ID，不重新执行管理员初始化，不覆盖源数据。
+
+下载该次运行的 `container-rehearsal-report` artifact，可核对源提交、源码树、镜像 ID、各步骤状态、恢复表记录数和文件校验结果。任何失败都阻断 aggregate validate。模拟 Docker 的脚本测试不等于真实容器运行通过。
+
+此演练仅允许 GitHub Actions 的一次性环境，不能把 CI 标志加到生产服务器后运行。临时数据库、归档、凭据和完整容器日志不作为 artifact 发布。
+
+## 集成和部署边界
+
+- 保留现有审查栈的提交历史；通过精确 head 验证后，再固定独立的 `release/controlled-trial-*` 分支作为试运行基线。
+- 主分支推送可能触发镜像发布和服务器部署。创建试运行分支与 Draft PR 不代表已合并或已部署。
+- CI 的 MinIO 源码构建仅用于测试；生产分发问题 #35 仍需可访问、经过版本治理的生产镜像。不能把这份旧版 CI 镜像直接发布为生产方案。
+- 当前恢复演练使用合成数据，证明的是当前镜像及数据库/对象文件恢复链路；真实目标的 TLS、异地备份、恢复时长与业务签字仍须独立验证。
+- 已有数据库的切换必须先停止写入并排空旧事件消费者，尤其是 #41 会计事件来源字段变更。不能在旧消费者运行时直接迁移，再用新版本重放历史事件。
+- 回退应同时固定应用镜像、Compose/脚本版本、数据库兼容性与备份。仅切换旧 image tag 不能证明数据库可回退。
+
+## 实际环境接入所需信息
+
+目标环境地址与现有访问方式、部署开关及环境保护状态、生产对象存储镜像/服务选择，以及负责验收的业务角色。未能通过当前工具读取 GitHub Secrets/Variables，不等于这些配置不存在；不得为了验证其存在而打印凭据或触发部署。
+
+接入之后应先在隔离试运行目标验证当前固定版本，再按业务清单验收采购、销售、库存、制造、工程文档和财务边界。生产切换需保留停写、备份、恢复与回退记录。
diff --git a/package.json b/package.json
index ddba18b..d14246b 100644
--- a/package.json
+++ b/package.json
@@ -20,7 +20,7 @@
"commitlint": "commitlint --config commitlint.config.cjs --from=HEAD~1 --to=HEAD",
"typecheck": "npm --prefix apps/api run typecheck && npm --prefix apps/web run typecheck",
"lint": "npm --prefix apps/api run lint && npm --prefix apps/web run lint",
- "test": "node --test scripts/*.test.mjs && npm --prefix apps/api run test -- --passWithNoTests && npm --prefix apps/web run test",
+ "test": "node --test scripts/*.test.mjs scripts/ci/*.test.mjs && npm --prefix apps/api run test -- --passWithNoTests && npm --prefix apps/web run test",
"build": "npm --prefix apps/api run build && npm --prefix apps/web run build",
"validate": "node scripts/prisma-static-check.mjs && npm run typecheck && npm run lint && npm run test && npm run build",
"audit:security": "npm audit --audit-level=high && npm --prefix apps/api audit --audit-level=high && npm --prefix apps/web audit --audit-level=high && npm --prefix apps/mobile audit --audit-level=high",
diff --git a/scripts/ci/compose.rehearsal.yml b/scripts/ci/compose.rehearsal.yml
new file mode 100644
index 0000000..733363e
--- /dev/null
+++ b/scripts/ci/compose.rehearsal.yml
@@ -0,0 +1,134 @@
+# Disposable GitHub CI only. The production Compose file and MinIO policy are
+# deliberately unchanged; this uses the existing verified-source CI MinIO build.
+# All images are built locally by container-rehearsal.mjs. Never publish this stack.
+x-runtime: &runtime
+ restart: "no"
+ logging:
+ driver: json-file
+ options:
+ max-size: "5m"
+ max-file: "2"
+
+services:
+ db:
+ <<: *runtime
+ image: postgres:15-alpine
+ mem_limit: 512m
+ environment:
+ POSTGRES_USER: ${POSTGRES_USER:?required}
+ POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?required}
+ POSTGRES_DB: ${POSTGRES_DB:?required}
+ volumes:
+ - pgdata:/var/lib/postgresql/data
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
+ interval: 3s
+ timeout: 3s
+ retries: 30
+
+ redis:
+ <<: *runtime
+ image: redis:7-alpine
+ mem_limit: 192m
+ command: redis-server --appendonly yes --save 60 1000
+ volumes:
+ - redisdata:/data
+ healthcheck:
+ test: ["CMD", "redis-cli", "ping"]
+ interval: 3s
+ timeout: 3s
+ retries: 30
+
+ minio:
+ <<: *runtime
+ image: ${REHEARSAL_MINIO_IMAGE:?required}
+ pull_policy: never
+ mem_limit: 768m
+ environment:
+ MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:?required}
+ MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:?required}
+ command: server /data
+ volumes:
+ - miniodata:/data
+ # The scratch CI image intentionally has no shell/curl/mc. The orchestrator
+ # probes its HTTP readiness from the API image on this private network.
+
+ migrate:
+ <<: *runtime
+ image: ${REHEARSAL_MIGRATE_IMAGE:?required}
+ pull_policy: never
+ mem_limit: 1g
+ depends_on:
+ db:
+ condition: service_healthy
+ environment:
+ DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
+ INIT_ADMIN_EMAIL: ${INIT_ADMIN_EMAIL:?required}
+ INIT_ADMIN_PASSWORD: ${INIT_ADMIN_PASSWORD:?required}
+ INIT_ADMIN_NAME: Container Rehearsal
+ INIT_COMPANY_NAME: ${INIT_COMPANY_NAME:?required}
+ command: ["sh", "-c", "npx --no-install prisma migrate deploy --schema=./prisma/schema.prisma && npm run init:prod"]
+
+ api:
+ <<: *runtime
+ image: ${REHEARSAL_API_IMAGE:?required}
+ pull_policy: never
+ mem_limit: 1g
+ depends_on:
+ db:
+ condition: service_healthy
+ redis:
+ condition: service_healthy
+ minio:
+ condition: service_started
+ environment:
+ NODE_ENV: production
+ DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
+ POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+ JWT_SECRET: ${JWT_SECRET:?required}
+ CORS_ORIGINS: http://127.0.0.1
+ REDIS_HOST: redis
+ REDIS_PORT: "6379"
+ MINIO_ENDPOINT: minio
+ MINIO_PORT: "9000"
+ MINIO_USE_SSL: "false"
+ MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+ MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+ AI_WRITE_ENABLED: "false"
+ PORT: "8000"
+ ports:
+ - "127.0.0.1::8000"
+ healthcheck:
+ test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:8000/api/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"]
+ interval: 3s
+ timeout: 3s
+ retries: 40
+ start_period: 15s
+
+ web:
+ <<: *runtime
+ image: ${REHEARSAL_WEB_IMAGE:?required}
+ pull_policy: never
+ mem_limit: 512m
+ depends_on:
+ api:
+ condition: service_healthy
+ environment:
+ NODE_ENV: production
+ PORT: "3000"
+ HOSTNAME: 0.0.0.0
+ NEXT_PUBLIC_API_BASE_URL: /api/proxy
+ API_BASE_URL: http://api:8000
+ ports:
+ - "127.0.0.1::3000"
+ healthcheck:
+ test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3000/login').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"]
+ interval: 3s
+ timeout: 3s
+ retries: 40
+ start_period: 10s
+
+volumes:
+ pgdata:
+ redisdata:
+ miniodata:
diff --git a/scripts/ci/container-rehearsal.mjs b/scripts/ci/container-rehearsal.mjs
new file mode 100644
index 0000000..22ca26b
--- /dev/null
+++ b/scripts/ci/container-rehearsal.mjs
@@ -0,0 +1,306 @@
+#!/usr/bin/env node
+import { randomBytes } from 'node:crypto';
+import { spawn } from 'node:child_process';
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
+const PROJECT_PATTERN = /^oneerp-ci-[a-f0-9]{24}(?:-restore)?$/;
+const IMAGE_ID_PATTERN = /^sha256:[a-f0-9]{64}$/;
+const assert = (condition, message) => { if (!condition) throw new Error(message); };
+
+export function requireCi(env) {
+ assert(env.CI === 'true' && env.GITHUB_ACTIONS === 'true' && env.ONEERP_CONTAINER_REHEARSAL === '1',
+ 'Disposable rehearsal requires CI=true, GITHUB_ACTIONS=true and ONEERP_CONTAINER_REHEARSAL=1');
+ assert(!env.DOCKER_HOST && (!env.DOCKER_CONTEXT || env.DOCKER_CONTEXT === 'default'),
+ 'Disposable rehearsal refuses an overridden Docker host or context');
+}
+
+export function composeArgs({ root, composeFile, envFile, project }, args) {
+ assert(PROJECT_PATTERN.test(project), 'Refusing a non-rehearsal Compose project');
+ return ['compose', '--project-directory', root, '--env-file', envFile,
+ '--file', composeFile, '--project-name', project, ...args];
+}
+
+export function loopbackPort(value) {
+ const match = /^127\.0\.0\.1:(\d+)$/.exec(value.trim());
+ const port = Number(match?.[1]);
+ assert(match && port >= 1 && port <= 65535, 'Compose did not publish exactly one loopback port');
+ return `http://127.0.0.1:${port}`;
+}
+
+export function summarizeJourney(report, expectedSteps) {
+ assert(report?.passed === true && Array.isArray(report.steps)
+ && report.steps.length === expectedSteps && report.steps.every((step) => step.passed === true),
+ 'HTTP journey did not pass its complete expected step set');
+ return { passed: true, passedSteps: report.steps.length, expectedSteps };
+}
+
+function summarizeRecovery(result) {
+ return { status: result.status,
+ checks: result.checks?.map(({ name, status }) => ({ name, status })),
+ verifiedFileCount: result.verifiedFileCount,
+ objectEvidenceDigest: result.objectEvidenceDigest,
+ businessTables: result.businessTables?.map(({ table, count }) => ({ table, count })),
+ };
+}
+
+// No shell, secret-bearing command text, or unbounded process output. Diagnostics
+// stay in memory and are never included in the public evidence artifact.
+export function runCommand(command, args, { cwd = ROOT, env = process.env,
+ timeoutMs = 120_000, signal, onFailureOutput } = {}) {
+ return new Promise((resolve, reject) => {
+ const child = spawn(command, args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], signal });
+ let output = '';
+ let diagnostics = '';
+ let timedOut = false;
+ let killTimer;
+ const timer = setTimeout(() => {
+ timedOut = true;
+ child.kill('SIGTERM');
+ killTimer = setTimeout(() => child.kill('SIGKILL'), 5_000);
+ }, timeoutMs);
+ const collect = (chunk) => {
+ output = `${output}${chunk}`.slice(-131_072);
+ diagnostics = `${diagnostics}${chunk}`.slice(-131_072);
+ };
+ child.stdout.on('data', collect);
+ // Do not mix stderr with machine-readable stdout.
+ child.stderr.on('data', (chunk) => { diagnostics = `${diagnostics}${chunk}`.slice(-131_072); });
+ child.once('error', (error) => { clearTimeout(timer); clearTimeout(killTimer); reject(error); });
+ child.once('close', (code) => {
+ clearTimeout(timer);
+ clearTimeout(killTimer);
+ if (timedOut || code !== 0) {
+ onFailureOutput?.(diagnostics);
+ reject(new Error(timedOut ? 'Command timed out' : `Command exited ${code}`));
+ } else resolve(output.trim());
+ });
+ });
+}
+
+export async function checkWeb(webBaseUrl, credentials, companyName, fetchImpl = fetch) {
+ const request = async (route, options = {}) => {
+ const response = await fetchImpl(`${webBaseUrl}${route}`, {
+ ...options, redirect: 'error', signal: AbortSignal.timeout(15_000),
+ });
+ assert(response.ok, 'Web or its API proxy returned a non-success response');
+ return response;
+ };
+ const page = await request('/login');
+ assert(page.headers.get('content-type')?.includes('text/html'), 'Web login did not return HTML');
+ const html = await page.text();
+ const assets = [...new Set([...html.matchAll(/(?:src|href)=["'](\/_next\/static\/[^"']+)["']/g)]
+ .map((match) => match[1]))];
+ assert(assets.length > 0 && assets.length <= 64 && assets.some((asset) => /\.js(?:\?|$)/.test(asset)),
+ 'Web login did not reference a bounded set of production JavaScript assets');
+ for (const asset of assets) {
+ const url = new URL(asset, webBaseUrl);
+ assert(url.origin === webBaseUrl && url.pathname.startsWith('/_next/static/'), 'Invalid Web static asset path');
+ const response = await request(`${url.pathname}${url.search}`);
+ assert(!response.headers.get('content-type')?.includes('text/html')
+ && (await response.arrayBuffer()).byteLength > 0, 'Web static asset was missing or returned HTML');
+ }
+ const health = await (await request('/api/proxy/health')).json();
+ assert(health.status === 'ok', 'Web proxy health did not reach the API');
+ const login = await (await request('/api/proxy/auth/login', {
+ method: 'POST', headers: { 'content-type': 'application/json' },
+ body: JSON.stringify(credentials),
+ })).json();
+ assert(typeof login.accessToken === 'string' && login.accessToken.length > 0
+ && login.companies?.filter((company) => company.name === companyName).length === 1,
+ 'Web proxy login did not select the synthetic company');
+ return { loginPage: 'passed', staticAssets: assets.length, apiHealthProxy: 'passed', loginProxy: 'passed' };
+}
+
+export async function runRehearsal({ root = ROOT, env = process.env, runner = runCommand,
+ fetchImpl = fetch, recover, log = console.log, signal } = {}) {
+ requireCi(env);
+ const reportPath = path.resolve(env.CONTAINER_REHEARSAL_REPORT
+ || path.join(env.RUNNER_TEMP || tmpdir(), 'container-rehearsal-report.json'));
+ const tempDir = await mkdtemp(path.join(env.RUNNER_TEMP || tmpdir(), 'oneerp-container-'));
+ const nonce = randomBytes(12).toString('hex');
+ const sourceProject = `oneerp-ci-${nonce}`;
+ const recoveryProject = `${sourceProject}-restore`;
+ const composeFile = path.join(root, 'scripts/ci/compose.rehearsal.yml');
+ const envFile = path.join(tempDir, 'rehearsal.env');
+ const imageTags = Object.fromEntries(['api', 'migrate', 'web', 'minio'].map((name) => [name, `${sourceProject}-${name}:local`]));
+ const generated = {
+ POSTGRES_USER: 'oneerp_ci', POSTGRES_DB: 'oneerp_ci', POSTGRES_PASSWORD: randomBytes(32).toString('hex'),
+ JWT_SECRET: randomBytes(48).toString('hex'), MINIO_ACCESS_KEY: `ci${randomBytes(12).toString('hex')}`,
+ MINIO_SECRET_KEY: randomBytes(32).toString('hex'), INIT_ADMIN_EMAIL: `${sourceProject}@example.invalid`,
+ INIT_ADMIN_PASSWORD: `Ci!${randomBytes(32).toString('hex')}`,
+ INIT_COMPANY_NAME: `OneERP Synthetic HTTP Journey ${nonce}`,
+ REHEARSAL_API_IMAGE: imageTags.api, REHEARSAL_MIGRATE_IMAGE: imageTags.migrate,
+ REHEARSAL_WEB_IMAGE: imageTags.web, REHEARSAL_MINIO_IMAGE: imageTags.minio,
+ };
+ for (const key of ['POSTGRES_PASSWORD', 'JWT_SECRET', 'MINIO_ACCESS_KEY', 'MINIO_SECRET_KEY', 'INIT_ADMIN_PASSWORD']) {
+ log(`::add-mask::${generated[key]}`);
+ }
+ const dockerEnv = { ...env, ...generated, COMPOSE_DISABLE_ENV_FILE: 'true', DOCKER_BUILDKIT: '1' };
+ for (const key of ['COMPOSE_FILE', 'COMPOSE_PROJECT_NAME', 'COMPOSE_PROFILES', 'COMPOSE_ENV_FILES']) delete dockerEnv[key];
+ const report = { schemaVersion: 1, scope: 'disposable-current-head-container-rehearsal',
+ status: 'failed', startedAt: new Date().toISOString(), commit: null, tree: null,
+ images: {}, checks: [], limitations: ['CI-only MinIO source image; production distribution issue #35 remains open',
+ 'Synthetic data; no production deployment, TLS, offsite backup, RPO certification or business sign-off'] };
+ const call = (command, args, options = {}) => runner(command, args, { cwd: root, env: dockerEnv, signal, ...options });
+ const compose = (args, options = {}, project = sourceProject) => call('docker',
+ composeArgs({ root, composeFile, envFile, project }, args), options);
+ const step = async (name, fn) => {
+ log(`[START] ${name}`);
+ try {
+ const result = await fn();
+ report.checks.push({ name, status: 'passed' });
+ log(`[PASS] ${name}`);
+ return result;
+ } catch {
+ report.checks.push({ name, status: 'failed' });
+ throw new Error(`Container rehearsal failed at ${name}`);
+ }
+ };
+ let failure;
+ let mayHaveContainers = false;
+ try {
+ await writeFile(envFile, Object.entries(generated).map(([key, value]) => `${key}=${JSON.stringify(value)}\n`).join(''), { mode: 0o600 });
+ await step('clean-current-head', async () => {
+ report.commit = await call('git', ['rev-parse', 'HEAD']);
+ report.tree = await call('git', ['rev-parse', 'HEAD^{tree}']);
+ assert(/^[a-f0-9]{40}$/.test(report.commit) && /^[a-f0-9]{40}$/.test(report.tree), 'Invalid Git identity');
+ assert(!(await call('git', ['status', '--porcelain', '--untracked-files=all'])), 'Build requires a clean source checkout');
+ });
+ await step('local-docker-and-compose', async () => {
+ const endpoint = await call('docker', ['context', 'inspect', '--format', '{{.Endpoints.docker.Host}}']);
+ assert(endpoint === 'unix:///var/run/docker.sock', 'Only the local GitHub runner Docker socket is allowed');
+ await call('docker', ['info', '--format', '{{.ServerVersion}}']);
+ await call('docker', ['compose', 'version']);
+ await compose(['config', '--quiet']);
+ });
+ for (const [name, dockerfile, target, context, minutes] of [
+ ['migrate', 'apps/api/Dockerfile', 'builder', '.', 12],
+ ['api', 'apps/api/Dockerfile', 'production', '.', 10],
+ ['web', 'apps/web/Dockerfile', 'production', '.', 15],
+ ['minio', 'scripts/ci/minio.Dockerfile', null, 'scripts/ci', 20],
+ ]) {
+ await step(`build-${name}-image`, async () => {
+ const args = ['build', '--progress=plain', '--file', dockerfile, '--tag', imageTags[name]];
+ if (target) args.push('--target', target, '--label', `org.opencontainers.image.revision=${report.commit}`);
+ args.push(context);
+ await call('docker', args, { timeoutMs: minutes * 60_000, onFailureOutput: (output) => {
+ // Builds receive no secret build args. Still redact generated values and
+ // prefix each line so untrusted build output cannot issue Actions commands.
+ let safe = output;
+ for (const value of Object.values(generated)) safe = safe.split(value).join('[REDACTED]');
+ safe = safe.replace(/[\x00-\x08\x0b-\x1f\x7f]/g, '').slice(-16_384);
+ log(safe.split('\n').slice(-80).map((line) => `[BUILD ${name}] ${line}`).join('\n'));
+ } });
+ const imageId = await call('docker', ['image', 'inspect', '--format', '{{.Id}}', imageTags[name]]);
+ assert(IMAGE_ID_PATTERN.test(imageId), 'Invalid built image ID');
+ report.images[name] = { id: imageId, target: target || 'CI-only verified source' };
+ });
+ }
+ await step('start-isolated-dependencies', async () => {
+ // Set before up: a partially failed up can still have created owned volumes.
+ mayHaveContainers = true;
+ await compose(['up', '--detach', '--wait', '--wait-timeout', '120', 'db', 'redis', 'minio'], { timeoutMs: 180_000 });
+ for (const name of ['db', 'redis', 'minio']) {
+ const container = await compose(['ps', '-q', name]);
+ assert(/^[a-f0-9]{12,64}$/.test(container), 'Expected exactly one dependency container');
+ const imageId = await call('docker', ['inspect', '--format', '{{.Image}}', container]);
+ assert(IMAGE_ID_PATTERN.test(imageId), 'Invalid dependency image ID');
+ if (name === 'minio') assert(imageId === report.images.minio.id, 'MinIO image changed after build');
+ else report.images[name] = { id: imageId };
+ }
+ await compose(['run', '--rm', '--no-deps', '--entrypoint', 'node', 'api', '-e',
+ "(async()=>{for(let i=0;i<60;i++){try{const r=await fetch('http://minio:9000/minio/health/ready',{signal:AbortSignal.timeout(2000)});if(r.ok)return}catch{}await new Promise(r=>setTimeout(r,1000))}process.exit(1)})()"],
+ { timeoutMs: 200_000 });
+ });
+ await step('migrate-and-initialize-production-schema', () => compose(['run', '--rm', '--no-deps', 'migrate'], { timeoutMs: 180_000 }));
+ await step('start-production-api-and-web', async () => {
+ await compose(['up', '--detach', '--no-deps', '--wait', '--wait-timeout', '180', 'api', 'web'], { timeoutMs: 240_000 });
+ for (const name of ['api', 'web']) {
+ const container = await compose(['ps', '-q', name]);
+ assert(/^[a-f0-9]{12,64}$/.test(container), 'Expected exactly one application container');
+ const imageId = await call('docker', ['inspect', '--format', '{{.Image}}', container]);
+ assert(imageId === report.images[name].id, 'Running application image differs from built image');
+ }
+ });
+ const apiBaseUrl = `${loopbackPort(await compose(['port', 'api', '8000']))}/api`;
+ const webBaseUrl = loopbackPort(await compose(['port', 'web', '3000']));
+ const credentials = { email: generated.INIT_ADMIN_EMAIL, password: generated.INIT_ADMIN_PASSWORD };
+ report.web = await step('web-login-and-api-proxy', () => checkWeb(webBaseUrl, credentials, generated.INIT_COMPANY_NAME, fetchImpl));
+ const journeyEnv = { ...dockerEnv, GITHUB_SHA: report.commit, API_BASE_URL: apiBaseUrl,
+ STOCKED_TRADE_ACCEPTANCE: '1', MANUFACTURING_ACCEPTANCE: '1',
+ BUSINESS_ACCEPTANCE_ADMIN_EMAIL: credentials.email, BUSINESS_ACCEPTANCE_ADMIN_PASSWORD: credentials.password,
+ BUSINESS_ACCEPTANCE_COMPANY_NAME: generated.INIT_COMPANY_NAME };
+ const journeyReports = {};
+ report.journeys = {};
+ for (const [name, file, expectedSteps] of [
+ ['stockedTrade', 'stocked-trade-acceptance.mjs', 13], ['manufacturing', 'manufacturing-acceptance.mjs', 12],
+ ]) {
+ const journeyPath = path.join(tempDir, `${name}.json`);
+ report.journeys[name] = await step(`http-${name}`, async () => {
+ await call(process.execPath, [path.join(root, 'scripts', file), '--report', journeyPath], { env: journeyEnv, timeoutMs: 300_000 });
+ journeyReports[name] = JSON.parse(await readFile(journeyPath, 'utf8'));
+ return summarizeJourney(journeyReports[name], expectedSteps);
+ });
+ }
+ report.recovery = await step('backup-and-fresh-volume-recovery', async () => {
+ const runRecovery = recover || (await import('./recovery-rehearsal.mjs')).runRecovery;
+ const recoveryReportPath = path.join(tempDir, 'recovery-report.json');
+ let result;
+ try {
+ result = await runRecovery({ root, composeFile, envFile, sourceProject, recoveryProject,
+ tempDir, reportPath: recoveryReportPath, commit: report.commit,
+ credentials, apiBaseUrl, webBaseUrl, journeyReports, dockerEnv });
+ } catch (error) {
+ try { report.recovery = summarizeRecovery(JSON.parse(await readFile(recoveryReportPath, 'utf8'))); }
+ catch { report.recovery = { status: 'failed' }; }
+ throw error;
+ }
+ assert(result?.status === 'passed', 'Recovery did not pass');
+ // Explicit allowlist: never copy generic context, failure output, or credentials.
+ return summarizeRecovery(result);
+ });
+ await step('source-remains-current-head', async () => {
+ assert(await call('git', ['rev-parse', 'HEAD']) === report.commit, 'Checkout changed during rehearsal');
+ assert(await call('git', ['rev-parse', 'HEAD^{tree}']) === report.tree, 'Source tree changed during rehearsal');
+ assert(!(await call('git', ['status', '--porcelain', '--untracked-files=all'])), 'Source changed during rehearsal');
+ });
+ } catch (error) {
+ failure = error;
+ } finally {
+ if (mayHaveContainers) {
+ let clean = true;
+ for (const project of [recoveryProject, sourceProject]) {
+ try {
+ // Cleanup must remain available after a cancellation signal.
+ await compose(['down', '--volumes', '--remove-orphans', '--timeout', '10'],
+ { signal: undefined, timeoutMs: 60_000 }, project);
+ } catch { clean = false; }
+ }
+ report.checks.push({ name: 'cleanup-owned-projects-and-volumes', status: clean ? 'passed' : 'failed' });
+ if (!clean) failure ||= new Error('Container rehearsal cleanup failed');
+ }
+ await rm(tempDir, { recursive: true, force: true });
+ report.status = failure ? 'failed' : 'passed';
+ report.endedAt = new Date().toISOString();
+ await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 });
+ }
+ if (failure) throw failure;
+ return report;
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
+ const controller = new AbortController();
+ for (const event of ['SIGINT', 'SIGTERM']) process.once(event, () => controller.abort());
+ try {
+ const report = await runRehearsal({ signal: controller.signal });
+ console.log(`Container rehearsal ${report.status}: ${report.commit}`);
+ } catch (error) {
+ // Stage names are safe; never print child output, credentials, or raw config.
+ console.error(error.message);
+ process.exitCode = 1;
+ }
+}
diff --git a/scripts/ci/container-rehearsal.test.mjs b/scripts/ci/container-rehearsal.test.mjs
new file mode 100644
index 0000000..22b425b
--- /dev/null
+++ b/scripts/ci/container-rehearsal.test.mjs
@@ -0,0 +1,153 @@
+import assert from 'node:assert/strict';
+import { mkdtemp, readFile, rm, writeFile, access } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import test from 'node:test';
+import { checkWeb, composeArgs, loopbackPort, requireCi, runRehearsal, summarizeJourney } from './container-rehearsal.mjs';
+
+const ci = { CI: 'true', GITHUB_ACTIONS: 'true', ONEERP_CONTAINER_REHEARSAL: '1' };
+const imageId = `sha256:${'a'.repeat(64)}`;
+const sha = 'b'.repeat(40);
+const tree = 'c'.repeat(40);
+const containerId = 'd'.repeat(64);
+
+test('opt-in and local Docker guards fail closed', () => {
+ for (const env of [{}, { CI: 'true' }, { ...ci, ONEERP_CONTAINER_REHEARSAL: '0' },
+ { ...ci, DOCKER_HOST: 'ssh://production' }, { ...ci, DOCKER_CONTEXT: 'production' }]) {
+ assert.throws(() => requireCi(env));
+ }
+ assert.doesNotThrow(() => requireCi(ci));
+ assert.throws(() => composeArgs({ project: 'oneerp-production' }, ['down', '--volumes']));
+ assert.throws(() => loopbackPort('0.0.0.0:8000'));
+ assert.throws(() => loopbackPort('127.0.0.1:8000\n127.0.0.1:8001'));
+ assert.equal(loopbackPort('127.0.0.1:32123\n'), 'http://127.0.0.1:32123');
+});
+
+test('complete journey count is required and report does not propagate context', () => {
+ const report = { passed: true, context: { token: 'do-not-publish' }, steps: Array.from({ length: 13 }, () => ({ passed: true })) };
+ assert.deepEqual(summarizeJourney(report, 13), { passed: true, passedSteps: 13, expectedSteps: 13 });
+ assert.throws(() => summarizeJourney(report, 12));
+ assert.throws(() => summarizeJourney({ ...report, steps: report.steps.map(() => ({ passed: false })) }, 13));
+});
+
+function webFetch(company = 'OneERP Synthetic HTTP Journey test', missingAsset = false) {
+ return async (url) => {
+ const route = new URL(url).pathname;
+ if (route === '/login') return new Response('', { headers: { 'content-type': 'text/html' } });
+ if (route.startsWith('/_next/static/')) return new Response(missingAsset ? 'missing' : 'content', { headers: { 'content-type': missingAsset ? 'text/html' : 'text/plain' } });
+ if (route.endsWith('/health')) return Response.json({ status: 'ok' });
+ if (route.endsWith('/auth/login')) return Response.json({ accessToken: 'private-token', companies: [{ name: company }] });
+ throw new Error('Unexpected HTTP request');
+ };
+}
+
+test('Web check requires real static assets and API proxy login', async () => {
+ const company = 'OneERP Synthetic HTTP Journey test';
+ const result = await checkWeb('http://127.0.0.1:3000', { email: 'test', password: 'secret' }, company, webFetch(company));
+ assert.equal(result.staticAssets, 2);
+ await assert.rejects(checkWeb('http://127.0.0.1:3000', {}, company, webFetch(company, true)), /static asset/);
+ await assert.rejects(checkWeb('http://127.0.0.1:3000', {}, 'wrong company', webFetch(company)), /synthetic company/);
+});
+
+async function harness(t, { failStage, dirty = false, endpoint = 'unix:///var/run/docker.sock' } = {}) {
+ const dir = await mkdtemp(path.join(tmpdir(), 'rehearsal-test-'));
+ t.after(() => rm(dir, { recursive: true, force: true }));
+ const reportPath = path.join(dir, 'report.json');
+ const commands = [];
+ let generated;
+ let envFile;
+ const runner = async (command, args, options) => {
+ commands.push({ command, args });
+ generated = options.env;
+ if (command === 'git') {
+ if (args[0] === 'status') return dirty ? ' M apps/api/src/main.ts' : '';
+ return args[1] === 'HEAD' ? sha : tree;
+ }
+ if (command === process.execPath) {
+ const count = args[0].endsWith('stocked-trade-acceptance.mjs') ? 13 : 12;
+ await writeFile(args[2], JSON.stringify({ passed: true, context: { token: 'private-token' }, steps: Array.from({ length: count }, () => ({ passed: true })) }));
+ return '';
+ }
+ assert.equal(command, 'docker');
+ if (args[0] === 'context') return endpoint;
+ if (args[0] === 'image' || args[0] === 'inspect') return imageId;
+ if (args[0] === 'compose' && args.includes('--env-file')) {
+ envFile = args[args.indexOf('--env-file') + 1];
+ const action = args.slice(args.indexOf('--project-name') + 2);
+ if (failStage === 'migrate' && action[0] === 'run' && action.at(-1) === 'migrate') throw new Error(generated.INIT_ADMIN_PASSWORD);
+ if (action[0] === 'ps') return containerId;
+ if (action[0] === 'port') return action[1] === 'api' ? '127.0.0.1:31000' : '127.0.0.1:31001';
+ }
+ return '';
+ };
+ const options = { root: '/reviewed/source', env: { ...ci, RUNNER_TEMP: dir, CONTAINER_REHEARSAL_REPORT: reportPath }, runner,
+ log: () => {}, fetchImpl: async (url, options) => webFetch(generated.INIT_COMPANY_NAME)(url, options),
+ recover: async (options) => {
+ assert.match(options.sourceProject, /^oneerp-ci-[a-f0-9]{24}$/);
+ assert.equal(options.recoveryProject, `${options.sourceProject}-restore`);
+ assert.equal(options.credentials.password, generated.INIT_ADMIN_PASSWORD);
+ assert.equal(options.journeyReports.manufacturing.steps.length, 12);
+ return { status: 'passed', checks: [{ name: 'restored-files', status: 'passed' }], verifiedFileCount: 1,
+ objectEvidenceDigest: 'e'.repeat(64), businessTables: [{ table: 'FileRecord', count: 1 }], credentials: 'never publish' };
+ } };
+ return { options, commands, reportPath, getGenerated: () => generated, getEnvFile: () => envFile };
+}
+
+test('dirty checkout and remote Docker daemon stop before build or cleanup', async (t) => {
+ for (const scenario of [{ dirty: true }, { endpoint: 'ssh://production' }]) {
+ const h = await harness(t, scenario);
+ await assert.rejects(runRehearsal(h.options));
+ assert.equal(h.commands.filter(({ args }) => args[0] === 'build' || args.includes('down')).length, 0);
+ assert.equal(JSON.parse(await readFile(h.reportPath, 'utf8')).status, 'failed');
+ }
+});
+
+test('migration failure cleans only allocated random projects and writes no secrets', async (t) => {
+ const h = await harness(t, { failStage: 'migrate' });
+ await assert.rejects(runRehearsal(h.options), /migrate-and-initialize/);
+ const cleanups = h.commands.filter(({ args }) => args.includes('down'));
+ assert.equal(cleanups.length, 2);
+ for (const { args } of cleanups) {
+ assert.match(args[args.indexOf('--project-name') + 1], /^oneerp-ci-[a-f0-9]{24}(?:-restore)?$/);
+ assert.ok(args.includes('--volumes'));
+ }
+ assert.equal(h.commands.some(({ command }) => command === process.execPath), false);
+ const raw = await readFile(h.reportPath, 'utf8');
+ assert.equal(raw.includes(h.getGenerated().INIT_ADMIN_PASSWORD), false);
+ assert.equal(JSON.parse(raw).status, 'failed');
+ await assert.rejects(access(h.getEnvFile()));
+});
+
+test('successful mocked rehearsal records actual source, image IDs, 25 steps and sanitized recovery', async (t) => {
+ const h = await harness(t);
+ const result = await runRehearsal(h.options);
+ assert.equal(result.status, 'passed');
+ assert.equal(result.commit, sha);
+ assert.equal(result.tree, tree);
+ assert.equal(result.images.api.id, imageId);
+ assert.equal(result.journeys.stockedTrade.passedSteps + result.journeys.manufacturing.passedSteps, 25);
+ assert.equal(result.recovery.verifiedFileCount, 1);
+ assert.equal(h.commands.filter(({ args }) => args[0] === 'build').length, 4);
+ const raw = await readFile(h.reportPath, 'utf8');
+ for (const secret of [h.getGenerated().POSTGRES_PASSWORD, h.getGenerated().JWT_SECRET,
+ h.getGenerated().MINIO_SECRET_KEY, h.getGenerated().INIT_ADMIN_PASSWORD, 'private-token', 'never publish']) {
+ assert.equal(raw.includes(secret), false);
+ }
+ await assert.rejects(access(h.getEnvFile()));
+});
+
+test('recovery failure retains only safe failed stage evidence and still cleans up', async (t) => {
+ const h = await harness(t);
+ h.options.recover = async ({ reportPath }) => {
+ await writeFile(reportPath, JSON.stringify({ status: 'failed',
+ checks: [{ name: 'restored-row-verification', status: 'failed', reason: 'secret details' }],
+ password: 'never-publish' }));
+ throw new Error('never-publish');
+ };
+ await assert.rejects(runRehearsal(h.options), /backup-and-fresh-volume-recovery/);
+ const raw = await readFile(h.reportPath, 'utf8');
+ assert.deepEqual(JSON.parse(raw).recovery, { status: 'failed', checks: [{ name: 'restored-row-verification', status: 'failed' }] });
+ assert.equal(raw.includes('never-publish'), false);
+ assert.equal(raw.includes('secret details'), false);
+ assert.equal(h.commands.filter(({ args }) => args.includes('down')).length, 2);
+});
diff --git a/scripts/ci/recovery-rehearsal.md b/scripts/ci/recovery-rehearsal.md
new file mode 100644
index 0000000..15a9187
--- /dev/null
+++ b/scripts/ci/recovery-rehearsal.md
@@ -0,0 +1,40 @@
+# Synthetic container recovery component
+
+`runRecovery` is invoked by the container rehearsal after HTTP journeys finish. It
+requires `CI=true`, `GITHUB_ACTIONS=true`, `ONEERP_CONTAINER_REHEARSAL=1`, a
+`oneerp-ci-*` source project and its exact `-restore` counterpart. Never invoke it
+against production or use its result as real-data recovery acceptance.
+
+Inputs: `root`, `composeFile`, `envFile`, `sourceProject`, `recoveryProject`,
+`tempDir`, `reportPath`, `commit`, `credentials: {email,password}`, and `dockerEnv`.
+The caller must have finished synthetic writes and background event processing.
+It must upload only the explicit sanitized reports, never the temporary directory.
+
+The component reuses the existing backup scripts' PostgreSQL logical dump and
+MinIO volume archive approach. It does not invoke `restore-drill.sh`, because
+that script rebuilds images and can initialize a database through dependencies.
+The bounded component instead:
+
+- rejects preexisting restore containers or volumes;
+- resolves source running image IDs and starts the same five immutable images;
+- removes published ports and migration dependencies, recreates project-scoped
+ volumes, and rejects external volumes and bind mounts;
+- uses `psql -X -v ON_ERROR_STOP=1`; SQL errors prevent subsequent API startup;
+- extracts MinIO bytes before starting restored MinIO;
+- compares every public table's count and ordered complete-row fingerprint,
+ including business record IDs; requires a journey engineering revision;
+- logs in through the real API and downloads every retained upload through its
+ API-issued signed URL, checking SHA-256 against the database and source;
+- verifies source rows remain unchanged and destroys only the restore project.
+
+The logical dump and archive stay in process memory. Resolved Compose credentials
+are private (0700 directory / 0600 file) and removed in `finally`. Reports contain
+only stage status, table counts and an aggregate file-evidence hash, never SQL,
+archives, environment values, credentials, file paths, tokens or signed URLs.
+
+Validation: `node --test scripts/ci/recovery-rehearsal.test.mjs` exercises scope,
+volume/image isolation and a mocked SQL failure with restore-only cleanup. These
+are orchestration tests, not evidence that a real container recovery succeeded.
+The GitHub container rehearsal must supply that evidence for its exact commit.
+Graphify graph data and executable were absent in this checkout; no graph refresh
+is claimed.
diff --git a/scripts/ci/recovery-rehearsal.mjs b/scripts/ci/recovery-rehearsal.mjs
new file mode 100644
index 0000000..394f788
--- /dev/null
+++ b/scripts/ci/recovery-rehearsal.mjs
@@ -0,0 +1,164 @@
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import path from 'node:path';
+
+const services = ['db', 'redis', 'minio', 'api', 'web'];
+export function assertRecoveryScope({ sourceProject, recoveryProject }, env = process.env) {
+ if (env.CI !== 'true' || env.GITHUB_ACTIONS !== 'true' || env.ONEERP_CONTAINER_REHEARSAL !== '1') throw new Error('Recovery requires disposable GitHub Actions CI');
+ if (!/^oneerp-ci-[a-z0-9-]+$/.test(sourceProject) || recoveryProject !== `${sourceProject}-restore`) throw new Error('Recovery project must be the isolated CI restore project');
+}
+export function isolatedConfig(source, imageIds) {
+ const result = { services: {}, volumes: {} };
+ for (const name of services) {
+ const original = source.services[name];
+ if (!original || !/^sha256:[a-f0-9]{64}$/.test(imageIds[name])) throw new Error(`Missing immutable image for ${name}`);
+ const service = structuredClone(original);
+ delete service.build; delete service.container_name; delete service.ports; delete service.depends_on;
+ delete service.networks; delete service.profiles;
+ service.image = imageIds[name]; service.pull_policy = 'never'; service.restart = 'no';
+ for (const volume of service.volumes || []) {
+ if (volume.type !== 'volume' || !volume.source || source.volumes?.[volume.source]?.external) throw new Error('Recovery permits only project-owned named volumes');
+ result.volumes[volume.source] = {};
+ }
+ result.services[name] = service;
+ }
+ return result;
+}
+
+// Read-only fingerprint includes every business row and its identity, not just counts.
+const fingerprintSql = `SELECT format('SELECT %L AS table_name, count(*) AS row_count, md5(coalesce(string_agg(row_to_json(t)::text, %L ORDER BY row_to_json(t)::text), %L)) AS fingerprint FROM %I.%I t;', tablename, E'\\n', '', schemaname, tablename) FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename;`;
+export const recoveryProbeCode = `
+const crypto = require('node:crypto');
+let input=''; process.stdin.on('data', b=>input+=b); process.stdin.on('end', async()=>{
+try {
+ const {email,password,companyName}=JSON.parse(input);
+ const credentials={email,password};
+ const base='http://127.0.0.1:8000/api';
+ async function req(p, options={}) { const r=await fetch(base+p, options); if(!r.ok) throw Error('HTTP '+r.status); const j=await r.json(); return j.data ?? j; }
+ const login=await req('/auth/login',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(credentials)});
+ const token=login.access_token ?? login.accessToken; if(!token) throw Error('Missing login token');
+ const companies=(login.companies || []).filter(c=>c.name===companyName);
+ if(!companyName || companies.length!==1 || !companies[0].id) throw Error('Missing or ambiguous synthetic company');
+ const files=await req('/files',{headers:{Authorization:'Bearer '+token,'x-company-id':companies[0].id}});
+ if(!Array.isArray(files) || files.length===0) throw Error('No journey upload to verify');
+ const result=[];
+ for(const f of files) {
+ if(!f.id || !f.checksumSha256 || !f.downloadUrl) throw Error('Incomplete file evidence');
+ const u=new URL(f.downloadUrl); if(u.hostname!=='minio' || u.port!=='9000') throw Error('Unexpected object origin');
+ const r=await fetch(u); if(!r.ok) throw Error('Object download failed');
+ const checksum=crypto.createHash('sha256').update(Buffer.from(await r.arrayBuffer())).digest('hex');
+ if(checksum!==f.checksumSha256) throw Error('Object bytes checksum mismatch');
+ result.push({id:f.id,checksumSha256:checksum});
+ }
+ result.sort((a,b)=>a.id.localeCompare(b.id));
+ process.stdout.write(JSON.stringify({files:result}));
+} catch { process.stderr.write('API recovery verification failed'); process.exitCode=1; }
+});`;
+
+export async function runRecovery(options) {
+ const { root, composeFile, envFile, sourceProject, recoveryProject, tempDir, reportPath, commit, credentials, dockerEnv = process.env } = options;
+ assertRecoveryScope(options);
+ const checks = [];
+ const report = { schemaVersion: 1, commit, scope: 'synthetic-ci-only', status: 'failed', checks };
+ const recoveryDir = path.join(tempDir, 'recovery-private');
+ mkdirSync(recoveryDir, { recursive: true, mode: 0o700 });
+ const restoreFile = path.join(recoveryDir, 'compose.json');
+ let allocated = false;
+ let stage = 'isolation-guards';
+ function docker(args, input) {
+ const result = spawnSync('docker', args, { cwd: root, env: dockerEnv, input, encoding: input instanceof Buffer ? null : 'utf8', maxBuffer: 128 * 1024 * 1024, timeout: 300000 });
+ if (result.error || result.status !== 0) throw new Error('Recovery Docker operation failed');
+ return result.stdout;
+ }
+ const source = (...args) => docker(['compose', '-p', sourceProject, '--env-file', envFile, '-f', composeFile, ...args]);
+ const restore = (...args) => docker(['compose', '-p', recoveryProject, '-f', restoreFile, ...args]);
+ function psql(project, sql) {
+ const file = project === sourceProject ? composeFile : restoreFile;
+ const envArgs = project === sourceProject ? ['--env-file', envFile] : [];
+ return docker(['compose', '-p', project, ...envArgs, '-f', file, 'exec', '-T', 'db', 'sh', '-c', 'psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At'], sql);
+ }
+ function snapshot(project) { return psql(project, psql(project, fingerprintSql)).trim(); }
+ function probe(project) {
+ const file = project === sourceProject ? composeFile : restoreFile;
+ const envArgs = project === sourceProject ? ['--env-file', envFile] : [];
+ return JSON.parse(docker(['compose', '-p', project, ...envArgs, '-f', file, 'exec', '-T', 'api', 'node', '-e', recoveryProbeCode], JSON.stringify({ ...credentials, companyName: dockerEnv.INIT_COMPANY_NAME })));
+ }
+ try {
+ // Refuse even stopped containers or leftover volumes: never restore into existing data.
+ for (const kind of ['container', 'volume']) {
+ if (docker([kind, 'ls', ...(kind === 'container' ? ['-a'] : []), '-q', '--filter', `label=com.docker.compose.project=${recoveryProject}`]).trim()) throw new Error('Recovery project is not fresh');
+ }
+ const config = JSON.parse(source('config', '--format', 'json'));
+ const imageIds = {};
+ const containers = {};
+ for (const name of services) {
+ containers[name] = source('ps', '-q', name).trim();
+ if (!/^[a-f0-9]{12,64}$/.test(containers[name])) throw new Error(`Missing source service ${name}`);
+ imageIds[name] = docker(['inspect', '--format', '{{.Image}}', containers[name]]).trim();
+ }
+ stage = 'source-api-and-backup';
+ const sourceFiles = probe(sourceProject);
+ // The event runner retries every 30s. Await a drained queue without altering it.
+ let drained = false;
+ for (let attempt = 0; attempt < 45; attempt++) {
+ const pending = psql(sourceProject, `SELECT count(*) FROM "EventDlq" WHERE status IN ('PENDING', 'RETRYING');`).trim();
+ if (pending === '0') { drained = true; break; }
+ if (!/^\d+$/.test(pending)) throw new Error('Invalid pending event count');
+ await new Promise(resolve => setTimeout(resolve, 2000));
+ }
+ if (!drained) throw new Error('Synthetic event queue did not drain');
+ const before = snapshot(sourceProject);
+ if (!before.split('\n').some(line => /^EngineeringDocumentRevision\|[1-9][0-9]*\|/.test(line))) throw new Error('Missing engineering journey revision');
+ const sql = source('exec', '-T', 'db', 'sh', '-c', 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --clean --if-exists --no-owner --no-privileges');
+ // The same PostgreSQL alpine image supplies tar; no additional mutable helper image.
+ const archive = docker(['run', '--rm', '--network', 'none', '--volumes-from', `${containers.minio}:ro`, '--entrypoint', 'tar', imageIds.db, 'czf', '-', '-C', '/data', '.'], Buffer.alloc(0));
+ if (!sql || archive.length === 0 || before !== snapshot(sourceProject)) throw new Error('Source changed during backup');
+ docker(['run', '--rm', '-i', '--network', 'none', '--entrypoint', 'tar', imageIds.db, 'tzf', '-'], archive);
+ checks.push({ name: 'backup-consistent', status: 'passed' });
+ writeFileSync(restoreFile, JSON.stringify(isolatedConfig(config, imageIds)), { mode: 0o600 });
+ allocated = true;
+ stage = 'postgres-and-object-restore';
+ restore('up', '-d', '--wait', '--wait-timeout', '180', 'db', 'redis');
+ // Create MinIO without starting it so extraction precedes any object-store writes.
+ restore('create', 'minio');
+ const minio = restore('ps', '-a', '-q', 'minio').trim();
+ if (!/^[a-f0-9]{12,64}$/.test(minio)) throw new Error('Missing restore object store');
+ psql(recoveryProject, sql);
+ docker(['run', '--rm', '-i', '--network', 'none', '--volumes-from', minio, '--entrypoint', 'tar', imageIds.db, 'xzf', '-', '-C', '/data'], archive);
+ stage = 'restored-row-verification';
+ const after = snapshot(recoveryProject);
+ if (before !== after) throw new Error('Restored business row identities or counts differ');
+ report.businessTables = before.split('\n').map(line => { const [table, count] = line.split('|'); return { table, count: Number(count) }; });
+ checks.push({ name: 'business-row-counts-and-identities', status: 'passed' });
+ stage = 'restored-api-and-image-verification';
+ restore('up', '-d', 'minio');
+ // Scratch MinIO has no healthcheck executable. Probe using the exact API image
+ // before application initialization can attempt to create its bucket.
+ restore('run', '--rm', '--no-deps', '--entrypoint', 'node', 'api', '-e',
+ "(async()=>{for(let i=0;i<60;i++){try{const r=await fetch('http://minio:9000/minio/health/ready',{signal:AbortSignal.timeout(2000)});if(r.ok)return}catch{}await new Promise(r=>setTimeout(r,1000))}process.exit(1)})()");
+ restore('up', '-d', '--wait', '--wait-timeout', '180', 'api', 'web');
+ for (const name of services) {
+ const id = restore('ps', '-q', name).trim();
+ if (docker(['inspect', '--format', '{{.Image}}', id]).trim() !== imageIds[name]) throw new Error('Restore image differs');
+ }
+ checks.push({ name: 'exact-running-image-reuse', status: 'passed' });
+ const recoveredFiles = probe(recoveryProject);
+ if (JSON.stringify(sourceFiles) !== JSON.stringify(recoveredFiles)) throw new Error('Recovered upload identity differs');
+ report.verifiedFileCount = recoveredFiles.files.length;
+ report.objectEvidenceDigest = createHash('sha256').update(JSON.stringify(recoveredFiles)).digest('hex');
+ checks.push({ name: 'restored-api-login-and-upload-bytes', status: 'passed' });
+ if (before !== snapshot(sourceProject)) throw new Error('Source business rows changed');
+ checks.push({ name: 'source-preserved', status: 'passed' });
+ report.status = 'passed';
+ } catch (error) {
+ // No raw Docker output, SQL, environment, signed URLs, tokens or archive content.
+ checks.push({ name: stage, status: 'failed', reason: 'Recovery verification failed; inspect the failed stage without publishing private data.' });
+ throw error;
+ } finally {
+ try { if (allocated) restore('down', '-v', '--remove-orphans'); }
+ catch (error) { report.status = 'failed'; checks.push({ name: 'restore-cleanup', status: 'failed' }); throw error; }
+ finally { rmSync(recoveryDir, { recursive: true, force: true }); writeFileSync(reportPath, JSON.stringify(report, null, 2)); }
+ }
+ return report;
+}
diff --git a/scripts/ci/recovery-rehearsal.test.mjs b/scripts/ci/recovery-rehearsal.test.mjs
new file mode 100644
index 0000000..f3edfdc
--- /dev/null
+++ b/scripts/ci/recovery-rehearsal.test.mjs
@@ -0,0 +1,83 @@
+import test from 'node:test';
+import { spawnSync } from 'node:child_process';
+import assert from 'node:assert/strict';
+import { mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { assertRecoveryScope, isolatedConfig, runRecovery, recoveryProbeCode } from './recovery-rehearsal.mjs';
+const scope = { sourceProject: 'oneerp-ci-abc', recoveryProject: 'oneerp-ci-abc-restore' };
+const env = { CI: 'true', GITHUB_ACTIONS: 'true', ONEERP_CONTAINER_REHEARSAL: '1' };
+const digest = `sha256:${'a'.repeat(64)}`;
+const names = ['db', 'redis', 'minio', 'api', 'web'];
+const images = Object.fromEntries(names.map(n => [n, digest]));
+const config = { services: Object.fromEntries(names.map(n => [n, { image: 'mutable:tag', ports: ['8000:8000'], depends_on: { migrate: {} }, volumes: n === 'db' ? [{ type: 'volume', source: 'data', target: '/data' }] : [] }])), volumes: { data: { name: 'source_data' } } };
+test('scope fails closed outside explicit isolated GitHub CI', () => {
+ assert.doesNotThrow(() => assertRecoveryScope(scope, env));
+ for (const key of Object.keys(env)) assert.throws(() => assertRecoveryScope(scope, { ...env, [key]: '' }));
+ assert.throws(() => assertRecoveryScope({ ...scope, recoveryProject: scope.sourceProject }, env));
+ assert.throws(() => assertRecoveryScope({ sourceProject: 'production', recoveryProject: 'production-restore' }, env));
+});
+test('derived stack pins exact images, fresh volumes, no migration or exposed ports', () => {
+ const isolated = isolatedConfig(config, images);
+ assert.deepEqual(isolated.volumes, { data: {} });
+ for (const service of Object.values(isolated.services)) {
+ assert.equal(service.image, digest); assert.equal(service.pull_policy, 'never');
+ assert.equal(service.ports, undefined); assert.equal(service.depends_on, undefined);
+ }
+ assert.equal(isolated.services.migrate, undefined);
+ const unsafe = structuredClone(config); unsafe.volumes.data.external = true;
+ assert.throws(() => isolatedConfig(unsafe, images));
+ unsafe.volumes.data.external = false; unsafe.services.db.volumes[0].type = 'bind';
+ assert.throws(() => isolatedConfig(unsafe, images));
+ assert.throws(() => isolatedConfig(config, { ...images, api: 'mutable:tag' }));
+});
+test('SQL failure halts before API start and cleans only restore project', async () => {
+ const dir = mkdtempSync(path.join(os.tmpdir(), 'recovery-mock-'));
+ const old = Object.fromEntries(Object.keys(env).map(k => [k, process.env[k]]));
+ Object.assign(process.env, env);
+ const log = path.join(dir, 'calls.jsonl');
+ const report = path.join(dir, 'report.json');
+ const mock = `#!/usr/bin/env -S node --
+const fs=require('node:fs'); const a=process.argv.slice(2); const input=fs.readFileSync(0,'utf8');
+fs.appendFileSync(process.env.MOCK_LOG,JSON.stringify({a,input})+'\\n');
+if(a[0]==='container'||a[0]==='volume') process.exit(0);
+if(a.includes('config')) process.stdout.write(${JSON.stringify(JSON.stringify(config))});
+else if(a[0]==='inspect') console.log(${JSON.stringify(digest)});
+else if(a.includes('ps')) console.log('aaaaaaaaaaaa');
+else if(a.includes('node')) console.log(JSON.stringify({files:[{id:'file',checksumSha256:'checksum'}]}));
+else if(a.some(x=>x.includes('pg_dump'))) console.log('SQL PRIVATE');
+else if(a.includes('czf')) process.stdout.write('archive');
+else if(a.some(x=>x.includes('psql'))) {
+ if(input.includes('SELECT count(*) FROM')) { console.log('0'); process.exit(0); }
+ if(a.includes('oneerp-ci-abc-restore') && input.includes('SQL PRIVATE')) process.exit(3);
+ console.log('EngineeringDocumentRevision|1|fingerprint');
+}
+`;
+ writeFileSync(path.join(dir, 'docker'), mock, { mode: 0o700 });
+ try {
+ await assert.rejects(runRecovery({ ...scope, root: dir, composeFile: '/mock/source.json', envFile: '/mock/ci.env', tempDir: dir, reportPath: report, credentials: { email: 'ci@example.test', password: 'secret-password-sentinel' }, dockerEnv: { ...process.env, PATH: `${dir}:${process.env.PATH}`, MOCK_LOG: log } }));
+ const calls = readFileSync(log, 'utf8').trim().split('\n').map(JSON.parse);
+ const sqlRestore = calls.find(c => c.a.includes(scope.recoveryProject) && c.input.includes('SQL PRIVATE'));
+ assert.ok(sqlRestore.a.some(x => x.includes('ON_ERROR_STOP=1')));
+ assert.ok(calls.some(c => c.a.includes(scope.recoveryProject) && c.a.includes('down')));
+ assert.ok(!calls.some(c => c.a.includes(scope.sourceProject) && c.a.includes('down')));
+ assert.ok(!calls.some(c => c.a.includes(scope.recoveryProject) && c.a.includes('up') && c.a.includes('api')));
+ const saved = readFileSync(report, 'utf8'); assert.equal(JSON.parse(saved).status, 'failed');
+ assert.ok(!saved.includes('SQL PRIVATE')); assert.ok(!saved.includes('secret-password-sentinel'));
+ } finally { for(const [k,v] of Object.entries(old)) { if(v===undefined) delete process.env[k]; else process.env[k]=v; } rmSync(dir,{recursive:true,force:true}); }
+});
+
+test('real API probe selects exact company and supplies tenant header', () => {
+ const setup = `global.fetch=async(url,options)=>{ url=String(url);
+ if(url.endsWith('/auth/login')) return {ok:true,json:async()=>({accessToken:'token',companies:[{id:'synthetic-id',name:'CI Company'}]})};
+ if(url.endsWith('/files')) {
+ if(options.headers['x-company-id']!=='synthetic-id') throw Error('Missing tenant');
+ return {ok:true,json:async()=>[{id:'file-id',checksumSha256:require('node:crypto').createHash('sha256').update('bytes').digest('hex'),downloadUrl:'http://minio:9000/object'}]};
+ }
+ return {ok:true,arrayBuffer:async()=>Buffer.from('bytes')};
+ };`;
+ const run = companyName => spawnSync(process.execPath, ['-e', setup + recoveryProbeCode], { input: JSON.stringify({email:'ci@example.test',password:'sentinel',companyName}), encoding:'utf8' });
+ const passed = run('CI Company'); assert.equal(passed.status, 0, passed.stderr);
+ assert.equal(JSON.parse(passed.stdout).files[0].id, 'file-id');
+ assert.notEqual(run('Wrong Company').status, 0);
+});
From 2d38c4fcd966168a61093f373cbda1880fc20126 Mon Sep 17 00:00:00 2001
From: R <53855466+cb8010d6@users.noreply.github.com>
Date: Sun, 27 Sep 2026 11:11:35 +0800
Subject: [PATCH 2/3] fix(ci): resolve rehearsal report path after runner
allocation
---
.github/workflows/ci.yml | 1 -
1 file changed, 1 deletion(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f4effb0..762634e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -349,7 +349,6 @@ jobs:
contents: read
env:
ONEERP_CONTAINER_REHEARSAL: '1'
- CONTAINER_REHEARSAL_REPORT: ${{ runner.temp }}/container-rehearsal-report.json
steps:
- name: Checkout exact source revision
uses: actions/checkout@v4
From 3b230aabfef41c116cc2a7db9f0f8c6dabc0b075 Mon Sep 17 00:00:00 2001
From: R <53855466+cb8010d6@users.noreply.github.com>
Date: Sun, 27 Sep 2026 11:18:12 +0800
Subject: [PATCH 3/3] feat(ops): verify compose runtime health before
acceptance
---
docs/CONTROLLED_TRIAL_DELIVERY.md | 2 +
docs/PRODUCTION_READINESS.md | 4 +-
scripts/deploy-check.ps1 | 184 ++++++++++++++++++----
scripts/deploy-check.sh | 185 ++++++++++++++++++++--
scripts/deploy-check.test.mjs | 249 ++++++++++++++++++++++++++++++
5 files changed, 573 insertions(+), 51 deletions(-)
create mode 100644 scripts/deploy-check.test.mjs
diff --git a/docs/CONTROLLED_TRIAL_DELIVERY.md b/docs/CONTROLLED_TRIAL_DELIVERY.md
index 3dc9e4b..658dc78 100644
--- a/docs/CONTROLLED_TRIAL_DELIVERY.md
+++ b/docs/CONTROLLED_TRIAL_DELIVERY.md
@@ -12,6 +12,8 @@ GitHub Actions 的 `CI / container-rehearsal` 从 PR head 构建现有 API produ
此演练仅允许 GitHub Actions 的一次性环境，不能把 CI 标志加到生产服务器后运行。临时数据库、归档、凭据和完整容器日志不作为 artifact 发布。
+恢复组件独立执行 `pg_dump`、`tar` 和 `psql`，不等于已经验收运维用的 `backup.*`、`restore.*` 脚本。它在合成业务写入结束、事件队列排空后归档；不证明并发写入中的跨存储一致性。恢复 Redis 使用新卷，原有会话与缓存连续性不在本次验收范围。
+
## 集成和部署边界
- 保留现有审查栈的提交历史；通过精确 head 验证后，再固定独立的 `release/controlled-trial-*` 分支作为试运行基线。
diff --git a/docs/PRODUCTION_READINESS.md b/docs/PRODUCTION_READINESS.md
index 6770406..0d9496b 100644
--- a/docs/PRODUCTION_READINESS.md
+++ b/docs/PRODUCTION_READINESS.md
@@ -74,7 +74,9 @@ Deploy workflow 会校验部署变量格式：`DEPLOY_PATH` 必须是无空格
远端部署目录必须保留生产 `.env`；workflow 会在部署前同步当前 commit 的 `docker-compose.prod.yml`、`scripts/audit-prod-config.sh` 和 `scripts/deploy-check.sh`，并在拉取镜像前审计远端 `.env`。部署完成后会自动执行远端 `scripts/deploy-check.sh`；`scripts/prod-smoke.*` 和 `scripts/business-acceptance.*` 仍需在受控试运行前单独执行并留存报告。
-`scripts/deploy-check.*` 不只检查 Docker Compose 命令可执行，还会把 `exited`、`unhealthy`、`restarting`、`dead` 等服务状态视为失败。
+`scripts/deploy-check.*` 会从所选 Compose 配置确认 `api`、`web`、`db`、`redis`、`minio` 五个服务已定义，并逐容器检查它们都在运行；配置了 healthcheck 的容器必须为 `healthy`。检查会等待最多 60 秒（可用 `DEPLOY_CHECK_TIMEOUT_SECONDS` 调整，范围 0–900 秒），`starting` 和尚未创建的容器会在期限内重试；超时、缺失服务、已退出或 unhealthy 容器会失败。若所选配置定义了 `migrate`，还要求其容器以退出码 0 完成；未定义 migrate 时跳过此项。Compose 文件中的 profile-only 服务（例如 `with-nginx` 下的 `nginx`）不会成为必需服务。
+
+Shell 脚本使用 `COMPOSE_FILE`、`ENV_FILE` 环境变量；PowerShell 脚本支持同名环境变量和 `-ComposeFile`、`-EnvFile` 参数。健康探测使用 `docker compose ps --all -q ` 和仅包含容器状态、health 状态、退出码的 `docker inspect --format` 输出，不打印渲染后的 Compose 环境值。
## P1：强烈建议 / Strongly Recommended
diff --git a/scripts/deploy-check.ps1 b/scripts/deploy-check.ps1
index f547ae0..f049718 100644
--- a/scripts/deploy-check.ps1
+++ b/scripts/deploy-check.ps1
@@ -1,13 +1,36 @@
param(
- [string]$ComposeFile = "docker-compose.ha-lite.yml",
- [string]$EnvFile = ".env",
+ [string]$ComposeFile = "",
+ [string]$EnvFile = "",
[string]$ApiUrl = "",
- [string]$WebUrl = ""
+ [string]$WebUrl = "",
+ [int]$TimeoutSeconds = -1,
+ [int]$PollIntervalSeconds = -1
)
$ErrorActionPreference = "Stop"
$root = Split-Path -Parent $PSScriptRoot
$results = New-Object System.Collections.Generic.List[object]
+$script:ComposeServices = @()
+$script:InspectFormat = '{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}|{{.State.ExitCode}}'
+
+if ($ComposeFile -eq "") { $ComposeFile = if (-not [string]::IsNullOrEmpty($env:COMPOSE_FILE)) { $env:COMPOSE_FILE } else { "docker-compose.ha-lite.yml" } }
+if ($EnvFile -eq "") { $EnvFile = if (-not [string]::IsNullOrEmpty($env:ENV_FILE)) { $env:ENV_FILE } else { ".env" } }
+function ConvertTo-BoundedSeconds {
+ param([string]$Value, [int]$Default, [int]$Maximum)
+ if ($Value -notmatch '^\d+$') { return $Default }
+ $normalized = $Value.TrimStart('0')
+ if ($normalized -eq "") { return 0 }
+ if ($normalized.Length -gt 3) { return $Maximum }
+ $parsed = [int]$normalized
+ if ($parsed -gt $Maximum) { return $Maximum }
+ return $parsed
+}
+
+if ($TimeoutSeconds -lt 0) { $TimeoutSeconds = ConvertTo-BoundedSeconds $env:DEPLOY_CHECK_TIMEOUT_SECONDS 60 900 }
+if ($PollIntervalSeconds -lt 0) { $PollIntervalSeconds = ConvertTo-BoundedSeconds $env:DEPLOY_CHECK_POLL_SECONDS 3 60 }
+if ($TimeoutSeconds -gt 900) { $TimeoutSeconds = 900 }
+if ($PollIntervalSeconds -gt 60) { $PollIntervalSeconds = 60 }
+if ($PollIntervalSeconds -lt 1) { $PollIntervalSeconds = 1 }
function Add-Result {
param([string]$Name, [bool]$Passed, [string]$Detail)
@@ -29,31 +52,132 @@ function Read-Env {
return $map
}
+function Invoke-Compose {
+ param([string[]]$ComposeArgs)
+ $baseArgs = @()
+ if ($EnvFile -ne "" -and (Test-Path $envPath)) { $baseArgs += @("--env-file", $envPath) }
+ $baseArgs += @("-f", $composePath)
+ $output = @(& docker compose @baseArgs @ComposeArgs 2>$null)
+ $exitCode = $LASTEXITCODE
+ return [pscustomobject]@{ ExitCode = $exitCode; Output = ($output -join [Environment]::NewLine) }
+}
+
+function Invoke-Docker {
+ param([string[]]$DockerArgs)
+ $output = @(& docker @DockerArgs 2>$null)
+ $exitCode = $LASTEXITCODE
+ return [pscustomobject]@{ ExitCode = $exitCode; Output = ($output -join [Environment]::NewLine) }
+}
+
function Test-Http {
param([string]$Url)
try {
- $response = Invoke-WebRequest -Uri $Url -UseBasicParsing -TimeoutSec 10
+ $response = Invoke-WebRequest -Uri $Url -UseBasicParsing -TimeoutSec 15
return $response.StatusCode -ge 200 -and $response.StatusCode -lt 500
- } catch {
- return $false
+ } catch { return $false }
+}
+
+function Test-ComposeConfig {
+ $config = Invoke-Compose -ComposeArgs @("config")
+ $services = Invoke-Compose -ComposeArgs @("config", "--services")
+ if ($services.ExitCode -eq 0) {
+ $script:ComposeServices = @($services.Output -split '\r?\n' | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" })
+ }
+ return [pscustomobject]@{
+ ConfigPassed = ($config.ExitCode -eq 0)
+ ServicesPassed = ($services.ExitCode -eq 0 -and $script:ComposeServices.Count -gt 0)
}
}
-function Invoke-Compose {
- param([string[]]$ComposeArgs)
- $baseArgs = @()
- if ($EnvFile -ne "" -and (Test-Path $envPath)) {
- $baseArgs += @("--env-file", $envPath)
+function Test-ComposeService {
+ param([string]$Service, [string]$Mode)
+ if ($script:ComposeServices -notcontains $Service) {
+ return [pscustomobject]@{ State = "failed"; Detail = "service is not defined in selected Compose configuration" }
}
- $baseArgs += @("-f", $composePath)
- docker compose @baseArgs @ComposeArgs
+
+ $psResult = Invoke-Compose -ComposeArgs @("ps", "--all", "-q", $Service)
+ if ($psResult.ExitCode -ne 0) { return [pscustomobject]@{ State = "failed"; Detail = "docker compose ps failed" } }
+ $ids = @($psResult.Output -split '\r?\n' | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" })
+ if ($ids.Count -eq 0) { return [pscustomobject]@{ State = "pending"; Detail = "container is not present yet" } }
+
+ $state = "ready"
+ $detail = "running and healthy"
+ foreach ($id in $ids) {
+ $inspect = Invoke-Docker -DockerArgs @("inspect", "--format", $script:InspectFormat, $id)
+ if ($inspect.ExitCode -ne 0) { return [pscustomobject]@{ State = "failed"; Detail = "container state inspection failed" } }
+ $parts = @($inspect.Output.Trim() -split '\|', 3)
+ if ($parts.Count -ne 3) { return [pscustomobject]@{ State = "failed"; Detail = "container state inspection returned invalid fields" } }
+ $status = $parts[0]
+ $health = $parts[1]
+ $exitCode = $parts[2]
+
+ if ($Mode -eq "migration") {
+ if ($status -eq "exited" -and $exitCode -eq "0") { continue }
+ if ($status -eq "running" -or $status -eq "created") {
+ if ($state -ne "failed") { $state = "pending" }
+ $detail = "migration status=$status"
+ continue
+ }
+ return [pscustomobject]@{ State = "failed"; Detail = "migration status=$status exitCode=$exitCode" }
+ }
+
+ if ($status -eq "running") {
+ if ($health -eq "healthy" -or $health -eq "none") { continue }
+ if ($health -eq "starting") {
+ if ($state -ne "failed") { $state = "pending" }
+ $detail = "container health is starting"
+ continue
+ }
+ if ($health -eq "unhealthy") { return [pscustomobject]@{ State = "failed"; Detail = "container health is unhealthy" } }
+ return [pscustomobject]@{ State = "failed"; Detail = "container health state is $health" }
+ }
+ if ($status -eq "created") {
+ if ($state -ne "failed") { $state = "pending" }
+ $detail = "container has not started yet"
+ continue
+ }
+ return [pscustomobject]@{ State = "failed"; Detail = "container status=$status" }
+ }
+ return [pscustomobject]@{ State = $state; Detail = $detail }
}
-function Test-ComposeServicesHealthy {
- param([string]$PsOutput)
- if ($PsOutput.Trim() -eq "") { return $false }
- if ($PsOutput -match '(?i)(exited|unhealthy|restarting|dead)') { return $false }
- return $true
+function Test-ComposeRuntime {
+ $services = @("api", "web", "db", "redis", "minio")
+ if ($script:ComposeServices -contains "migrate") { $services += "migrate" }
+ $modeByService = @{ migrate = "migration" }
+ $deadline = [DateTime]::UtcNow.AddSeconds($TimeoutSeconds)
+ $latest = @{}
+ $timedOut = $false
+
+ while ($true) {
+ $pending = $false
+ $failed = $false
+ foreach ($service in $services) {
+ $mode = if ($modeByService.ContainsKey($service)) { $modeByService[$service] } else { "runtime" }
+ $probe = Test-ComposeService -Service $service -Mode $mode
+ $latest[$service] = $probe
+ if ($probe.State -eq "pending") { $pending = $true }
+ if ($probe.State -eq "failed") { $failed = $true }
+ }
+ if ($failed -or -not $pending) { break }
+ if ([DateTime]::UtcNow -ge $deadline) { $timedOut = $true; break }
+ Start-Sleep -Seconds $PollIntervalSeconds
+ }
+
+ $passed = $true
+ foreach ($service in $services) {
+ $probe = $latest[$service]
+ if ($probe.State -eq "ready") {
+ Add-Result "runtime-$service" $true $probe.Detail
+ } else {
+ $passed = $false
+ $detail = $probe.Detail
+ if ($probe.State -eq "pending" -and $timedOut) { $detail = "timed out waiting: $detail" }
+ elseif ($probe.State -eq "pending") { $detail = "not ready: $detail" }
+ Add-Result "runtime-$service" $false $detail
+ }
+ }
+ return $passed
}
Push-Location $root
@@ -65,22 +189,20 @@ try {
if ($apiPort -eq "") { $apiPort = "8000" }
$webPort = [string]$envMap["WEB_PORT"]
if ($webPort -eq "") { $webPort = "3000" }
- if ($ApiUrl -eq "" -and $env:API_BASE_URL -ne "") {
+ if ($ApiUrl -eq "" -and -not [string]::IsNullOrEmpty($env:API_BASE_URL)) {
$ApiUrl = $env:API_BASE_URL
if ($ApiUrl -notmatch '/api/health$') { $ApiUrl = "$($ApiUrl.TrimEnd('/'))/health" }
}
- if ($WebUrl -eq "" -and $env:WEB_BASE_URL -ne "") {
- $WebUrl = $env:WEB_BASE_URL
- }
+ if ($WebUrl -eq "" -and -not [string]::IsNullOrEmpty($env:WEB_BASE_URL)) { $WebUrl = $env:WEB_BASE_URL }
if ($ApiUrl -eq "") { $ApiUrl = "http://localhost:$apiPort/api/health" }
if ($WebUrl -eq "") { $WebUrl = "http://localhost:$webPort/" }
- docker version | Out-Null
- Add-Result "docker" ($LASTEXITCODE -eq 0) "Docker CLI is available"
-
- Invoke-Compose -ComposeArgs @("config") | Out-Null
- Add-Result "compose-config" ($LASTEXITCODE -eq 0) "$ComposeFile is valid"
+ $docker = Invoke-Docker -DockerArgs @("version")
+ Add-Result "docker" ($docker.ExitCode -eq 0) "Docker CLI is available"
+ $config = Test-ComposeConfig
+ Add-Result "compose-config" $config.ConfigPassed "$ComposeFile is valid"
+ Add-Result "compose-services" $config.ServicesPassed "Compose service names are available"
Add-Result "env-file" (Test-Path $envPath) "$EnvFile exists"
foreach ($name in @("POSTGRES_PASSWORD", "JWT_SECRET", "MINIO_SECRET_KEY", "INIT_ADMIN_PASSWORD")) {
@@ -89,9 +211,7 @@ try {
Add-Result "secret-$name" $strong "length=$($value.Length)"
}
- $ps = Invoke-Compose -ComposeArgs @("ps")
- Add-Result "compose-ps" (($LASTEXITCODE -eq 0) -and (Test-ComposeServicesHealthy -PsOutput "$ps")) "services must be running and not unhealthy/exited"
-
+ Add-Result "compose-ps" (Test-ComposeRuntime) "required services are present and ready; migration exits successfully when declared"
Add-Result "api-health" (Test-Http -Url $ApiUrl) $ApiUrl
Add-Result "web-root" (Test-Http -Url $WebUrl) $WebUrl
@@ -104,9 +224,7 @@ try {
results = $results
}
$report | ConvertTo-Json -Depth 5 | Set-Content -Path (Join-Path $root "deploy-check-report.json") -Encoding UTF8
- if ($failed.Count -gt 0) {
- throw "$($failed.Count) deploy check(s) failed. See deploy-check-report.json"
- }
+ if ($failed.Count -gt 0) { throw "$($failed.Count) deploy check(s) failed. See deploy-check-report.json" }
} finally {
Pop-Location
}
diff --git a/scripts/deploy-check.sh b/scripts/deploy-check.sh
index d2c560b..b6bc486 100644
--- a/scripts/deploy-check.sh
+++ b/scripts/deploy-check.sh
@@ -1,5 +1,6 @@
#!/usr/bin/env sh
set -eu
+set -f
ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)"
COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.ha-lite.yml}"
@@ -11,7 +12,6 @@ case "$ENV_FILE" in
/*) ENV_PATH="$ENV_FILE" ;;
*) ENV_PATH="$ROOT/$ENV_FILE" ;;
esac
-
case "$COMPOSE_FILE" in
/*) COMPOSE_PATH="$COMPOSE_FILE" ;;
*) COMPOSE_PATH="$ROOT/$COMPOSE_FILE" ;;
@@ -30,12 +30,7 @@ env_value() {
check() {
name="$1"
shift
- if "$@"; then
- echo "[PASS] $name"
- else
- echo "[FAIL] $name"
- FAILED=$((FAILED + 1))
- fi
+ if "$@"; then echo "[PASS] $name"; else echo "[FAIL] $name"; FAILED=$((FAILED + 1)); fi
}
strong_secret() {
@@ -52,15 +47,170 @@ compose() {
fi
}
-compose_services_healthy() {
- output="$(compose ps 2>&1)" || return 1
- [ "$output" != "" ] || return 1
- printf '%s\n' "$output" | grep -Eiq '(exited|unhealthy|restarting|dead)' && return 1
- return 0
+compose_config_valid() { compose config >/dev/null 2>&1; }
+
+load_compose_services() {
+ CONFIG_SERVICES="$(compose config --services 2>/dev/null)" || return 1
+ [ -n "$CONFIG_SERVICES" ]
+}
+
+compose_has_service() {
+ [ -n "$CONFIG_SERVICES" ] && printf '%s\n' "$CONFIG_SERVICES" | grep -Fxq "$1"
+}
+
+INSPECT_FORMAT='{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}|{{.State.ExitCode}}'
+PROBE_STATE=''
+PROBE_DETAIL=''
+
+probe_service() {
+ probe_name="$1"
+ probe_mode="$2"
+ ids="$(compose ps --all -q "$probe_name" 2>/dev/null)" || {
+ PROBE_STATE=failed
+ PROBE_DETAIL='docker compose ps failed'
+ return
+ }
+ if [ -z "$ids" ]; then
+ PROBE_STATE=pending
+ PROBE_DETAIL='container is not present yet'
+ return
+ fi
+
+ PROBE_STATE=ready
+ PROBE_DETAIL='running and healthy'
+ for container_id in $ids; do
+ [ -n "$container_id" ] || continue
+ state_line="$(docker inspect --format "$INSPECT_FORMAT" "$container_id" 2>/dev/null)" || {
+ PROBE_STATE=failed
+ PROBE_DETAIL='container state inspection failed'
+ break
+ }
+ status="${state_line%%|*}"
+ rest="${state_line#*|}"
+ health="${rest%%|*}"
+ exit_code="${rest#*|}"
+ if [ "$status" = "$state_line" ] || [ "$health" = "$rest" ] || [ "$exit_code" = "$rest" ]; then
+ PROBE_STATE=failed
+ PROBE_DETAIL='container state inspection returned invalid fields'
+ break
+ fi
+
+ if [ "$probe_mode" = migration ]; then
+ if [ "$status" = exited ] && [ "$exit_code" = 0 ]; then
+ :
+ elif [ "$status" = running ] || [ "$status" = created ]; then
+ [ "$PROBE_STATE" = failed ] || PROBE_STATE=pending
+ PROBE_DETAIL="migration status=$status"
+ else
+ PROBE_STATE=failed
+ PROBE_DETAIL="migration status=$status exitCode=$exit_code"
+ break
+ fi
+ elif [ "$status" = running ]; then
+ case "$health" in
+ healthy|none) : ;;
+ starting)
+ [ "$PROBE_STATE" = failed ] || PROBE_STATE=pending
+ PROBE_DETAIL='container health is starting'
+ ;;
+ unhealthy)
+ PROBE_STATE=failed
+ PROBE_DETAIL='container health is unhealthy'
+ break
+ ;;
+ *)
+ PROBE_STATE=failed
+ PROBE_DETAIL="container health state is $health"
+ break
+ ;;
+ esac
+ elif [ "$status" = created ]; then
+ [ "$PROBE_STATE" = failed ] || PROBE_STATE=pending
+ PROBE_DETAIL='container has not started yet'
+ else
+ PROBE_STATE=failed
+ PROBE_DETAIL="container status=$status"
+ break
+ fi
+ done
}
-compose_config_valid() {
- compose config >/dev/null
+DEPLOY_CHECK_TIMEOUT_SECONDS="${DEPLOY_CHECK_TIMEOUT_SECONDS:-60}"
+DEPLOY_CHECK_POLL_SECONDS="${DEPLOY_CHECK_POLL_SECONDS:-3}"
+case "$DEPLOY_CHECK_TIMEOUT_SECONDS" in ''|*[!0-9]*) DEPLOY_CHECK_TIMEOUT_SECONDS=60 ;; esac
+case "$DEPLOY_CHECK_POLL_SECONDS" in ''|*[!0-9]*) DEPLOY_CHECK_POLL_SECONDS=3 ;; esac
+DEPLOY_CHECK_TIMEOUT_SECONDS="$(printf '%s' "$DEPLOY_CHECK_TIMEOUT_SECONDS" | sed 's/^0*//')"
+DEPLOY_CHECK_POLL_SECONDS="$(printf '%s' "$DEPLOY_CHECK_POLL_SECONDS" | sed 's/^0*//')"
+[ -n "$DEPLOY_CHECK_TIMEOUT_SECONDS" ] || DEPLOY_CHECK_TIMEOUT_SECONDS=0
+[ -n "$DEPLOY_CHECK_POLL_SECONDS" ] || DEPLOY_CHECK_POLL_SECONDS=0
+[ "${#DEPLOY_CHECK_TIMEOUT_SECONDS}" -le 3 ] || DEPLOY_CHECK_TIMEOUT_SECONDS=900
+[ "${#DEPLOY_CHECK_POLL_SECONDS}" -le 2 ] || DEPLOY_CHECK_POLL_SECONDS=60
+[ "$DEPLOY_CHECK_TIMEOUT_SECONDS" -le 900 ] || DEPLOY_CHECK_TIMEOUT_SECONDS=900
+[ "$DEPLOY_CHECK_POLL_SECONDS" -le 60 ] || DEPLOY_CHECK_POLL_SECONDS=60
+[ "$DEPLOY_CHECK_POLL_SECONDS" -gt 0 ] || DEPLOY_CHECK_POLL_SECONDS=1
+
+wait_for_compose_runtime() {
+ required_services='api web db redis minio'
+ migration_declared=0
+ compose_has_service migrate && migration_declared=1
+ runtime_report=''
+ runtime_pending=0
+ runtime_failure=0
+ timed_out=0
+ deadline=$(( $(date +%s) + DEPLOY_CHECK_TIMEOUT_SECONDS ))
+
+ while :; do
+ runtime_report=''
+ runtime_pending=0
+ runtime_failure=0
+ for runtime_service in $required_services; do
+ if compose_has_service "$runtime_service"; then
+ probe_service "$runtime_service" runtime
+ else
+ PROBE_STATE=failed
+ PROBE_DETAIL='service is not defined in selected Compose configuration'
+ fi
+ runtime_report="${runtime_report}${runtime_service}|${PROBE_STATE}|${PROBE_DETAIL}
+"
+ [ "$PROBE_STATE" = pending ] && runtime_pending=1
+ [ "$PROBE_STATE" = failed ] && runtime_failure=1
+ done
+
+ if [ "$migration_declared" -eq 1 ]; then
+ probe_service migrate migration
+ runtime_report="${runtime_report}migrate|${PROBE_STATE}|${PROBE_DETAIL}
+"
+ [ "$PROBE_STATE" = pending ] && runtime_pending=1
+ [ "$PROBE_STATE" = failed ] && runtime_failure=1
+ fi
+
+ [ "$runtime_failure" -eq 1 ] && break
+ [ "$runtime_pending" -eq 0 ] && break
+ if [ "$(date +%s)" -ge "$deadline" ]; then
+ timed_out=1
+ break
+ fi
+ sleep "$DEPLOY_CHECK_POLL_SECONDS"
+ done
+
+ runtime_passed=1
+ while IFS='|' read -r runtime_service outcome detail; do
+ [ -n "$runtime_service" ] || continue
+ if [ "$outcome" = ready ]; then
+ echo "[PASS] runtime-$runtime_service"
+ else
+ runtime_passed=0
+ if [ "$outcome" = pending ] && [ "$timed_out" -eq 1 ]; then
+ detail="timed out waiting: $detail"
+ elif [ "$outcome" = pending ]; then
+ detail="not ready: $detail"
+ fi
+ echo "[FAIL] runtime-$runtime_service - $detail"
+ fi
+ done < "$RESULTS" < pathEntries.map((entry) => path.join(entry, command)).find((candidate) => existsSync(candidate));
+const powershell = process.env.PWSH_PATH || findOnPath(process.platform === 'win32' ? 'pwsh.exe' : 'pwsh') ||
+ (process.platform === 'win32' ? findOnPath('powershell.exe') : undefined);
+const powershellProbe = powershell ? spawnSync(powershell, ['-NoProfile', '-Command', 'exit 0']) : undefined;
+const hasPowerShell = Boolean(powershell && !powershellProbe.error && powershellProbe.status === 0);
+
+const fakeDocker = String.raw`#!/usr/bin/env node
+import fs from 'node:fs';
+const args = process.argv.slice(2);
+const scenario = process.env.FAKE_DOCKER_CASE || 'passing';
+if (process.env.FAKE_DOCKER_LOG) fs.appendFileSync(process.env.FAKE_DOCKER_LOG, JSON.stringify(args) + '\n');
+if (args[0] === 'version') process.exit(0);
+if (args[0] === 'inspect') {
+ const id = args[args.length - 1];
+ const service = id.replace(/^id-/, '');
+ let status = 'running';
+ let health = 'healthy';
+ let code = '0';
+ if (scenario === 'exited' && service === 'api') { status = 'exited'; health = 'none'; code = '1'; }
+ if (scenario === 'starting' && service === 'api') health = 'starting';
+ if (scenario === 'starting-then-healthy' && service === 'api') {
+ const log = process.env.FAKE_DOCKER_LOG;
+ const priorCalls = log && fs.existsSync(log)
+ ? fs.readFileSync(log, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line))
+ .filter((call) => call[0] === 'inspect' && call[call.length - 1] === 'id-api').length
+ : 0;
+ if (priorCalls === 1) health = 'starting';
+ }
+ if (scenario === 'unhealthy' && service === 'db') health = 'unhealthy';
+ if (service === 'migrate') {
+ status = scenario === 'migration-running' ? 'running' : 'exited';
+ health = 'none';
+ code = scenario === 'migration-failed' ? '7' : '0';
+ }
+ process.stdout.write([status, health, code].join('|') + '\n');
+ process.exit(0);
+}
+if (args[0] === 'compose') {
+ const composeArgs = args.slice(1);
+ if (composeArgs.includes('config')) {
+ if (composeArgs.includes('--services')) {
+ let services = (process.env.FAKE_COMPOSE_SERVICES || 'api,web,db,redis,minio,migrate,nginx').split(',');
+ if (scenario === 'missing-config-service') services = services.filter((name) => name !== 'api');
+ process.stdout.write(services.join('\n') + '\n');
+ }
+ process.exit(0);
+ }
+ if (composeArgs.includes('ps')) {
+ const q = composeArgs.indexOf('-q');
+ const service = q >= 0 ? composeArgs[q + 1] : '';
+ if (((scenario === 'missing-service' && service === 'api') || (scenario === 'migration-missing' && service === 'migrate')) || !service || service === 'nginx') process.exit(0);
+ process.stdout.write('id-' + service + '\n');
+ process.exit(0);
+ }
+}
+process.stderr.write('unexpected fake docker arguments\n');
+process.exit(2);
+`;
+
+function fixture() {
+ const root = mkdtempSync(path.join(tmpdir(), 'oneerp-deploy-check-'));
+ const scripts = path.join(root, 'scripts');
+ const bin = path.join(root, 'bin');
+ mkdirSync(scripts);
+ mkdirSync(bin);
+ copyFileSync(path.join(repoRoot, 'scripts', 'deploy-check.sh'), path.join(scripts, 'deploy-check.sh'));
+ copyFileSync(path.join(repoRoot, 'scripts', 'deploy-check.ps1'), path.join(scripts, 'deploy-check.ps1'));
+ writeFileSync(path.join(root, 'selected-compose.yml'), 'services: {}\n');
+ writeFileSync(path.join(root, 'docker-compose.ha-lite.yml'), 'services: {}\n');
+ const envFile = path.join(root, 'selected.env');
+ const secretValues = [
+ 'pg-secret-value-for-test-0001',
+ 'jwt-secret-value-for-test-0002',
+ 'minio-secret-value-for-test-003',
+ 'admin-secret-value-for-test-004',
+ ];
+ writeFileSync(envFile, [
+ `POSTGRES_PASSWORD=${secretValues[0]}`,
+ `JWT_SECRET=${secretValues[1]}`,
+ `MINIO_SECRET_KEY=${secretValues[2]}`,
+ `INIT_ADMIN_PASSWORD=${secretValues[3]}`,
+ 'API_PORT=8000',
+ 'WEB_PORT=3000',
+ ].join('\n') + '\n');
+ copyFileSync(envFile, path.join(root, '.env'));
+ const fakeDockerPath = path.join(bin, 'docker.mjs');
+ writeFileSync(fakeDockerPath, fakeDocker);
+ if (process.platform === 'win32') {
+ writeFileSync(path.join(bin, 'docker.cmd'), '@echo off\r\nnode "%~dp0docker.mjs" %*\r\n');
+ } else {
+ const dockerPath = path.join(bin, 'docker');
+ writeFileSync(dockerPath, '#!/bin/sh\nexec node "$(dirname "$0")/docker.mjs" "$@"\n');
+ chmodSync(dockerPath, 0o755);
+ const curlPath = path.join(bin, 'curl');
+ writeFileSync(curlPath, '#!/bin/sh\nexit 0\n');
+ chmodSync(curlPath, 0o755);
+ }
+ return { root, bin, envFile, composeFile: path.join(root, 'selected-compose.yml'), secretValues };
+}
+
+function runShell(fx, scenario, timeoutSeconds = 0) {
+ const log = path.join(fx.root, 'docker-args.jsonl');
+ const result = spawnSync('sh', [path.join(fx.root, 'scripts', 'deploy-check.sh')], {
+ cwd: fx.root,
+ encoding: 'utf8',
+ env: {
+ ...process.env,
+ PATH: `${fx.bin}${path.delimiter}${process.env.PATH || ''}`,
+ COMPOSE_FILE: 'selected-compose.yml',
+ ENV_FILE: 'selected.env',
+ DEPLOY_CHECK_TIMEOUT_SECONDS: String(timeoutSeconds),
+ DEPLOY_CHECK_POLL_SECONDS: '1',
+ FAKE_DOCKER_CASE: scenario,
+ FAKE_DOCKER_LOG: log,
+ },
+ });
+ return { ...result, log, report: readFileSync(path.join(fx.root, 'deploy-check-report.json'), 'utf8') };
+}
+
+function runPowerShell(fx, scenario, timeoutSeconds = 0, selectedFiles = true) {
+ const log = path.join(fx.root, 'docker-args.jsonl');
+ const script = path.join(fx.root, 'scripts', 'deploy-check.ps1').replaceAll("'", "''");
+ const command = `function Invoke-WebRequest { param($Uri, [switch]$UseBasicParsing, $TimeoutSec) [pscustomobject]@{ StatusCode = 200 } }; & '${script}' -TimeoutSeconds ${timeoutSeconds}`;
+ const env = {
+ ...process.env,
+ PATH: `${fx.bin}${path.delimiter}${process.env.PATH || ''}`,
+ COMPOSE_FILE: 'selected-compose.yml',
+ ENV_FILE: 'selected.env',
+ FAKE_DOCKER_CASE: scenario,
+ FAKE_DOCKER_LOG: log,
+ };
+ if (!selectedFiles) {
+ delete env.COMPOSE_FILE;
+ delete env.ENV_FILE;
+ delete env.API_BASE_URL;
+ delete env.WEB_BASE_URL;
+ }
+ const result = spawnSync(powershell, ['-NoProfile', '-Command', command], {
+ cwd: fx.root,
+ encoding: 'utf8',
+ env,
+ });
+ return { ...result, log, report: readFileSync(path.join(fx.root, 'deploy-check-report.json'), 'utf8') };
+}
+
+function assertNoSecretLeak(result, secretValues) {
+ const combined = `${result.stdout || ''}\n${result.stderr || ''}\n${result.report}`;
+ for (const value of secretValues) assert.equal(combined.includes(value), false, 'secret value was emitted');
+}
+
+const scenarios = [
+ { name: 'missing service container', scenario: 'missing-service', output: 'runtime-api', detail: 'not present' },
+ { name: 'service exited', scenario: 'exited', output: 'runtime-api', detail: 'status=exited' },
+ { name: 'health still starting at the deadline', scenario: 'starting', output: 'runtime-api', detail: 'timed out' },
+ { name: 'service unhealthy', scenario: 'unhealthy', output: 'runtime-db', detail: 'unhealthy' },
+ { name: 'migration exit code is nonzero', scenario: 'migration-failed', output: 'runtime-migrate', detail: 'exitCode=7' },
+ { name: 'declared migration container is missing', scenario: 'migration-missing', output: 'runtime-migrate', detail: 'not present' },
+ { name: 'declared migration is still running at the deadline', scenario: 'migration-running', output: 'runtime-migrate', detail: 'status=running' },
+ { name: 'required service missing from selected config', scenario: 'missing-config-service', output: 'runtime-api', detail: 'not defined' },
+];
+
+for (const { name, scenario, output, detail } of scenarios) {
+ test(`deploy-check.sh fails when ${name}`, (t) => {
+ const fx = fixture();
+ t.after(() => rmSync(fx.root, { recursive: true, force: true }));
+ const result = runShell(fx, scenario);
+ assert.notEqual(result.status, 0, result.stdout);
+ assert.match(result.stdout, new RegExp(`\\[FAIL\\] ${output}`));
+ assert.match(result.stdout, new RegExp(detail));
+ assertNoSecretLeak(result, fx.secretValues);
+ });
+
+ test(`deploy-check.ps1 fails when ${name}`, { skip: !hasPowerShell }, (t) => {
+ const fx = fixture();
+ t.after(() => rmSync(fx.root, { recursive: true, force: true }));
+ const result = runPowerShell(fx, scenario);
+ assert.notEqual(result.status, 0, result.stdout);
+ assert.match(result.stdout, new RegExp(`\\[FAIL\\] runtime-${output.replace('runtime-', '')}`));
+ assert.match(result.stdout, new RegExp(detail));
+ assertNoSecretLeak(result, fx.secretValues);
+ });
+}
+
+test('deploy-check.sh passes for healthy runtime services and a successful one-shot migration', (t) => {
+ const fx = fixture();
+ t.after(() => rmSync(fx.root, { recursive: true, force: true }));
+ const result = runShell(fx, 'passing');
+ assert.equal(result.status, 0, result.stdout + result.stderr);
+ assert.match(result.stdout, /\[PASS\] runtime-api/);
+ assert.match(result.stdout, /\[PASS\] runtime-migrate/);
+ assert.match(result.report, /"failed": 0/);
+ const calls = readFileSync(result.log, 'utf8').trim().split('\n').map((line) => JSON.parse(line));
+ assert.ok(calls.some((args) => args.includes('--env-file') && args.includes(fx.envFile)));
+ assert.ok(calls.some((args) => args.includes('-f') && args.includes(fx.composeFile)));
+ assert.ok(calls.some((args) => args.includes('--all') && args.includes('-q')));
+ assert.equal(calls.some((args) => args.includes('ps') && args.includes('nginx')), false, 'profile-only nginx must not be required');
+ assertNoSecretLeak(result, fx.secretValues);
+});
+
+test('deploy-check.ps1 passes for healthy runtime services and a successful one-shot migration', { skip: !hasPowerShell }, (t) => {
+ const fx = fixture();
+ t.after(() => rmSync(fx.root, { recursive: true, force: true }));
+ const result = runPowerShell(fx, 'passing');
+ assert.equal(result.status, 0, result.stdout + result.stderr);
+ assert.match(result.stdout, /\[PASS\] runtime-api/);
+ assert.match(result.stdout, /\[PASS\] runtime-migrate/);
+ const calls = readFileSync(result.log, 'utf8').trim().split('\n').map((line) => JSON.parse(line));
+ assert.ok(calls.some((args) => args.includes('--env-file') && args.includes(fx.envFile)));
+ assert.ok(calls.some((args) => args.includes('-f') && args.includes(fx.composeFile)));
+ assert.ok(calls.some((args) => args.includes('--all') && args.includes('-q')));
+ assert.equal(calls.some((args) => args.includes('ps') && args.includes('nginx')), false, 'profile-only nginx must not be required');
+ assertNoSecretLeak(result, fx.secretValues);
+});
+
+test('deploy-check.sh waits for health starting to become healthy', (t) => {
+ const fx = fixture();
+ t.after(() => rmSync(fx.root, { recursive: true, force: true }));
+ const result = runShell(fx, 'starting-then-healthy', 5);
+ assert.equal(result.status, 0, result.stdout + result.stderr);
+ assert.match(result.stdout, /\[PASS\] runtime-api/);
+});
+
+test('deploy-check.ps1 waits for health starting to become healthy', { skip: !hasPowerShell }, (t) => {
+ const fx = fixture();
+ t.after(() => rmSync(fx.root, { recursive: true, force: true }));
+ const result = runPowerShell(fx, 'starting-then-healthy', 5);
+ assert.equal(result.status, 0, result.stdout + result.stderr);
+ assert.match(result.stdout, /\[PASS\] runtime-api/);
+});
+
+test('deploy-check.ps1 uses HA-lite and .env defaults when Compose env vars are unset', { skip: !hasPowerShell }, (t) => {
+ const fx = fixture();
+ t.after(() => rmSync(fx.root, { recursive: true, force: true }));
+ const result = runPowerShell(fx, 'passing', 0, false);
+ assert.equal(result.status, 0, result.stdout + result.stderr);
+ const calls = readFileSync(result.log, 'utf8').trim().split('\n').map((line) => JSON.parse(line));
+ assert.ok(calls.some((args) => args.includes('-f') && args.includes(path.join(fx.root, 'docker-compose.ha-lite.yml'))));
+ assert.ok(calls.some((args) => args.includes('--env-file') && args.includes(path.join(fx.root, '.env'))));
+});