diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml deleted file mode 100644 index fdf235d..0000000 --- a/.github/workflows/build-docker-image.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Create and publish pixelpact docker image - -on: - push: - branches: - - main - create: - tags: - - v* - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - build-and-push-image: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Log in to the Container registry - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Build and push Docker image - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 - with: - context: server - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3a8cdcf --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,159 @@ +name: CI + +on: [push] + +# Only the latest push per ref is interesting. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install Nix + uses: nixbuild/nix-quick-install-action@9f63be77f412a248c9d9a65a4c82cf066cdf8f0c # v35 + with: + # Keep dev shell inputs alive in the store so they end up in the cache. + nix_conf: | + keep-env-derivations = true + keep-outputs = true + + - name: Restore and save Nix store + uses: nix-community/cache-nix-action@7df957e333c1e5da7721f60227dbba6d06080569 # v7 + with: + primary-key: nix-${{ runner.os }}-${{ hashFiles('flake.lock', 'flake.nix', 'nix/**.nix') }} + restore-prefixes-first-match: nix-${{ runner.os }}- + # Keep the cache small enough to stay fast to up- and download. + gc-max-store-size-linux: 1G + purge: true + purge-prefixes: nix-${{ runner.os }}- + purge-created: 0 + purge-primary-key: never + + - name: Build dev shell + run: nix develop --command true + + - name: Restore npm cache and playwright browsers + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: | + ~/.npm + ~/.cache/ms-playwright + key: deps-${{ runner.os }}-${{ hashFiles('server/package-lock.json') }} + restore-keys: deps-${{ runner.os }}- + + - name: Check formatting + run: nix develop --command treefmt --ci + + - name: Install dependencies + run: nix develop --command bash -c 'cd server && npm ci --prefer-offline --no-audit --no-fund' + + - name: Install chromium + run: nix develop --command bash -c 'cd server && npx playwright install chromium-headless-shell' + + - name: Run Tests + run: nix develop --command bash -c 'cd server && npm test' + + # Builds the image, proves it works end to end against the golden samples, and + # only then publishes it. Build, test and push share one job so the artifact + # that gets pushed is byte-for-byte the one the acceptance suite just drove. + docker-image: + needs: test + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + # The image under test ships its own browsers; playwright is only a + # transitive install here. + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install Nix + uses: nixbuild/nix-quick-install-action@9f63be77f412a248c9d9a65a4c82cf066cdf8f0c # v35 + with: + nix_conf: | + keep-env-derivations = true + keep-outputs = true + + - name: Restore Nix store + uses: nix-community/cache-nix-action@7df957e333c1e5da7721f60227dbba6d06080569 # v7 + with: + primary-key: nix-${{ runner.os }}-${{ hashFiles('flake.lock', 'flake.nix', 'nix/**.nix') }} + restore-prefixes-first-match: nix-${{ runner.os }}- + # The test job owns this cache; this job only reads it. + save: false + + - name: Build dev shell + run: nix develop --command true + + - name: Restore npm cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.npm + key: npm-${{ runner.os }}-${{ hashFiles('server/package-lock.json') }} + restore-keys: npm-${{ runner.os }}- + + # Runs before the build so the labels are baked into the image we test. + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build Docker image + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: server + push: false + load: true + tags: pixelpact:ci + labels: ${{ steps.meta.outputs.labels }} + + - name: Install dependencies + run: nix develop --command bash -c 'cd server && npm ci --prefer-offline --no-audit --no-fund' + + - name: Run acceptance tests + env: + PIXELPACT_IMAGE: pixelpact:ci + run: nix develop --command bash -c 'cd server && npm run test:acceptance' + + - name: Upload acceptance failure artifacts + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: acceptance-failures + path: server/acceptance/out/ + if-no-files-found: ignore + + - name: Log in to the Container registry + if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Pushes the exact image the acceptance suite just verified, rather than + # rebuilding it. + - name: Push Docker image + if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') + env: + TAGS: ${{ steps.meta.outputs.tags }} + run: | + printf '%s\n' "$TAGS" | while IFS= read -r tag; do + [ -n "$tag" ] || continue + docker tag pixelpact:ci "$tag" + docker push "$tag" + done diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index d266138..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Run tests - -on: [push] - -# Only the latest push per ref is interesting. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Install Nix - uses: nixbuild/nix-quick-install-action@9f63be77f412a248c9d9a65a4c82cf066cdf8f0c # v35 - with: - # Keep dev shell inputs alive in the store so they end up in the cache. - nix_conf: | - keep-env-derivations = true - keep-outputs = true - - - name: Restore and save Nix store - uses: nix-community/cache-nix-action@7df957e333c1e5da7721f60227dbba6d06080569 # v7 - with: - primary-key: nix-${{ runner.os }}-${{ hashFiles('flake.lock', 'flake.nix', 'nix/**.nix') }} - restore-prefixes-first-match: nix-${{ runner.os }}- - # Keep the cache small enough to stay fast to up- and download. - gc-max-store-size-linux: 1G - purge: true - purge-prefixes: nix-${{ runner.os }}- - purge-created: 0 - purge-primary-key: never - - - name: Build dev shell - run: nix develop --command true - - - name: Restore npm cache and playwright browsers - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: | - ~/.npm - ~/.cache/ms-playwright - key: deps-${{ runner.os }}-${{ hashFiles('server/package-lock.json') }} - restore-keys: deps-${{ runner.os }}- - - - name: Check formatting - run: nix develop --command treefmt --ci - - - name: Install dependencies - run: nix develop --command bash -c 'cd server && npm ci --prefer-offline --no-audit --no-fund' - - - name: Install chromium - run: nix develop --command bash -c 'cd server && npx playwright install chromium-headless-shell' - - - name: Run Tests - run: nix develop --command bash -c 'cd server && npm test' diff --git a/.gitignore b/.gitignore index 1e1be6b..8f76519 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ node_modules/ .idea pixelpact/coverage -.direnv/ \ No newline at end of file +.direnv/ +server/acceptance/out/ diff --git a/docs/developer-setup.md b/docs/developer-setup.md index 0fb867e..86952b6 100644 --- a/docs/developer-setup.md +++ b/docs/developer-setup.md @@ -12,3 +12,15 @@ npm ci npx playwright install chromium-headless-shell start-server ``` + +## Tests + +```bash +cd server +npm test # unit and integration tests against src/ +npm run test:acceptance # black-box tests against the docker image +``` + +The acceptance suite builds the image and drives it over HTTP against a set of +golden samples. It needs docker, and it is what gates the published image in CI. +See [server/acceptance/README.md](../server/acceptance/README.md). diff --git a/nix/devshell.nix b/nix/devshell.nix index 2b09335..ed6b00c 100644 --- a/nix/devshell.nix +++ b/nix/devshell.nix @@ -16,7 +16,8 @@ in { programs.alejandra.enable = true; programs.prettier.enable = true; programs.prettier.package = pkgs.prettier; - settings.global.excludes = ["*-lock.json"]; + # Acceptance goldens are generated output; prettier must not rewrite them. + settings.global.excludes = ["*-lock.json" "server/acceptance/cases/*/golden/*"]; }; packages = [pkgs.nodejs]; diff --git a/server/acceptance/README.md b/server/acceptance/README.md new file mode 100644 index 0000000..0620a8d --- /dev/null +++ b/server/acceptance/README.md @@ -0,0 +1,171 @@ +# Acceptance tests + +Black-box tests that drive the **docker image** over HTTP against a growing set +of golden samples. They exist so a dependency bump that changes rendering, the +API, or the logging cannot reach ghcr.io unnoticed — which is what makes +Renovate's automerge safe. + +Unit and integration tests (`npm test`) cover `src/` on the host. These cover +the artifact users actually run. Nothing in here imports from `src/`. + +## Running + +```bash +npm run test:acceptance # assert against the goldens +npm run test:acceptance -- -u # regenerate them +``` + +The image is built once per run and left cached. To drive an image you already +have — this is what CI does — pass it in: + +```bash +PIXELPACT_IMAGE=pixelpact:ci npm run test:acceptance +``` + +An image passed in this way is never rebuilt or removed. + +Goldens follow vitest's own snapshot rules: `-u` rewrites them, a normal local +run fills in whatever is missing, and on CI a missing golden is a failure. + +## What a case looks like + +One directory per case, and the case _is_ a vitest test. It says what to do; it +asserts nothing. + +```js +import { pixelpactCase } from "../../lib/pixelpact.js"; + +const { scenario, render, check, fixture } = pixelpactCase(import.meta.dirname); +const viewport = { width: 800, height: 600 }; + +scenario("checks a rendered page against itself", async () => { + const actualHtml = fixture("input.mhtml"); + + const reference = await render({ actualHtml, viewport }); + await check({ actualHtml, expected: reference.png, viewport }); +}); +``` + +`pixelpactCase(dir, env?)` gives you: + +| | | +| ----------------------------------------------------- | --------------------------------------- | +| `scenario(name, body)` | `it` plus the golden pinning | +| `render({ actualHtml, viewport, fullpage?, style? })` | `{ status, body, png }` | +| `check({ actualHtml, expected, viewport, ... })` | `{ status, body, png, expected, diff }` | +| `post(route, body)` | raw escape hatch | +| `fixture(relativePath)` | a `Buffer` read next to the case | + +Buffers in, Buffers out — base64 never appears in a case. Nothing throws on a +non-2xx; the status is simply part of what gets pinned. `env` defaults to +`{ LOG_LEVEL: "debug" }`. + +Each test gets its own container on an ephemeral port, so the whole log stream +belongs to that one test and cases cannot leak into each other. + +### Two rules + +- **One `scenario` per case file.** Goldens are flat under `golden/`, so a second + one would collide. A variant is a new case directory. +- **Never hand-edit `golden/`.** It is generated output; review it in the diff. + +## What gets pinned + +Everything the case did, without the case naming any of it: + +``` +cases/form/golden/ + transcript.json every request and response, payloads elided + 01-render.actual.png each response payload, byte-exact + 02-check.actual.png + 02-check.expected.png + 02-check.diff.png + logs.jsonl the container's whole log stream, normalized +``` + +`transcript.json` is a list of `requests`, each with its `response`, plus the +container's exit code — so a broken shutdown path shows up too: + +```json +{ + "requests": [ + { + "method": "POST", + "path": "/render", + "body": { "actualHtml": "", "viewport": {...} }, + "response": { + "status": 200, + "body": { "actual": " 01-render.actual.png>" } + } + } + ], + "container": { "exitCode": 0 } +} +``` + +Payloads over 256 bytes become that descriptor. A **response** payload also gets +written out as its own byte-exact file. A **request** payload gets the descriptor +only — it is either a fixture in this repository or an earlier response, and the +sha is what makes that linkage visible (above, the next request's `expected` sha +is the same `db92d1ea66e2`). + +The transcript overlaps the logs on purpose: the logs are the server's +self-report, the transcript is what a client observed. When only the logs move, +logging regressed — which is not something a log golden can tell you on its own. +The transcript is also the only record of the **response body's shape**: a +dropped or added response key changes nothing in the logs. + +### Why byte-exact pixels work + +Goldens are only ever generated inside the image. There is no host-rendering +path, so fontconfig and the bundled browser are fixed and a render is +reproducible. A PNG that changes by one byte is a real change worth looking at. + +### What the logs drop, and why + +`time`, `pid` and `hostname` go away. Values that differ on every run are +replaced by a placeholder rather than removed, so the _field_ stays pinned and +one appearing or disappearing is still a failure: + +- `responseTime` and anything `*DurationMs` → `` +- `req.host` → ``, `remoteAddress` → `
`, `remotePort` → `` +- the render workspace (a fresh `mkdtemp`) → `` +- the listen address → `
` + +Stable values stay verbatim on purpose. `browserVersion` is the useful one: it +is how a playwright bump announces itself. + +A line that is not JSON is kept as `{"raw": "..."}` rather than dropped — a +stray `console.log` or a half-formatted stack is broken logging and should fail. +Anything on stderr is marked `{"stream": "stderr"}`; it means a crash, not +logging. + +Readiness is detected by waiting for the server's own "Server listening" line +instead of probing a route, so no harness request pollutes the log golden or +shifts the request ids. A case therefore has to run at a level where `info` is +emitted. + +## Adding a case + +1. `mkdir cases//` and drop in an `input.mhtml`. +2. Write `case.test.js` as above. +3. `npm run test:acceptance -- -u`. +4. **Read the generated goldens** before committing. That review is the test. + +To produce an `input.mhtml`, either save a page from Chrome DevTools +(⋮ → More tools → Save page as → Webpage, Single File) or call the CDP +`Page.captureSnapshot` the way `clients/js/playwright` does. + +`.gitattributes` pins `*.mhtml` to CRLF — MIME boundary parsing depends on it, +so keep an editor from rewriting the line endings. + +A case that needs no fixture of its own can read another's: +`fixture("../plain/input.mhtml")`. What a case must **not** do is read another +case's golden — every case renders its own reference, so goldens stay pure +outputs that regenerate in any order. + +## When something fails + +Text goldens diff inline in the vitest output. For a PNG, both sides plus a +pixelmatch visualisation land in `acceptance/out//` (gitignored; CI +uploads it as the `acceptance-failures` artifact). diff --git a/server/acceptance/cases/default-log-level/case.test.js b/server/acceptance/cases/default-log-level/case.test.js new file mode 100644 index 0000000..505c941 --- /dev/null +++ b/server/acceptance/cases/default-log-level/case.test.js @@ -0,0 +1,13 @@ +import { pixelpactCase } from "../../lib/pixelpact.js"; + +// Every other case runs at debug. This one runs at the level the image actually +// ships, so the goldens pin what an operator sees in production - and a render +// that starts warning, or stops reporting itself, shows up here. +const { scenario, render, fixture } = pixelpactCase(import.meta.dirname, { + LOG_LEVEL: "info", +}); +const viewport = { width: 800, height: 600 }; + +scenario("logs a request and its result at the shipped log level", async () => { + await render({ actualHtml: fixture("../plain/input.mhtml"), viewport }); +}); diff --git a/server/acceptance/cases/default-log-level/golden/01-render.actual.png b/server/acceptance/cases/default-log-level/golden/01-render.actual.png new file mode 100644 index 0000000..a60d7de Binary files /dev/null and b/server/acceptance/cases/default-log-level/golden/01-render.actual.png differ diff --git a/server/acceptance/cases/default-log-level/golden/logs.jsonl b/server/acceptance/cases/default-log-level/golden/logs.jsonl new file mode 100644 index 0000000..b8bead3 --- /dev/null +++ b/server/acceptance/cases/default-log-level/golden/logs.jsonl @@ -0,0 +1,7 @@ +{"level":"info","name":"pixelpact","msg":"Server listening at
"} +{"level":"info","name":"pixelpact","msg":"Server listening at
"} +{"level":"info","name":"pixelpact","reqId":"req-1","req":{"method":"POST","url":"/render","host":"","remoteAddress":"
","remotePort":""},"msg":"incoming request"} +{"level":"info","name":"pixelpact","reqId":"req-1","viewport":{"width":800,"height":600},"fullpage":false,"mhtmlConverter":true,"styled":false,"actualBytes":9497,"renderDurationMs":"","msg":"Render completed"} +{"level":"info","name":"pixelpact","reqId":"req-1","res":{"statusCode":200},"responseTime":"","msg":"request completed"} +{"level":"info","name":"pixelpact","signal":"SIGTERM","msg":"Shutting down"} +{"level":"info","name":"pixelpact","msg":"Shutdown complete"} diff --git a/server/acceptance/cases/default-log-level/golden/transcript.json b/server/acceptance/cases/default-log-level/golden/transcript.json new file mode 100644 index 0000000..2e2d250 --- /dev/null +++ b/server/acceptance/cases/default-log-level/golden/transcript.json @@ -0,0 +1,24 @@ +{ + "requests": [ + { + "method": "POST", + "path": "/render", + "body": { + "actualHtml": "", + "viewport": { + "width": 800, + "height": 600 + } + }, + "response": { + "status": 200, + "body": { + "actual": " 01-render.actual.png>" + } + } + } + ], + "container": { + "exitCode": 0 + } +} diff --git a/server/acceptance/cases/form/case.test.js b/server/acceptance/cases/form/case.test.js new file mode 100644 index 0000000..97ee067 --- /dev/null +++ b/server/acceptance/cases/form/case.test.js @@ -0,0 +1,15 @@ +import { pixelpactCase } from "../../lib/pixelpact.js"; + +const { scenario, render, check, fixture } = pixelpactCase(import.meta.dirname); +const viewport = { width: 800, height: 600 }; + +// Blink captures a stylesheet as its own MIME part; this is the shape the real client sends. +scenario( + "checks a document with a CSS sub-resource against its own render", + async () => { + const actualHtml = fixture("input.mhtml"); + + const reference = await render({ actualHtml, viewport }); + await check({ actualHtml, expected: reference.png, viewport }); + }, +); diff --git a/server/acceptance/cases/form/golden/01-render.actual.png b/server/acceptance/cases/form/golden/01-render.actual.png new file mode 100644 index 0000000..e472d7f Binary files /dev/null and b/server/acceptance/cases/form/golden/01-render.actual.png differ diff --git a/server/acceptance/cases/form/golden/02-check.actual.png b/server/acceptance/cases/form/golden/02-check.actual.png new file mode 100644 index 0000000..e472d7f Binary files /dev/null and b/server/acceptance/cases/form/golden/02-check.actual.png differ diff --git a/server/acceptance/cases/form/golden/02-check.diff.png b/server/acceptance/cases/form/golden/02-check.diff.png new file mode 100644 index 0000000..a1572ad Binary files /dev/null and b/server/acceptance/cases/form/golden/02-check.diff.png differ diff --git a/server/acceptance/cases/form/golden/02-check.expected.png b/server/acceptance/cases/form/golden/02-check.expected.png new file mode 100644 index 0000000..e472d7f Binary files /dev/null and b/server/acceptance/cases/form/golden/02-check.expected.png differ diff --git a/server/acceptance/cases/form/golden/logs.jsonl b/server/acceptance/cases/form/golden/logs.jsonl new file mode 100644 index 0000000..bbc1464 --- /dev/null +++ b/server/acceptance/cases/form/golden/logs.jsonl @@ -0,0 +1,22 @@ +{"level":"info","name":"pixelpact","msg":"Server listening at
"} +{"level":"info","name":"pixelpact","msg":"Server listening at
"} +{"level":"info","name":"pixelpact","reqId":"req-1","req":{"method":"POST","url":"/render","host":"","remoteAddress":"
","remotePort":""},"msg":"incoming request"} +{"level":"debug","name":"pixelpact","reqId":"req-1","mhtmlBytes":2151,"workspaceDirectory":"","msg":"Rendering page source"} +{"level":"debug","name":"pixelpact","reqId":"req-1","indexFile":"/index.html","htmlBytes":1768,"msg":"Converted MHTML to HTML"} +{"level":"debug","name":"pixelpact","reqId":"req-1","browserVersion":"153.0.8010.12","durationMs":"","msg":"Browser launched"} +{"level":"debug","name":"pixelpact","reqId":"req-1","url":"file:///index.html","msg":"Loading page"} +{"level":"debug","name":"pixelpact","reqId":"req-1","url":"file:///index.html","loadDurationMs":"","screenshotDurationMs":"","bytes":13500,"msg":"Screenshot taken"} +{"level":"debug","name":"pixelpact","reqId":"req-1","msg":"Browser closed"} +{"level":"info","name":"pixelpact","reqId":"req-1","viewport":{"width":800,"height":600},"fullpage":false,"mhtmlConverter":true,"styled":false,"actualBytes":13500,"renderDurationMs":"","msg":"Render completed"} +{"level":"info","name":"pixelpact","reqId":"req-1","res":{"statusCode":200},"responseTime":"","msg":"request completed"} +{"level":"info","name":"pixelpact","reqId":"req-2","req":{"method":"POST","url":"/check","host":"","remoteAddress":"
","remotePort":""},"msg":"incoming request"} +{"level":"debug","name":"pixelpact","reqId":"req-2","mhtmlBytes":2151,"workspaceDirectory":"","msg":"Rendering page source"} +{"level":"debug","name":"pixelpact","reqId":"req-2","indexFile":"/index.html","htmlBytes":1768,"msg":"Converted MHTML to HTML"} +{"level":"debug","name":"pixelpact","reqId":"req-2","browserVersion":"153.0.8010.12","durationMs":"","msg":"Browser launched"} +{"level":"debug","name":"pixelpact","reqId":"req-2","url":"file:///index.html","msg":"Loading page"} +{"level":"debug","name":"pixelpact","reqId":"req-2","url":"file:///index.html","loadDurationMs":"","screenshotDurationMs":"","bytes":13500,"msg":"Screenshot taken"} +{"level":"debug","name":"pixelpact","reqId":"req-2","msg":"Browser closed"} +{"level":"info","name":"pixelpact","reqId":"req-2","viewport":{"width":800,"height":600},"fullpage":false,"mhtmlConverter":true,"styled":false,"expectedBytes":13500,"actualBytes":13500,"numDiffPixels":0,"renderDurationMs":"","compareDurationMs":"","msg":"Check completed"} +{"level":"info","name":"pixelpact","reqId":"req-2","res":{"statusCode":200},"responseTime":"","msg":"request completed"} +{"level":"info","name":"pixelpact","signal":"SIGTERM","msg":"Shutting down"} +{"level":"info","name":"pixelpact","msg":"Shutdown complete"} diff --git a/server/acceptance/cases/form/golden/transcript.json b/server/acceptance/cases/form/golden/transcript.json new file mode 100644 index 0000000..c73b852 --- /dev/null +++ b/server/acceptance/cases/form/golden/transcript.json @@ -0,0 +1,45 @@ +{ + "requests": [ + { + "method": "POST", + "path": "/render", + "body": { + "actualHtml": "", + "viewport": { + "width": 800, + "height": 600 + } + }, + "response": { + "status": 200, + "body": { + "actual": " 01-render.actual.png>" + } + } + }, + { + "method": "POST", + "path": "/check", + "body": { + "actualHtml": "", + "expected": "", + "viewport": { + "width": 800, + "height": 600 + } + }, + "response": { + "status": 200, + "body": { + "actual": " 02-check.actual.png>", + "expected": " 02-check.expected.png>", + "diff": " 02-check.diff.png>", + "numDiffPixels": 0 + } + } + } + ], + "container": { + "exitCode": 0 + } +} diff --git a/server/testdata/template.mhtml b/server/acceptance/cases/form/input.mhtml similarity index 100% rename from server/testdata/template.mhtml rename to server/acceptance/cases/form/input.mhtml diff --git a/server/acceptance/cases/fullpage/case.test.js b/server/acceptance/cases/fullpage/case.test.js new file mode 100644 index 0000000..ee63c1f --- /dev/null +++ b/server/acceptance/cases/fullpage/case.test.js @@ -0,0 +1,12 @@ +import { pixelpactCase } from "../../lib/pixelpact.js"; + +const { scenario, render, fixture } = pixelpactCase(import.meta.dirname); +const viewport = { width: 800, height: 600 }; + +// Both renders are pinned, so the goldens show the flag actually reaching playwright. +scenario("captures the whole page only when fullpage is set", async () => { + const actualHtml = fixture("input.mhtml"); + + await render({ actualHtml, viewport }); + await render({ actualHtml, viewport, fullpage: true }); +}); diff --git a/server/acceptance/cases/fullpage/golden/01-render.actual.png b/server/acceptance/cases/fullpage/golden/01-render.actual.png new file mode 100644 index 0000000..1a11e27 Binary files /dev/null and b/server/acceptance/cases/fullpage/golden/01-render.actual.png differ diff --git a/server/acceptance/cases/fullpage/golden/02-render.actual.png b/server/acceptance/cases/fullpage/golden/02-render.actual.png new file mode 100644 index 0000000..4dfdfaf Binary files /dev/null and b/server/acceptance/cases/fullpage/golden/02-render.actual.png differ diff --git a/server/acceptance/cases/fullpage/golden/logs.jsonl b/server/acceptance/cases/fullpage/golden/logs.jsonl new file mode 100644 index 0000000..239f00c --- /dev/null +++ b/server/acceptance/cases/fullpage/golden/logs.jsonl @@ -0,0 +1,22 @@ +{"level":"info","name":"pixelpact","msg":"Server listening at
"} +{"level":"info","name":"pixelpact","msg":"Server listening at
"} +{"level":"info","name":"pixelpact","reqId":"req-1","req":{"method":"POST","url":"/render","host":"","remoteAddress":"
","remotePort":""},"msg":"incoming request"} +{"level":"debug","name":"pixelpact","reqId":"req-1","mhtmlBytes":1269,"workspaceDirectory":"","msg":"Rendering page source"} +{"level":"debug","name":"pixelpact","reqId":"req-1","indexFile":"/index.html","htmlBytes":1198,"msg":"Converted MHTML to HTML"} +{"level":"debug","name":"pixelpact","reqId":"req-1","browserVersion":"153.0.8010.12","durationMs":"","msg":"Browser launched"} +{"level":"debug","name":"pixelpact","reqId":"req-1","url":"file:///index.html","msg":"Loading page"} +{"level":"debug","name":"pixelpact","reqId":"req-1","url":"file:///index.html","loadDurationMs":"","screenshotDurationMs":"","bytes":11450,"msg":"Screenshot taken"} +{"level":"debug","name":"pixelpact","reqId":"req-1","msg":"Browser closed"} +{"level":"info","name":"pixelpact","reqId":"req-1","viewport":{"width":800,"height":600},"fullpage":false,"mhtmlConverter":true,"styled":false,"actualBytes":11450,"renderDurationMs":"","msg":"Render completed"} +{"level":"info","name":"pixelpact","reqId":"req-1","res":{"statusCode":200},"responseTime":"","msg":"request completed"} +{"level":"info","name":"pixelpact","reqId":"req-2","req":{"method":"POST","url":"/render","host":"","remoteAddress":"
","remotePort":""},"msg":"incoming request"} +{"level":"debug","name":"pixelpact","reqId":"req-2","mhtmlBytes":1269,"workspaceDirectory":"","msg":"Rendering page source"} +{"level":"debug","name":"pixelpact","reqId":"req-2","indexFile":"/index.html","htmlBytes":1198,"msg":"Converted MHTML to HTML"} +{"level":"debug","name":"pixelpact","reqId":"req-2","browserVersion":"153.0.8010.12","durationMs":"","msg":"Browser launched"} +{"level":"debug","name":"pixelpact","reqId":"req-2","url":"file:///index.html","msg":"Loading page"} +{"level":"debug","name":"pixelpact","reqId":"req-2","url":"file:///index.html","loadDurationMs":"","screenshotDurationMs":"","bytes":25863,"msg":"Screenshot taken"} +{"level":"debug","name":"pixelpact","reqId":"req-2","msg":"Browser closed"} +{"level":"info","name":"pixelpact","reqId":"req-2","viewport":{"width":800,"height":600},"fullpage":true,"mhtmlConverter":true,"styled":false,"actualBytes":25863,"renderDurationMs":"","msg":"Render completed"} +{"level":"info","name":"pixelpact","reqId":"req-2","res":{"statusCode":200},"responseTime":"","msg":"request completed"} +{"level":"info","name":"pixelpact","signal":"SIGTERM","msg":"Shutting down"} +{"level":"info","name":"pixelpact","msg":"Shutdown complete"} diff --git a/server/acceptance/cases/fullpage/golden/transcript.json b/server/acceptance/cases/fullpage/golden/transcript.json new file mode 100644 index 0000000..7279978 --- /dev/null +++ b/server/acceptance/cases/fullpage/golden/transcript.json @@ -0,0 +1,42 @@ +{ + "requests": [ + { + "method": "POST", + "path": "/render", + "body": { + "actualHtml": "", + "viewport": { + "width": 800, + "height": 600 + } + }, + "response": { + "status": 200, + "body": { + "actual": " 01-render.actual.png>" + } + } + }, + { + "method": "POST", + "path": "/render", + "body": { + "actualHtml": "", + "viewport": { + "width": 800, + "height": 600 + }, + "fullpage": true + }, + "response": { + "status": 200, + "body": { + "actual": " 02-render.actual.png>" + } + } + } + ], + "container": { + "exitCode": 0 + } +} diff --git a/server/acceptance/cases/fullpage/input.mhtml b/server/acceptance/cases/fullpage/input.mhtml new file mode 100644 index 0000000..9ccbf3a --- /dev/null +++ b/server/acceptance/cases/fullpage/input.mhtml @@ -0,0 +1,33 @@ +From: +Snapshot-Content-Location: http://localhost:8000/tall +Subject: Tall +Date: Thu, 29 Jun 2023 10:36:55 -0000 +MIME-Version: 1.0 +Content-Type: multipart/related; + type="text/html"; + boundary="----MultipartBoundary--fullpageTallerThanTheViewport000000----" + + +------MultipartBoundary--fullpageTallerThanTheViewport000000---- +Content-Type: text/html +Content-ID: +Content-Transfer-Encoding: quoted-printable +Content-Location: http://localhost:8000/tall + + + Tall + + + +

Band one

The viewport ends inside band two.

+

Band two

+

Band three

+

Band four

Only a fullpage capture reaches this.

+ +------MultipartBoundary--fullpageTallerThanTheViewport000000------ diff --git a/server/acceptance/cases/invalid-request/case.test.js b/server/acceptance/cases/invalid-request/case.test.js new file mode 100644 index 0000000..8cfc8bb --- /dev/null +++ b/server/acceptance/cases/invalid-request/case.test.js @@ -0,0 +1,9 @@ +import { pixelpactCase } from "../../lib/pixelpact.js"; + +const { scenario, check, fixture } = pixelpactCase(import.meta.dirname); + +// The 400, the error envelope and the logged error all land in the goldens +// without this case naming any of them. +scenario("rejects a check without a viewport", async () => { + await check({ actualHtml: fixture("../plain/input.mhtml") }); +}); diff --git a/server/acceptance/cases/invalid-request/golden/logs.jsonl b/server/acceptance/cases/invalid-request/golden/logs.jsonl new file mode 100644 index 0000000..54a2ed7 --- /dev/null +++ b/server/acceptance/cases/invalid-request/golden/logs.jsonl @@ -0,0 +1,7 @@ +{"level":"info","name":"pixelpact","msg":"Server listening at
"} +{"level":"info","name":"pixelpact","msg":"Server listening at
"} +{"level":"info","name":"pixelpact","reqId":"req-1","req":{"method":"POST","url":"/check","host":"","remoteAddress":"
","remotePort":""},"msg":"incoming request"} +{"level":"warn","name":"pixelpact","reqId":"req-1","statusCode":400,"code":"FST_ERR_VALIDATION","reason":"body must have required property 'expected'","validation":[{"instancePath":"","schemaPath":"#/required","keyword":"required","params":{"missingProperty":"expected"},"message":"must have required property 'expected'"}],"msg":"Request rejected"} +{"level":"info","name":"pixelpact","reqId":"req-1","res":{"statusCode":400},"responseTime":"","msg":"request completed"} +{"level":"info","name":"pixelpact","signal":"SIGTERM","msg":"Shutting down"} +{"level":"info","name":"pixelpact","msg":"Shutdown complete"} diff --git a/server/acceptance/cases/invalid-request/golden/transcript.json b/server/acceptance/cases/invalid-request/golden/transcript.json new file mode 100644 index 0000000..aabfdc0 --- /dev/null +++ b/server/acceptance/cases/invalid-request/golden/transcript.json @@ -0,0 +1,21 @@ +{ + "requests": [ + { + "method": "POST", + "path": "/check", + "body": { + "actualHtml": "" + }, + "response": { + "status": 400, + "body": { + "message": "body must have required property 'expected'", + "statusCode": 400 + } + } + } + ], + "container": { + "exitCode": 0 + } +} diff --git a/server/acceptance/cases/mismatch/case.test.js b/server/acceptance/cases/mismatch/case.test.js new file mode 100644 index 0000000..35acda0 --- /dev/null +++ b/server/acceptance/cases/mismatch/case.test.js @@ -0,0 +1,22 @@ +import { pixelpactCase } from "../../lib/pixelpact.js"; + +const { scenario, render, check, fixture } = pixelpactCase(import.meta.dirname); +const viewport = { width: 800, height: 600 }; + +// Same document as ../form, one heading changed. The reference is rendered here +// rather than read from form/golden, so no golden is ever an input to a case. +scenario( + "reports a diff when the reference came from different HTML", + async () => { + const reference = await render({ + actualHtml: fixture("../form/input.mhtml"), + viewport, + }); + + await check({ + actualHtml: fixture("input.mhtml"), + expected: reference.png, + viewport, + }); + }, +); diff --git a/server/acceptance/cases/mismatch/golden/01-render.actual.png b/server/acceptance/cases/mismatch/golden/01-render.actual.png new file mode 100644 index 0000000..e472d7f Binary files /dev/null and b/server/acceptance/cases/mismatch/golden/01-render.actual.png differ diff --git a/server/acceptance/cases/mismatch/golden/02-check.actual.png b/server/acceptance/cases/mismatch/golden/02-check.actual.png new file mode 100644 index 0000000..2b8e3b0 Binary files /dev/null and b/server/acceptance/cases/mismatch/golden/02-check.actual.png differ diff --git a/server/acceptance/cases/mismatch/golden/02-check.diff.png b/server/acceptance/cases/mismatch/golden/02-check.diff.png new file mode 100644 index 0000000..9fbccfb Binary files /dev/null and b/server/acceptance/cases/mismatch/golden/02-check.diff.png differ diff --git a/server/acceptance/cases/mismatch/golden/02-check.expected.png b/server/acceptance/cases/mismatch/golden/02-check.expected.png new file mode 100644 index 0000000..e472d7f Binary files /dev/null and b/server/acceptance/cases/mismatch/golden/02-check.expected.png differ diff --git a/server/acceptance/cases/mismatch/golden/logs.jsonl b/server/acceptance/cases/mismatch/golden/logs.jsonl new file mode 100644 index 0000000..e3f8ba9 --- /dev/null +++ b/server/acceptance/cases/mismatch/golden/logs.jsonl @@ -0,0 +1,22 @@ +{"level":"info","name":"pixelpact","msg":"Server listening at
"} +{"level":"info","name":"pixelpact","msg":"Server listening at
"} +{"level":"info","name":"pixelpact","reqId":"req-1","req":{"method":"POST","url":"/render","host":"","remoteAddress":"
","remotePort":""},"msg":"incoming request"} +{"level":"debug","name":"pixelpact","reqId":"req-1","mhtmlBytes":2151,"workspaceDirectory":"","msg":"Rendering page source"} +{"level":"debug","name":"pixelpact","reqId":"req-1","indexFile":"/index.html","htmlBytes":1768,"msg":"Converted MHTML to HTML"} +{"level":"debug","name":"pixelpact","reqId":"req-1","browserVersion":"153.0.8010.12","durationMs":"","msg":"Browser launched"} +{"level":"debug","name":"pixelpact","reqId":"req-1","url":"file:///index.html","msg":"Loading page"} +{"level":"debug","name":"pixelpact","reqId":"req-1","url":"file:///index.html","loadDurationMs":"","screenshotDurationMs":"","bytes":13500,"msg":"Screenshot taken"} +{"level":"debug","name":"pixelpact","reqId":"req-1","msg":"Browser closed"} +{"level":"info","name":"pixelpact","reqId":"req-1","viewport":{"width":800,"height":600},"fullpage":false,"mhtmlConverter":true,"styled":false,"actualBytes":13500,"renderDurationMs":"","msg":"Render completed"} +{"level":"info","name":"pixelpact","reqId":"req-1","res":{"statusCode":200},"responseTime":"","msg":"request completed"} +{"level":"info","name":"pixelpact","reqId":"req-2","req":{"method":"POST","url":"/check","host":"","remoteAddress":"
","remotePort":""},"msg":"incoming request"} +{"level":"debug","name":"pixelpact","reqId":"req-2","mhtmlBytes":2154,"workspaceDirectory":"","msg":"Rendering page source"} +{"level":"debug","name":"pixelpact","reqId":"req-2","indexFile":"/index.html","htmlBytes":1771,"msg":"Converted MHTML to HTML"} +{"level":"debug","name":"pixelpact","reqId":"req-2","browserVersion":"153.0.8010.12","durationMs":"","msg":"Browser launched"} +{"level":"debug","name":"pixelpact","reqId":"req-2","url":"file:///index.html","msg":"Loading page"} +{"level":"debug","name":"pixelpact","reqId":"req-2","url":"file:///index.html","loadDurationMs":"","screenshotDurationMs":"","bytes":14163,"msg":"Screenshot taken"} +{"level":"debug","name":"pixelpact","reqId":"req-2","msg":"Browser closed"} +{"level":"info","name":"pixelpact","reqId":"req-2","viewport":{"width":800,"height":600},"fullpage":false,"mhtmlConverter":true,"styled":false,"expectedBytes":13500,"actualBytes":14163,"numDiffPixels":911,"renderDurationMs":"","compareDurationMs":"","msg":"Check completed"} +{"level":"info","name":"pixelpact","reqId":"req-2","res":{"statusCode":200},"responseTime":"","msg":"request completed"} +{"level":"info","name":"pixelpact","signal":"SIGTERM","msg":"Shutting down"} +{"level":"info","name":"pixelpact","msg":"Shutdown complete"} diff --git a/server/acceptance/cases/mismatch/golden/transcript.json b/server/acceptance/cases/mismatch/golden/transcript.json new file mode 100644 index 0000000..a691c8c --- /dev/null +++ b/server/acceptance/cases/mismatch/golden/transcript.json @@ -0,0 +1,45 @@ +{ + "requests": [ + { + "method": "POST", + "path": "/render", + "body": { + "actualHtml": "", + "viewport": { + "width": 800, + "height": 600 + } + }, + "response": { + "status": 200, + "body": { + "actual": " 01-render.actual.png>" + } + } + }, + { + "method": "POST", + "path": "/check", + "body": { + "actualHtml": "", + "expected": "", + "viewport": { + "width": 800, + "height": 600 + } + }, + "response": { + "status": 200, + "body": { + "actual": " 02-check.actual.png>", + "expected": " 02-check.expected.png>", + "diff": " 02-check.diff.png>", + "numDiffPixels": 911 + } + } + } + ], + "container": { + "exitCode": 0 + } +} diff --git a/server/acceptance/cases/mismatch/input.mhtml b/server/acceptance/cases/mismatch/input.mhtml new file mode 100644 index 0000000..89c5176 --- /dev/null +++ b/server/acceptance/cases/mismatch/input.mhtml @@ -0,0 +1,68 @@ +From: +Snapshot-Content-Location: http://localhost:8000/ +Subject: Document +Date: Thu, 29 Jun 2023 10:36:55 -0000 +MIME-Version: 1.0 +Content-Type: multipart/related; + type="text/html"; + boundary="----MultipartBoundary--mJEnOoe6QIyy5FUC8dMc2jWfHPkkc7zRmcu5QqHSL4----" + + +------MultipartBoundary--mJEnOoe6QIyy5FUC8dMc2jWfHPkkc7zRmcu5QqHSL4---- +Content-Type: text/html +Content-ID: +Content-Transfer-Encoding: quoted-printable +Content-Location: http://localhost:8000/ + + + =20 + + + + Document + + +

Hello Mismatch

+ +
+ +

+ + +

+ + +

+ +
+

+ + + +
+ + +------MultipartBoundary--mJEnOoe6QIyy5FUC8dMc2jWfHPkkc7zRmcu5QqHSL4---- +Content-Type: text/css +Content-Transfer-Encoding: quoted-printable +Content-Location: http://localhost:8000/example.css + +@charset "utf-8"; + +body { background-color: pink; } + +form { + margin: 20px; + padding: 10px; + border: 2px solid black; + background-color: white; +} + +input, textarea, button { + margin: 10px 0; + padding: 5px; + font-size: 1rem; +} +------MultipartBoundary--mJEnOoe6QIyy5FUC8dMc2jWfHPkkc7zRmcu5QqHSL4------ diff --git a/server/acceptance/cases/plain/case.test.js b/server/acceptance/cases/plain/case.test.js new file mode 100644 index 0000000..3a10936 --- /dev/null +++ b/server/acceptance/cases/plain/case.test.js @@ -0,0 +1,11 @@ +import { pixelpactCase } from "../../lib/pixelpact.js"; + +const { scenario, render, check, fixture } = pixelpactCase(import.meta.dirname); +const viewport = { width: 800, height: 600 }; + +scenario("checks a single-part document against its own render", async () => { + const actualHtml = fixture("input.mhtml"); + + const reference = await render({ actualHtml, viewport }); + await check({ actualHtml, expected: reference.png, viewport }); +}); diff --git a/server/acceptance/cases/plain/golden/01-render.actual.png b/server/acceptance/cases/plain/golden/01-render.actual.png new file mode 100644 index 0000000..a60d7de Binary files /dev/null and b/server/acceptance/cases/plain/golden/01-render.actual.png differ diff --git a/server/acceptance/cases/plain/golden/02-check.actual.png b/server/acceptance/cases/plain/golden/02-check.actual.png new file mode 100644 index 0000000..a60d7de Binary files /dev/null and b/server/acceptance/cases/plain/golden/02-check.actual.png differ diff --git a/server/acceptance/cases/plain/golden/02-check.diff.png b/server/acceptance/cases/plain/golden/02-check.diff.png new file mode 100644 index 0000000..0adecdb Binary files /dev/null and b/server/acceptance/cases/plain/golden/02-check.diff.png differ diff --git a/server/acceptance/cases/plain/golden/02-check.expected.png b/server/acceptance/cases/plain/golden/02-check.expected.png new file mode 100644 index 0000000..a60d7de Binary files /dev/null and b/server/acceptance/cases/plain/golden/02-check.expected.png differ diff --git a/server/acceptance/cases/plain/golden/logs.jsonl b/server/acceptance/cases/plain/golden/logs.jsonl new file mode 100644 index 0000000..236f820 --- /dev/null +++ b/server/acceptance/cases/plain/golden/logs.jsonl @@ -0,0 +1,22 @@ +{"level":"info","name":"pixelpact","msg":"Server listening at
"} +{"level":"info","name":"pixelpact","msg":"Server listening at
"} +{"level":"info","name":"pixelpact","reqId":"req-1","req":{"method":"POST","url":"/render","host":"","remoteAddress":"
","remotePort":""},"msg":"incoming request"} +{"level":"debug","name":"pixelpact","reqId":"req-1","mhtmlBytes":950,"workspaceDirectory":"","msg":"Rendering page source"} +{"level":"debug","name":"pixelpact","reqId":"req-1","indexFile":"/index.html","htmlBytes":921,"msg":"Converted MHTML to HTML"} +{"level":"debug","name":"pixelpact","reqId":"req-1","browserVersion":"153.0.8010.12","durationMs":"","msg":"Browser launched"} +{"level":"debug","name":"pixelpact","reqId":"req-1","url":"file:///index.html","msg":"Loading page"} +{"level":"debug","name":"pixelpact","reqId":"req-1","url":"file:///index.html","loadDurationMs":"","screenshotDurationMs":"","bytes":9497,"msg":"Screenshot taken"} +{"level":"debug","name":"pixelpact","reqId":"req-1","msg":"Browser closed"} +{"level":"info","name":"pixelpact","reqId":"req-1","viewport":{"width":800,"height":600},"fullpage":false,"mhtmlConverter":true,"styled":false,"actualBytes":9497,"renderDurationMs":"","msg":"Render completed"} +{"level":"info","name":"pixelpact","reqId":"req-1","res":{"statusCode":200},"responseTime":"","msg":"request completed"} +{"level":"info","name":"pixelpact","reqId":"req-2","req":{"method":"POST","url":"/check","host":"","remoteAddress":"
","remotePort":""},"msg":"incoming request"} +{"level":"debug","name":"pixelpact","reqId":"req-2","mhtmlBytes":950,"workspaceDirectory":"","msg":"Rendering page source"} +{"level":"debug","name":"pixelpact","reqId":"req-2","indexFile":"/index.html","htmlBytes":921,"msg":"Converted MHTML to HTML"} +{"level":"debug","name":"pixelpact","reqId":"req-2","browserVersion":"153.0.8010.12","durationMs":"","msg":"Browser launched"} +{"level":"debug","name":"pixelpact","reqId":"req-2","url":"file:///index.html","msg":"Loading page"} +{"level":"debug","name":"pixelpact","reqId":"req-2","url":"file:///index.html","loadDurationMs":"","screenshotDurationMs":"","bytes":9497,"msg":"Screenshot taken"} +{"level":"debug","name":"pixelpact","reqId":"req-2","msg":"Browser closed"} +{"level":"info","name":"pixelpact","reqId":"req-2","viewport":{"width":800,"height":600},"fullpage":false,"mhtmlConverter":true,"styled":false,"expectedBytes":9497,"actualBytes":9497,"numDiffPixels":0,"renderDurationMs":"","compareDurationMs":"","msg":"Check completed"} +{"level":"info","name":"pixelpact","reqId":"req-2","res":{"statusCode":200},"responseTime":"","msg":"request completed"} +{"level":"info","name":"pixelpact","signal":"SIGTERM","msg":"Shutting down"} +{"level":"info","name":"pixelpact","msg":"Shutdown complete"} diff --git a/server/acceptance/cases/plain/golden/transcript.json b/server/acceptance/cases/plain/golden/transcript.json new file mode 100644 index 0000000..c2ee808 --- /dev/null +++ b/server/acceptance/cases/plain/golden/transcript.json @@ -0,0 +1,45 @@ +{ + "requests": [ + { + "method": "POST", + "path": "/render", + "body": { + "actualHtml": "", + "viewport": { + "width": 800, + "height": 600 + } + }, + "response": { + "status": 200, + "body": { + "actual": " 01-render.actual.png>" + } + } + }, + { + "method": "POST", + "path": "/check", + "body": { + "actualHtml": "", + "expected": "", + "viewport": { + "width": 800, + "height": 600 + } + }, + "response": { + "status": 200, + "body": { + "actual": " 02-check.actual.png>", + "expected": " 02-check.expected.png>", + "diff": " 02-check.diff.png>", + "numDiffPixels": 0 + } + } + } + ], + "container": { + "exitCode": 0 + } +} diff --git a/server/acceptance/cases/plain/input.mhtml b/server/acceptance/cases/plain/input.mhtml new file mode 100644 index 0000000..72c54f3 --- /dev/null +++ b/server/acceptance/cases/plain/input.mhtml @@ -0,0 +1,26 @@ +From: +Snapshot-Content-Location: http://localhost:8000/plain +Subject: Plain +Date: Thu, 29 Jun 2023 10:36:55 -0000 +MIME-Version: 1.0 +Content-Type: multipart/related; + type="text/html"; + boundary="----MultipartBoundary--plainSingleParteNoSubResources0000000000----" + + +------MultipartBoundary--plainSingleParteNoSubResources0000000000---- +Content-Type: text/html +Content-ID: +Content-Transfer-Encoding: quoted-printable +Content-Location: http://localhost:8000/plain + + + Plain + + + +

Plain document

+

A single MIME part with no sub-resources.

+ +------MultipartBoundary--plainSingleParteNoSubResources0000000000------ diff --git a/server/acceptance/lib/container.js b/server/acceptance/lib/container.js new file mode 100644 index 0000000..4304b48 --- /dev/null +++ b/server/acceptance/lib/container.js @@ -0,0 +1,155 @@ +import { spawn } from "node:child_process"; + +const READY_DEADLINE_MS = 60_000; +const READY_POLL_MS = 250; +/** What `index.js` logs once fastify is bound. */ +const LISTENING_MESSAGE = "Server listening at"; + +/** Runs `docker` with {@link args}. + * @param {string[]} args + * @param {{stream?: boolean}} options `stream` inherits stdio instead of capturing it, + * so a cold `docker build` reports progress as it happens. + * @returns {Promise<{stdout: string, stderr: string, code: number}>} + */ +export function docker(args, { stream = false } = {}) { + return new Promise((resolve, reject) => { + const child = spawn("docker", args, { + stdio: stream + ? ["ignore", "inherit", "inherit"] + : ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout?.on("data", (chunk) => (stdout += chunk)); + child.stderr?.on("data", (chunk) => (stderr += chunk)); + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) { + resolve({ stdout, stderr, code }); + } else { + reject( + new Error( + `docker ${args.join(" ")} exited with ${code}${stderr ? `\n${stderr}` : ""}`, + ), + ); + } + }); + }); +} + +/** Starts a container of {@link image} and waits until it is listening. + * + * The image hardcodes port 8888 with no PORT override, so the host side is + * published as an ephemeral port to keep concurrent runs from colliding. + * + * @param {string} image + * @param {Record} env passed through as `-e KEY=VALUE` + * @returns {Promise<{id: string, baseUrl: string}>} + */ +export async function startContainer(image, env) { + const environment = Object.entries(env).flatMap(([key, value]) => [ + "-e", + `${key}=${value}`, + ]); + const { stdout } = await docker([ + "run", + "--detach", + "--publish", + "127.0.0.1:0:8888", + ...environment, + image, + ]); + const id = stdout.trim(); + + try { + const container = { id, baseUrl: await resolveBaseUrl(id) }; + await waitUntilReady(container); + return container; + } catch (error) { + // Never leak a container just because it failed to come up. + await removeContainer({ id }).catch(() => {}); + throw error; + } +} + +async function resolveBaseUrl(id) { + const { stdout } = await docker(["port", id, "8888/tcp"]); + const port = stdout.trim().split("\n")[0]?.split(":").pop(); + if (!port) { + throw new Error(`Failed to resolve the published port of container ${id}`); + } + return `http://127.0.0.1:${port}`; +} + +/** Waits for the server to say it is listening. + * + * Readiness is read from the log rather than probed over HTTP on purpose: a + * probe request would land in the log golden of every single case and shift + * every reqId. It also needs no route, which the server is short of - there is + * no GET endpoint and no healthcheck. + * + * This means a case must run at a level where `info` is emitted. Every case + * does; a quieter one would need a different signal. + */ +async function waitUntilReady(container) { + const deadline = Date.now() + READY_DEADLINE_MS; + while (Date.now() < deadline) { + const { stdout, stderr } = await readContainerLogs(container); + if (stdout.includes(LISTENING_MESSAGE)) { + return; + } + if (!(await isRunning(container))) { + throw new Error( + `Container ${container.id} exited before it started listening.` + + logTail({ stdout, stderr }), + ); + } + await new Promise((resolve) => setTimeout(resolve, READY_POLL_MS)); + } + throw new Error( + `Container ${container.id} did not log "${LISTENING_MESSAGE}" within ` + + `${READY_DEADLINE_MS}ms.` + + logTail(await readContainerLogs(container).catch(() => ({}))), + ); +} + +async function isRunning(container) { + const { stdout } = await docker([ + "inspect", + "--format", + "{{.State.Running}}", + container.id, + ]); + return stdout.trim() === "true"; +} + +function logTail({ stdout = "", stderr = "" }) { + return `\n--- container logs ---\n${stdout}${stderr}`; +} + +/** Stops the container so the log stream covers the whole lifecycle, + * including whatever the SIGTERM handler does. + * @returns {Promise} the container's exit code + */ +export async function stopContainer(container) { + await docker(["stop", "--timeout", "5", container.id]); + const { stdout } = await docker([ + "inspect", + "--format", + "{{.State.ExitCode}}", + container.id, + ]); + return Number(stdout.trim()); +} + +/** Reads everything the container wrote. stdout and stderr are captured + * separately because docker demultiplexes them; stderr is appended last and + * marked by {@link normalizeLogs}, since anything there is a crash, not logging. */ +export async function readContainerLogs(container) { + const { stdout, stderr } = await docker(["logs", container.id]); + return { stdout, stderr }; +} + +export async function removeContainer(container) { + await docker(["rm", "--force", container.id]); +} diff --git a/server/acceptance/lib/image.js b/server/acceptance/lib/image.js new file mode 100644 index 0000000..671fbc1 --- /dev/null +++ b/server/acceptance/lib/image.js @@ -0,0 +1,30 @@ +import path from "node:path"; +import { docker } from "./container.js"; + +const BUILT_TAG = "pixelpact:acceptance"; + +/** vitest `globalSetup`: settles which image the suite drives and whether + * goldens may be written, then hands both to the cases via `inject()`. + * + * CI builds the image once and passes it in via `PIXELPACT_IMAGE`; locally we + * build it ourselves and leave it in place so the next run is warm. + */ +export default async function setup(project) { + const provided = process.env.PIXELPACT_IMAGE; + if (provided) { + // Fail loudly here rather than once per case with a confusing `docker run` error. + await docker(["image", "inspect", provided]); + } else { + const serverDirectory = path.resolve(import.meta.dirname, "../.."); + await docker(["build", "--tag", BUILT_TAG, serverDirectory], { + stream: true, + }); + } + + project.provide("image", provided ?? BUILT_TAG); + // "all" with -u, "new" locally (write what is missing), "none" on CI (missing is a failure). + project.provide( + "updateSnapshot", + project.config.snapshotOptions.updateSnapshot, + ); +} diff --git a/server/acceptance/lib/pixelpact.js b/server/acceptance/lib/pixelpact.js new file mode 100644 index 0000000..c01ce84 --- /dev/null +++ b/server/acceptance/lib/pixelpact.js @@ -0,0 +1,119 @@ +import fs from "node:fs"; +import path from "node:path"; +import { afterEach, beforeEach, inject, it } from "vitest"; +import { + readContainerLogs, + removeContainer, + startContainer, + stopContainer, +} from "./container.js"; +import { createRecorder, normalizeLogs, pinGoldens } from "./transcript.js"; + +/** Everything a case needs. A case says what to do; the goldens say what happened. + * + * Each test gets its own container, so the whole log stream belongs to that one + * test and cases cannot leak state into each other. + * + * @param {string} dir the case directory, as `import.meta.dirname` + * @param {Record} env container environment; `LOG_LEVEL=debug` + * by default so the goldens capture the full render trace + */ +export function pixelpactCase(dir, env = { LOG_LEVEL: "debug" }) { + let container; + let recorder; + + beforeEach(async () => { + recorder = createRecorder(); + container = await startContainer(inject("image"), env); + }); + + afterEach(async () => { + if (container) { + await removeContainer(container); + container = undefined; + } + }); + + /** Reads a file next to the case. `"../form/input.mhtml"` works as written. */ + const fixture = (relativePath) => + fs.readFileSync(path.resolve(dir, relativePath)); + + /** Raw escape hatch. Never throws on a non-2xx - the status is part of the transcript. */ + async function post(route, body) { + const response = await fetch(`${container.baseUrl}${route}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + const raw = await response.text(); + let responseBody; + try { + responseBody = JSON.parse(raw); + } catch { + // A non-JSON body from a JSON API is itself worth pinning. + responseBody = { nonJsonBody: raw }; + } + recorder.record({ + method: "POST", + route, + requestBody: body, + status: response.status, + responseBody, + }); + return { status: response.status, body: responseBody }; + } + + async function render({ actualHtml, ...options }) { + const { status, body } = await post("/render", { + actualHtml: asText(actualHtml), + ...options, + }); + return { status, body, png: asPng(body.actual) }; + } + + async function check({ actualHtml, expected, ...options }) { + const { status, body } = await post("/check", { + actualHtml: asText(actualHtml), + ...(expected === undefined ? {} : { expected: asBase64(expected) }), + ...options, + }); + return { + status, + body, + png: asPng(body.actual), + expected: asPng(body.expected), + diff: asPng(body.diff), + }; + } + + /** `it` plus the pinning, so a case body only has to say what to do. + * + * The pinning runs inside the test rather than in `afterEach` for two + * reasons: `toMatchFileSnapshot` needs a test context, and a body that + * throws should report its own error instead of a pile of golden diffs. + */ + function scenario(name, body) { + it(name, async () => { + await body(); + const exitCode = await stopContainer(container); + const logs = normalizeLogs(await readContainerLogs(container)); + await pinGoldens({ + dir, + requests: recorder.requests, + container: { exitCode }, + logs, + updateSnapshot: inject("updateSnapshot"), + }); + }); + } + + return { scenario, render, check, post, fixture }; +} + +// Buffers in, Buffers out - base64 never appears in a case. +const asText = (value) => + Buffer.isBuffer(value) ? value.toString("utf8") : value; +const asBase64 = (value) => + Buffer.isBuffer(value) ? value.toString("base64") : value; +const asPng = (value) => + typeof value === "string" ? Buffer.from(value, "base64") : undefined; diff --git a/server/acceptance/lib/transcript.js b/server/acceptance/lib/transcript.js new file mode 100644 index 0000000..7a00006 --- /dev/null +++ b/server/acceptance/lib/transcript.js @@ -0,0 +1,289 @@ +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { expect } from "vitest"; +import pixelmatch from "pixelmatch"; +import { PNG } from "pngjs"; + +/** Strings at least this long are payloads, not values worth reading inline. */ +const BLOB_MIN_LENGTH = 256; +const PNG_SIGNATURE = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, +]); +const TRANSCRIPT = "transcript.json"; +const LOGS = "logs.jsonl"; + +/** Collects every request a case makes so the whole exchange can be pinned at once. + * Cases assert nothing themselves; this is what makes that possible. */ +export function createRecorder() { + const requests = []; + return { + record(exchange) { + // The number only ever surfaces in the payload filenames; the transcript + // gets its order from the array. + requests.push({ number: requests.length + 1, ...exchange }); + }, + requests, + }; +} + +/** Pins the recorded exchange, the response payloads and the log stream + * against `/golden/`. + * + * Text goldens go through vitest's own `toMatchFileSnapshot`, so `-u` handling + * is public API. Buffers are compared byte-exact and follow the same + * `updateSnapshot` semantics by hand: "all" rewrites, "new" fills in what is + * missing, "none" (CI) treats a missing golden as a failure. + */ +export async function pinGoldens({ + dir, + requests, + container, + logs, + updateSnapshot, +}) { + const goldenDirectory = path.join(dir, "golden"); + const transcript = { requests: [], container }; + const payloads = new Map(); + + for (const { + number, + method, + route, + requestBody, + status, + responseBody, + } of requests) { + const operation = route.replace(/^\//, ""); + transcript.requests.push({ + method, + path: route, + // No `payloads`, so a request payload gets a descriptor but no file of its + // own: it is either a repository fixture or an earlier response, and the + // sha is what shows that linkage. + body: elide(requestBody, {}), + response: { + status, + body: elide(responseBody, { number, operation, payloads }), + }, + }); + } + + if (updateSnapshot === "all") { + await fs.rm(goldenDirectory, { recursive: true, force: true }); + } + await fs.mkdir(goldenDirectory, { recursive: true }); + + for (const [name, bytes] of payloads) { + await pinBuffer({ dir, goldenDirectory, name, bytes, updateSnapshot }); + } + + await expect(`${JSON.stringify(transcript, null, 2)}\n`).toMatchFileSnapshot( + path.join(goldenDirectory, TRANSCRIPT), + ); + await expect( + logs.map((entry) => `${JSON.stringify(entry)}\n`).join(""), + ).toMatchFileSnapshot(path.join(goldenDirectory, LOGS)); + + await assertNothingObsolete(goldenDirectory, [ + ...payloads.keys(), + TRANSCRIPT, + LOGS, + ]); +} + +/** Replaces payload strings with a descriptor, and records the bytes to pin + * when {@link payloads} is supplied. */ +function elide(body, { number, operation, payloads }) { + const elided = {}; + for (const [key, value] of Object.entries(body ?? {})) { + if (typeof value !== "string" || value.length < BLOB_MIN_LENGTH) { + elided[key] = value; + continue; + } + const { kind, bytes, extension } = classify(value); + let name; + if (payloads) { + name = `${String(number).padStart(2, "0")}-${operation}.${key}.${extension}`; + payloads.set(name, bytes); + } + const digest = crypto + .createHash("sha256") + .update(bytes) + .digest("hex") + .slice(0, 12); + elided[key] = + `<${kind} ${bytes.length} bytes sha256:${digest}${name ? ` -> ${name}` : ""}>`; + } + return elided; +} + +function classify(value) { + const decoded = decodeBase64(value); + if (decoded?.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE)) { + return { kind: "png", bytes: decoded, extension: "png" }; + } + const bytes = Buffer.from(value, "utf8"); + const isMhtml = /^(From: |MIME-Version:)/m.test(value); + return { + kind: isMhtml ? "mhtml" : "text", + bytes, + extension: isMhtml ? "mhtml" : "txt", + }; +} + +function decodeBase64(value) { + if (!/^[A-Za-z0-9+/]+={0,2}$/.test(value)) { + return undefined; + } + const decoded = Buffer.from(value, "base64"); + return decoded.length > 0 ? decoded : undefined; +} + +async function pinBuffer({ + dir, + goldenDirectory, + name, + bytes, + updateSnapshot, +}) { + const file = path.join(goldenDirectory, name); + const golden = await fs.readFile(file).catch(() => undefined); + + if (updateSnapshot === "all" || (updateSnapshot === "new" && !golden)) { + await fs.writeFile(file, bytes); + return; + } + if (!golden) { + throw new Error( + `Golden golden/${name} is missing. Regenerate with \`npm run test:acceptance -- -u\`.`, + ); + } + if (Buffer.compare(golden, bytes) !== 0) { + const out = await dumpFailure({ dir, name, actual: bytes, golden }); + throw new Error( + `Golden golden/${name} differs (${golden.length} -> ${bytes.length} bytes). ` + + `Wrote both sides to ${out}.`, + ); + } +} + +/** A case that stops making a request must not leave its golden behind. */ +async function assertNothingObsolete(goldenDirectory, accounted) { + const present = await fs.readdir(goldenDirectory); + const obsolete = present.filter((name) => !accounted.includes(name)); + if (obsolete.length > 0) { + throw new Error( + `Obsolete goldens no longer produced by this case: ${obsolete.join(", ")}. ` + + `Remove them, or regenerate with \`npm run test:acceptance -- -u\`.`, + ); + } +} + +async function dumpFailure({ dir, name, actual, golden }) { + const out = path.join(dir, "..", "..", "out", path.basename(dir)); + await fs.mkdir(out, { recursive: true }); + await fs.writeFile(path.join(out, `actual-${name}`), actual); + await fs.writeFile(path.join(out, `golden-${name}`), golden); + if (name.endsWith(".png")) { + await writePixelDiff(path.join(out, `pixeldiff-${name}`), actual, golden); + } + return path.relative(path.join(dir, "..", ".."), out); +} + +/** A human-readable diff for the failure artifacts only - never an assertion. */ +async function writePixelDiff(file, actual, golden) { + try { + const actualPng = PNG.sync.read(actual); + const goldenPng = PNG.sync.read(golden); + if ( + actualPng.width !== goldenPng.width || + actualPng.height !== goldenPng.height + ) { + return; + } + const diff = new PNG({ width: actualPng.width, height: actualPng.height }); + pixelmatch( + goldenPng.data, + actualPng.data, + diff.data, + actualPng.width, + actualPng.height, + { threshold: 0 }, + ); + await fs.writeFile(file, PNG.sync.write(diff)); + } catch { + // Debugging aid; never the reason a test fails. + } +} + +/** Turns the container's output into stable, comparable entries. + * + * Unparseable stdout is kept as `raw` rather than dropped - a stray + * `console.log` or a half-formatted stack is broken logging and should fail. + * Anything on stderr is a crash, not logging, so it is marked as such. + */ +export function normalizeLogs({ stdout, stderr }) { + return [ + ...lines(stdout).map(normalizeLine), + ...lines(stderr).map((line) => ({ stream: "stderr", raw: scrub(line) })), + ]; +} + +function lines(output) { + return output.split("\n").filter((line) => line.trim().length > 0); +} + +function normalizeLine(line) { + try { + // Wall clock, pid and hostname differ on every run and say nothing. + const { time, pid, hostname, ...rest } = JSON.parse(line); + return scrubDeep(rest); + } catch { + return { raw: scrub(line) }; + } +} + +/** Fields whose value is different on every run. The placeholder keeps the + * field itself pinned, so one appearing or disappearing is still a failure - + * only its unstable value is dropped. Timings, durations and byte counts that + * *are* stable (actualBytes, numDiffPixels, browserVersion) stay verbatim: + * browserVersion in particular is how a playwright bump announces itself. */ +const VOLATILE = { + responseTime: "", + host: "", + remoteAddress: "
", + remotePort: "", +}; + +const isDuration = (key) => /^(duration|.*Duration)Ms$/.test(key); + +function scrub(value) { + return ( + value + // A fresh mkdtemp per request. + .replace(/\/tmp\/pixelpact-[A-Za-z0-9]+/g, "") + // The container's bridge IP and the published ephemeral port. + .replace(/http:\/\/\d+\.\d+\.\d+\.\d+:\d+/g, "
") + ); +} + +function scrubDeep(value) { + if (typeof value === "string") { + return scrub(value); + } + if (Array.isArray(value)) { + return value.map(scrubDeep); + } + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([key, nested]) => { + if (isDuration(key)) { + return [key, ""]; + } + return [key, key in VOLATILE ? VOLATILE[key] : scrubDeep(nested)]; + }), + ); + } + return value; +} diff --git a/server/package.json b/server/package.json index 6dd3f6c..f904372 100644 --- a/server/package.json +++ b/server/package.json @@ -9,7 +9,8 @@ "scripts": { "start": "LOG_LEVEL=debug node --watch src/index.js | pino-pretty", "test": "vitest run", - "test:coverage": "vitest run --coverage" + "test:coverage": "vitest run --coverage", + "test:acceptance": "vitest run --config vitest.acceptance.config.js" }, "dependencies": { "fastify": "5.12.4", diff --git a/server/src/integrationtest.test.js b/server/src/integrationtest.test.js deleted file mode 100644 index 3439f65..0000000 --- a/server/src/integrationtest.test.js +++ /dev/null @@ -1,79 +0,0 @@ -import { startApiServer } from "./api.js"; -import { getLocalAddress } from "./helpers.js"; -import fs from "fs/promises"; - -describe("check integration test", () => { - let instance, baseUrl; - - beforeAll(async () => { - instance = await startApiServer(0); - baseUrl = getLocalAddress(instance); - }); - - afterAll(async () => { - await instance.close(); - }); - - it("succeeds when calling check with the reference image generated using render", async () => { - const viewport = { width: 1920, height: 1024 }; - const htmlContent = await mhtmlOf("

Hello World

"); - const reference = await render(htmlContent, viewport); - - const result = await check(htmlContent, reference, viewport); - - expect(result.expected).toBe(reference); - expect(result.actual).toBe(reference); - expect(result.numDiffPixels).toBe(0); - }); - - it("fails when calling check with a reference obtained from a different HTML content", async () => { - const viewport = { width: 1920, height: 1024 }; - const reference = await render( - await mhtmlOf("

Hello Jack

"), - viewport, - ); - const result = await check( - await mhtmlOf("

Hello Jill

"), - reference, - viewport, - ); - expect(result.expected).toBe(reference); - expect(result.actual).not.toBe(reference); - expect(result.numDiffPixels).toBeGreaterThan(0); - }); - - async function render(actualHtml, viewport) { - const response = await fetch(`${baseUrl}/render`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ actualHtml, viewport }), - }); - - expect(response.status).toBe(200); - - const body = await response.json(); - expect(body.actual.length).toBeGreaterThan(0); - - return body.actual; - } - - async function check(actualHtml, expected, viewport) { - const response = await fetch(`${baseUrl}/check`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ actualHtml, expected, viewport }), - }); - - const body = await response.json(); - expect(body.diff.length).toBeGreaterThan(0); - expect(body.actual.length).toBeGreaterThan(0); - expect(body.expected.length).toBeGreaterThan(0); - - return body; - } -}); - -async function mhtmlOf(html) { - const template = (await fs.readFile("testdata/template.mhtml")).toString(); - return template.replace("Hello World", html); -} diff --git a/server/vitest.acceptance.config.js b/server/vitest.acceptance.config.js new file mode 100644 index 0000000..c2e4d65 --- /dev/null +++ b/server/vitest.acceptance.config.js @@ -0,0 +1,16 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, + environment: "node", + include: ["acceptance/cases/**/*.test.js"], + globalSetup: ["acceptance/lib/image.js"], + // Each case launches a container plus a chromium instance (~200MB). + pool: "forks", + fileParallelism: false, + // A cold `docker build` has to fit in globalSetup. + hookTimeout: 600_000, + testTimeout: 120_000, + }, +});